Skip to content

T
transaction-log-desensitization
Commit 1400f1ee authored by DeleMing's avatar DeleMing
5 Browse files
Options

<dev> 

1. 修改问题
parent e390bb16
Pipeline #21555 passed with stages
      in 3 minutes and 19 seconds
      Expand all Hide whitespace changes
      Inline Side-by-side
      Showing with 210 additions and 27 deletions
      src/main/java/com/zorkdata/desensitization/constans/GeneralConstants.java
      View file @ 1400f1ee
      ...@@ -22,5 +22,7 @@ public final class GeneralConstants { ...@@ -22,5 +22,7 @@ public final class GeneralConstants {
      public static final String SYMBOL_STR = "&"; public static final String SYMBOL_STR = "&";
      public static final String CODE_MSG = "CODE,MSG;"; public static final String CODE_MSG = "CODE,MSG;";
      public static final String SEMICOLON = ";"; public static final String SEMICOLON = ";";
      public static final int NUM2 = 2;
      public static final int NUM4 = 4;
      } }
      src/main/java/com/zorkdata/desensitization/function/DesensitizationFunction.java
      View file @ 1400f1ee
      ...@@ -110,9 +110,21 @@ public class DesensitizationFunction<T, R> extends RichFlatMapFunction<Tuple2<Ob ...@@ -110,9 +110,21 @@ public class DesensitizationFunction<T, R> extends RichFlatMapFunction<Tuple2<Ob
      if (sourceMessage.contains(GeneralConstants.CODE_MSG)) { if (sourceMessage.contains(GeneralConstants.CODE_MSG)) {
      String[] ansArrTemp = sourceMessage.split(GeneralConstants.CODE_MSG); String[] ansArrTemp = sourceMessage.split(GeneralConstants.CODE_MSG);
      String ansArr = ansArrTemp[1]; String ansArr = ansArrTemp[1];
      int commaIndex = ansArr.indexOf(GeneralConstants.SEMICOLON); // 此处还有其他情况
      String keysString = ansArr.substring(0, commaIndex); String[] ans = ansArr.split(GeneralConstants.SEMICOLON);
      String valuesString = ansArr.substring(commaIndex); String keysString = null;
      String valuesString = null;
      if (ans.length >= GeneralConstants.NUM4) {
      keysString = ans[2];
      valuesString = ans[3];
      } else {
      int commaIndex = ansArr.indexOf(GeneralConstants.SEMICOLON);
      keysString = ansArr.substring(0, commaIndex);
      valuesString = ansArr.substring(commaIndex);
      if (ansArr.length() > commaIndex) {
      valuesString = ansArr.substring(commaIndex + 1);
      }
      }
      String[] keysArr = keysString.split(GeneralConstants.COMMA); String[] keysArr = keysString.split(GeneralConstants.COMMA);
      List<Integer> valueIndex = new ArrayList<>(); List<Integer> valueIndex = new ArrayList<>();
      for (int i = 0; i < keysArr.length; i++) { for (int i = 0; i < keysArr.length; i++) {
      ...@@ -121,11 +133,20 @@ public class DesensitizationFunction<T, R> extends RichFlatMapFunction<Tuple2<Ob ...@@ -121,11 +133,20 @@ public class DesensitizationFunction<T, R> extends RichFlatMapFunction<Tuple2<Ob
      valueIndex.add(i); valueIndex.add(i);
      } }
      } }
      messageSuffixDesc if (ans.length >= GeneralConstants.NUM4) {
      .append(ansArrTemp[0]) messageSuffixDesc
      .append(GeneralConstants.CODE_MSG) .append(ansArrTemp[0])
      .append(ansArrTemp[1].split(GeneralConstants.SEMICOLON)[0]) .append(GeneralConstants.CODE_MSG)
      .append(GeneralConstants.COMMA); .append(ans[0]).append(GeneralConstants.SEMICOLON)
      .append(ans[1]).append(GeneralConstants.SEMICOLON)
      .append(ans[2]).append(GeneralConstants.SEMICOLON);
      } else {
      messageSuffixDesc
      .append(ansArrTemp[0])
      .append(GeneralConstants.CODE_MSG)
      .append(ansArrTemp[1].split(GeneralConstants.SEMICOLON)[0])
      .append(GeneralConstants.SEMICOLON);
      }
      String[] valueArr = valuesString.split(GeneralConstants.COMMA); String[] valueArr = valuesString.split(GeneralConstants.COMMA);
      int valueSize = valueArr.length / keysArr.length; int valueSize = valueArr.length / keysArr.length;
      List<Integer> ansValueIndex = getAnsValueIndex(valueIndex, valueSize, keysArr.length); List<Integer> ansValueIndex = getAnsValueIndex(valueIndex, valueSize, keysArr.length);
      ...@@ -142,6 +163,12 @@ public class DesensitizationFunction<T, R> extends RichFlatMapFunction<Tuple2<Ob ...@@ -142,6 +163,12 @@ public class DesensitizationFunction<T, R> extends RichFlatMapFunction<Tuple2<Ob
      messageSuffixDesc.append(value).append(GeneralConstants.COMMA); messageSuffixDesc.append(value).append(GeneralConstants.COMMA);
      } }
      } }
      if (ans.length >= GeneralConstants.NUM4) {
      messageSuffixDesc.append(GeneralConstants.SEMICOLON);
      for (int i = 4; i < ans.length; i++) {
      messageSuffixDesc.append(ans[i]).append(GeneralConstants.SEMICOLON);
      }
      }
      } else { } else {
      log.error("message不包含CODE,MSG;"); log.error("message不包含CODE,MSG;");
      messageSuffixDesc.append(messageSuffixDesc); messageSuffixDesc.append(messageSuffixDesc);
      ...@@ -183,11 +210,20 @@ public class DesensitizationFunction<T, R> extends RichFlatMapFunction<Tuple2<Ob ...@@ -183,11 +210,20 @@ public class DesensitizationFunction<T, R> extends RichFlatMapFunction<Tuple2<Ob
      if (messageSuffix.contains(GeneralConstants.CODE_MSG)) { if (messageSuffix.contains(GeneralConstants.CODE_MSG)) {
      String[] ansArrTemp = messageSuffix.split(GeneralConstants.CODE_MSG); String[] ansArrTemp = messageSuffix.split(GeneralConstants.CODE_MSG);
      String ansArr = ansArrTemp[1]; String ansArr = ansArrTemp[1];
      int commaIndex = ansArr.indexOf(GeneralConstants.SEMICOLON); // 此处还有其他情况
      String keysString = ansArr.substring(0, commaIndex); String[] ans = ansArr.split(GeneralConstants.SEMICOLON);
      String valuesString = ansArr.substring(commaIndex); String keysString = null;
      if (ansArr.length() > commaIndex) { String valuesString = null;
      valuesString = ansArr.substring(commaIndex + 1); if (ans.length >= GeneralConstants.NUM4) {
      keysString = ans[2];
      valuesString = ans[3];
      } else {
      int commaIndex = ansArr.indexOf(GeneralConstants.SEMICOLON);
      keysString = ansArr.substring(0, commaIndex);
      valuesString = ansArr.substring(commaIndex);
      if (ansArr.length() > commaIndex) {
      valuesString = ansArr.substring(commaIndex + 1);
      }
      } }
      String[] keysArr = keysString.split(GeneralConstants.COMMA); String[] keysArr = keysString.split(GeneralConstants.COMMA);
      List<Integer> valueIndex = new ArrayList<>(); List<Integer> valueIndex = new ArrayList<>();
      ...@@ -197,11 +233,20 @@ public class DesensitizationFunction<T, R> extends RichFlatMapFunction<Tuple2<Ob ...@@ -197,11 +233,20 @@ public class DesensitizationFunction<T, R> extends RichFlatMapFunction<Tuple2<Ob
      valueIndex.add(i); valueIndex.add(i);
      } }
      } }
      messageSuffixDesc if (ans.length >= GeneralConstants.NUM4) {
      .append(ansArrTemp[0]) messageSuffixDesc
      .append(GeneralConstants.CODE_MSG) .append(ansArrTemp[0])
      .append(ansArrTemp[1].split(GeneralConstants.SEMICOLON)[0]) .append(GeneralConstants.CODE_MSG)
      .append(GeneralConstants.SEMICOLON); .append(ans[0]).append(GeneralConstants.SEMICOLON)
      .append(ans[1]).append(GeneralConstants.SEMICOLON)
      .append(ans[2]).append(GeneralConstants.SEMICOLON);
      } else {
      messageSuffixDesc
      .append(ansArrTemp[0])
      .append(GeneralConstants.CODE_MSG)
      .append(ansArrTemp[1].split(GeneralConstants.SEMICOLON)[0])
      .append(GeneralConstants.SEMICOLON);
      }
      String[] valueArr = valuesString.split(GeneralConstants.COMMA); String[] valueArr = valuesString.split(GeneralConstants.COMMA);
      int valueSize = valueArr.length / keysArr.length; int valueSize = valueArr.length / keysArr.length;
      List<Integer> ansValueIndex = getAnsValueIndex(valueIndex, valueSize, keysArr.length); List<Integer> ansValueIndex = getAnsValueIndex(valueIndex, valueSize, keysArr.length);
      ...@@ -218,6 +263,12 @@ public class DesensitizationFunction<T, R> extends RichFlatMapFunction<Tuple2<Ob ...@@ -218,6 +263,12 @@ public class DesensitizationFunction<T, R> extends RichFlatMapFunction<Tuple2<Ob
      messageSuffixDesc.append(value).append(GeneralConstants.COMMA); messageSuffixDesc.append(value).append(GeneralConstants.COMMA);
      } }
      } }
      if (ans.length >= GeneralConstants.NUM4) {
      messageSuffixDesc.append(GeneralConstants.SEMICOLON);
      for (int i = 4; i < ans.length; i++) {
      messageSuffixDesc.append(ans[i]).append(GeneralConstants.SEMICOLON);
      }
      }
      } else { } else {
      log.error("message不包含CODE,MSG;"); log.error("message不包含CODE,MSG;");
      messageSuffixDesc.append(messageSuffixDesc); messageSuffixDesc.append(messageSuffixDesc);
      ......
      src/main/java/com/zorkdata/desensitization/hadoop/HdfsLogDesensitization.java
      View file @ 1400f1ee
      ...@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.LocatedFileStatus; ...@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.LocatedFileStatus;
      import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
      import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.RemoteIterator;
      import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.NullWritable;
      import org.apache.hadoop.io.compress.CompressionCodec;
      • Ghost User @ghost ·

        🔽 Remove this unused import 'org.apache.hadoop.io.compress.CompressionCodec'. 📘

      Please register or sign in to reply
      import org.apache.hadoop.mapred.FileOutputFormat; import org.apache.hadoop.mapred.FileOutputFormat;
      import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobConf;
      ......
      src/main/resources/application.yml
      View file @ 1400f1ee
      ...@@ -6,8 +6,8 @@ end_time: "2021-02-26 23:59:59" ...@@ -6,8 +6,8 @@ end_time: "2021-02-26 23:59:59"
      # 任务配置 # 任务配置
      job_name: "国泰交易日志脱敏job" job_name: "国泰交易日志脱敏job"
      # 并行度HdfsLogDesensitization # 并行度HdfsLogDesensitization
      source.parallelism: "4" source.parallelism: "1"
      transformer.parallelism: "4" transformer.parallelism: "1"
      sink.parallelism: "1" sink.parallelism: "1"
      # 是否过滤维度中hostname字段，若填*则不过滤，若需要包含如‘c9’的hostname，则填写c9 # 是否过滤维度中hostname字段，若填*则不过滤，若需要包含如‘c9’的hostname，则填写c9
      ...@@ -16,9 +16,9 @@ match.hostname: "*" ...@@ -16,9 +16,9 @@ match.hostname: "*"
      # 密码 # 密码
      password: "trdpwd,newpwd,oldfundpwd,newfundpwd,bankpwd,dynamicpwd1,dynamicpwd2,dynamicpwd,fundpwd,newbankpwd" password: "trdpwd,newpwd,oldfundpwd,newfundpwd,bankpwd,dynamicpwd1,dynamicpwd2,dynamicpwd,fundpwd,newbankpwd"
      # 姓名key # 姓名key
      name: "custname,otherlinkman,longname,YXYHMC,deputyname" name: "custname,otherlinkman,longname,YXYHMC,deputyname,name,fundname"
      # 手机号key # 手机号key
      mobile: "netaddr,telno,mobileno,faxno,hometelno,worktelno,fax,mobil,call" mobile: "netaddr,telno,mobileno,faxno,hometelno,worktelno,fax,mobil,call,g_stationaddr,terminalinfo"
      # 电话号码key # 电话号码key
      phone: "netaddr,telno,mobileno,faxno,hometelno,worktelno,fax,mobil,call" phone: "netaddr,telno,mobileno,faxno,hometelno,worktelno,fax,mobil,call"
      # 邮箱key # 邮箱key
      ...@@ -48,7 +48,7 @@ hdfs_user: "hdfs" ...@@ -48,7 +48,7 @@ hdfs_user: "hdfs"
      # hdfs日志源文件地址，若source为hdfs，则该地址必传，必须以斜杠结尾 # hdfs日志源文件地址，若source为hdfs，则该地址必传，必须以斜杠结尾
      hdfs_src: "/tmp/datawarehouse/net22/" hdfs_src: "/tmp/datawarehouse/net22/"
      # hdfs日志写入地址，非必传，默认写到hdfs-src目录下的output目录下，必须以斜杠结尾 # hdfs日志写入地址，非必传，默认写到hdfs-src目录下的output目录下，必须以斜杠结尾
      hdfs_dest: "/tmp/datawarehouse/net22/output2/" hdfs_dest: "/tmp/datawarehouse/net22/output14/"
      # cdh下载配置 # cdh下载配置
      # cdh能执行hdfs命令的机器的ip # cdh能执行hdfs命令的机器的ip
      ......
      src/main/resources/bin/start_and_download.sh
      View file @ 1400f1ee
      ...@@ -84,7 +84,8 @@ expect <<EOF ...@@ -84,7 +84,8 @@ expect <<EOF
      expect "]# " { send "rm -rf $SCP_PATH\n" } expect "]# " { send "rm -rf $SCP_PATH\n" }
      expect "]# " { send "su $CDH_HDFS_USER\n" } expect "]# " { send "su $CDH_HDFS_USER\n" }
      expect "]$ " { send "hadoop fs -copyToLocal $HDFS_DEST $DOWNLOAD_PATH\n" } expect "]$ " { send "hadoop fs -copyToLocal $HDFS_DEST $DOWNLOAD_PATH\n" }
      expect "]# " { send "exit\n" } expect "]$ " { send "sleep 10\n" }
      expect "]$ " { send "exit\n" }
      expect "]# " { expect "]# " {
      send "scp -r $SCP_PATH root@$LOCAL_IP:/tmp/\n" send "scp -r $SCP_PATH root@$LOCAL_IP:/tmp/\n"
      expect { expect {
      ......
      • Ghost User @ghost ·

        SonarQube analysis reported 193 issues

        • ⛔ 4 blocker
        • 🚫 34 critical
        • ⚠ 131 major
        • 🔽 21 minor
        • ℹ 3 info

        Watch the comments in this conversation to review them.

        Top 30 extra issues

        Note: The following issues were found on lines that were not modified in the commit. Because these issues can't be reported as line comments, they are summarized here:

        1. ⛔ Remove this hard-coded password. 📘
        2. ⛔ Remove this hard-coded password. 📘
        3. 🚫 Define a constant instead of duplicating this literal " {\n" 11 times. 📘
        4. 🚫 [Define a constant instead of duplicating this literal " "type": \n" 11 times. 📘
        5. 🚫 Define a constant instead of duplicating this literal " "string",\n" 6 times. 📘
        6. 🚫 Define a constant instead of duplicating this literal " "null"\n" 6 times. 📘
        7. 🚫 [Define a constant instead of duplicating this literal " ]\n" 11 times.](https://git.zorkdata.com/liaomingtao/transaction-log-desensitization/blob/1400f1ee0c47ad1167852ee0df00d38885e940a2/src/main/java/com/zorkdata/desensitization/avro/AvroSchemaDef.java#L23) 📘
        8. 🚫 Define a constant instead of duplicating this literal " },\n" 9 times. 📘
        9. 🚫 Define a constant instead of duplicating this literal " "null",\n" 5 times. 📘
        10. 🚫 Define a constant instead of duplicating this literal " {\n" 5 times. 📘
        11. 🚫 Define a constant instead of duplicating this literal " "type": "map",\n" 5 times. 📘
        12. 🚫 Define a constant instead of duplicating this literal " "values": "string"\n" 3 times. 📘
        13. 🚫 Define a constant instead of duplicating this literal " }\n" 5 times. 📘
        14. 🚫 Define a constant instead of duplicating this literal "序列化失败" 13 times. 📘
        15. 🚫 Refactor this method to reduce its Cognitive Complexity from 22 to the 15 allowed. 📘
        16. 🚫 Refactor this method to reduce its Cognitive Complexity from 138 to the 15 allowed. 📘
        17. 🚫 Refactor this method to reduce its Cognitive Complexity from 22 to the 15 allowed. 📘
        18. 🚫 Refactor this method to reduce its Cognitive Complexity from 22 to the 15 allowed. 📘
        19. 🚫 Refactor this method to reduce its Cognitive Complexity from 19 to the 15 allowed. 📘
        20. 🚫 Change this "try" to a try-with-resources. (sonar.java.source not set. Assuming 7 or greater.) 📘
        21. 🚫 Refactor this code to not throw exceptions in finally blocks. 📘
        22. 🚫 Refactor this code to not throw exceptions in finally blocks. 📘
        23. 🚫 Change this "try" to a try-with-resources. (sonar.java.source not set. Assuming 7 or greater.) 📘
        24. 🚫 Define a constant instead of duplicating this literal "jobName" 6 times. 📘
        25. 🚫 Define a constant instead of duplicating this literal "2020-11-07 21:22:20" 6 times. 📘
        26. 🚫 Define a constant instead of duplicating this literal "2020-11-07 23:40:30" 6 times. 📘
        27. 🚫 Define a constant instead of duplicating this literal "netaddr,telno,mobileno,faxno,hometelno,worktelno,fax,mobil,call" 6 times. 📘
        28. 🚫 Define a constant instead of duplicating this literal "email" 3 times. 📘
        29. 🚫 Define a constant instead of duplicating this literal "idno,deputyidno" 3 times. 📘
        30. 🚫 Define a constant instead of duplicating this literal "lmbankid" 3 times. 📘
        • ... 159 more
      Markdown is supported
      0% or
      You are about to add 0 people to the discussion. Proceed with caution.
      Finish editing this message first!
      Please register or to comment