Skip to content

T
transactionLogMask
Commit 243313f1 authored by 王海鹰's avatar 王海鹰
4 Browse files
Options

脱敏保留源数据格式； 

代码规范
parent eff193d3
Pipeline #14540 failed with stages
          in 46 seconds
          Hide whitespace changes
          Inline Side-by-side
          Showing with 37 additions and 29 deletions
          src/main/java/com/zorkdata/datamask/TransactionLogMask.java
          View file @ 243313f1
          ...@@ -37,5 +37,4 @@ public class TransactionLogMask { ...@@ -37,5 +37,4 @@ public class TransactionLogMask {
          KafkaMsgMaskUtil.maskKafkaMsg(conf); KafkaMsgMaskUtil.maskKafkaMsg(conf);
          } }
          } }
          } }
          src/main/java/com/zorkdata/datamask/constant/ParamConstants.java
          View file @ 243313f1
          package com.zorkdata.datamask.constant; package com.zorkdata.datamask.constant;
          import java.util.Date;
          /** /**
          * Description : * Description: 查询参数常量
          * *
          * @author : wanghaiying (<a href="wanghaiying@zorkdata.com.cn">wanghaiying@zorkdata.com.cn</a>) * @author: wanghaiying (<a href="wanghaiying@zorkdata.com.cn">wanghaiying@zorkdata.com.cn</a>)
          * Date : Create in 2020/10/20 15:32 * Date: Create in 2020/10/20 15:32
          */ */
          public interface ParamConstants { public interface ParamConstants {
          ......
          src/main/java/com/zorkdata/datamask/constant/RegExpConstants.java
          View file @ 243313f1
          package com.zorkdata.datamask.constant; package com.zorkdata.datamask.constant;
          /** /**
          * Description : * Description : 正则表达式常量
          * *
          * @author : wanghaiying (<a href="wanghaiying@zorkdata.com.cn">wanghaiying@zorkdata.com.cn</a>) * @author : wanghaiying (<a href="wanghaiying@zorkdata.com.cn">wanghaiying@zorkdata.com.cn</a>)
          * Date : Create in 2020/10/20 15:32 * Date : Create in 2020/10/20 15:32
          ......
          src/main/java/com/zorkdata/datamask/constant/StrConstants.java
          View file @ 243313f1
          ...@@ -10,6 +10,4 @@ public interface StrConstants { ...@@ -10,6 +10,4 @@ public interface StrConstants {
          String FILE_SEPARATOR = "/"; String FILE_SEPARATOR = "/";
          String AVRO_SUFFIX = ".avro"; String AVRO_SUFFIX = ".avro";
          String EMPTY_STR = ""; String EMPTY_STR = "";
          } }
          src/main/java/com/zorkdata/datamask/domain/LogData.java
          View file @ 243313f1
          package com.zorkdata.datamask.domain; package com.zorkdata.datamask.domain;
          import lombok.Data; import lombok.Data;
          import org.apache.avro.mapred.AvroWrapper;
          import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.io.WritableComparable;
          import java.io.DataInput; import java.io.DataInput;
          ...@@ -12,7 +11,7 @@ import java.util.Map; ...@@ -12,7 +11,7 @@ import java.util.Map;
          /** /**
          * @author wanghaiying * @author wanghaiying
          * @Description LogData * @Description 日志实体类
          * @Email wanghaiying@zork.com.cn * @Email wanghaiying@zork.com.cn
          * @Date 2020/9/25 10:00 * @Date 2020/9/25 10:00
          */ */
          ...@@ -62,11 +61,4 @@ public class LogData implements Serializable, WritableComparable { ...@@ -62,11 +61,4 @@ public class LogData implements Serializable, WritableComparable {
          @Override @Override
          public void readFields(DataInput dataInput) throws IOException { public void readFields(DataInput dataInput) throws IOException {
          } }
          // @Override
          // public String toString() {
          // return new DateTime(timestamp).toDate().getTime() + " ZorkLogData{" + "logTypeName='" + logTypeName + '\'' + ", timestamp='" + timestamp + '\'' + ", source='"
          // + source + '\'' + ", offset='" + offset + '\'' + ", dimensions=" + dimensions + ", measures=" + measures
          // + ", normalFields=" + normalFields + '}';
          // }
          } }
          src/main/java/com/zorkdata/datamask/hadoop/HdfsLogMaskUtil.java
          View file @ 243313f1
          ...@@ -102,7 +102,7 @@ public class HdfsLogMaskUtil { ...@@ -102,7 +102,7 @@ public class HdfsLogMaskUtil {
          //根据日志事件的timestamp做过滤 //根据日志事件的timestamp做过滤
          Long timestamp = DateUtils.utc2timestamp(logData.getTimestamp()); Long timestamp = DateUtils.utc2timestamp(logData.getTimestamp());
          boolean flag = null != timestamp && timestamp > hdfsLogQueryParam.getStartTime() boolean flag = null != timestamp && timestamp > hdfsLogQueryParam.getStartTime()
          && timestamp < hdfsLogQueryParam.getEndTime() || Boolean.TRUE; && timestamp < hdfsLogQueryParam.getEndTime();
          if (flag) { if (flag) {
          Map maskResult = maskUtil.mask(logData.getNormalFields(), fieldsWhiteList); Map maskResult = maskUtil.mask(logData.getNormalFields(), fieldsWhiteList);
          ...@@ -132,7 +132,6 @@ public class HdfsLogMaskUtil { ...@@ -132,7 +132,6 @@ public class HdfsLogMaskUtil {
          return tupple; return tupple;
          } }
          }).output(hadoopOutputFormat); }).output(hadoopOutputFormat);
          try { try {
          env.execute("国泰交易日志脱敏job"); env.execute("国泰交易日志脱敏job");
          } catch (Exception e) { } catch (Exception e) {
          ......
          src/main/java/com/zorkdata/datamask/util/MaskUtil.java
          View file @ 243313f1
          ...@@ -17,6 +17,17 @@ public class MaskUtil implements Serializable { ...@@ -17,6 +17,17 @@ public class MaskUtil implements Serializable {
          public static final int DEFAULT_MAP_CAPACITY = 16; public static final int DEFAULT_MAP_CAPACITY = 16;
          /**
          * 数据格式信息
          */
          //todo 抽取到配置文件
          • sonarqube @sonarqube ·

            ⚠ 字段【dataFormats】必须使用javadoc形式的注释 📘 ℹ Complete the task associated to this TODO comment. 📘

          Please register or sign in to reply
          private List<String> dataFormats = new ArrayList<String>(){{
          • sonarqube @sonarqube ·

            ⚠ Move the contents of this initializer to a standard constructor or to field initializers. 📘 🔽 Use another way to initialize this instance. 📘

          Please register or sign in to reply
          add(",");
          add(".");
          add("@");
          add("-");
          }};
          /** /**
          * 姓名正则 * 姓名正则
          */ */
          ...@@ -88,7 +99,6 @@ public class MaskUtil implements Serializable { ...@@ -88,7 +99,6 @@ public class MaskUtil implements Serializable {
          patterns.add(Pattern.compile(this.macRegExp)); patterns.add(Pattern.compile(this.macRegExp));
          patterns.add(Pattern.compile(this.emailRegExp)); patterns.add(Pattern.compile(this.emailRegExp));
          patterns.add(Pattern.compile(this.ipRegExp)); patterns.add(Pattern.compile(this.ipRegExp));
          patterns.add(Pattern.compile(this.nameRegExp));
          patterns.add(Pattern.compile(this.idRegExp18)); patterns.add(Pattern.compile(this.idRegExp18));
          patterns.add(Pattern.compile(this.idRegExp15)); patterns.add(Pattern.compile(this.idRegExp15));
          patterns.add(Pattern.compile(this.bankCardRegExp)); patterns.add(Pattern.compile(this.bankCardRegExp));
          ...@@ -104,9 +114,15 @@ public class MaskUtil implements Serializable { ...@@ -104,9 +114,15 @@ public class MaskUtil implements Serializable {
          if (matcher.find()) { if (matcher.find()) {
          String replaceStr = ""; String replaceStr = "";
          for (int i = 0; i < matcher.group().length(); i++) { for (int i = 0; i < matcher.group().length(); i++) {
          replaceStr = replaceStr.concat("*"); String s = String.valueOf(matcher.group().charAt(i));
          if(dataFormats.contains(s)){
          replaceStr = replaceStr.concat(s);
          }else{
          replaceStr = replaceStr.concat("*");
          }
          } }
          value = value.replace(matcher.group(), replaceStr); value = value.replace(matcher.group(), replaceStr);
          System.out.println("\n");
          • sonarqube @sonarqube ·

            ⚠ Replace this use of System.out or System.err by a logger. 📘

          Please register or sign in to reply
          } }
          } }
          map.put(k, value); map.put(k, value);
          ...@@ -118,15 +134,20 @@ public class MaskUtil implements Serializable { ...@@ -118,15 +134,20 @@ public class MaskUtil implements Serializable {
          } }
          public static void main(String[] args) { public static void main(String[] args) {
          MaskUtil maskUtil = new MaskUtil("[\\u4e00-\\u9fa5]{1,20}|[a-zA-Z\\\\.\\\\s]{1,20}", "((13[0-9])|(14[5,7])|(15[0-3,5-9])|(17[0,3,5-8])|(18[0-9])|(147))\\d{8}", MaskUtil maskUtil = new MaskUtil("[\\u4e00-\\u9fa5]{1,20}|[a-zA-Z\\\\.\\\\s]{1,20}",
          "(\\d{3,4}-)?\\d{6,8}", "\\w+([-+.]\\w+)*@\\w+([-.]\\w+)*\\.\\w+([-.]\\w+)*", "[1-9]\\d{7}((0\\d)|(1[0-2]))(([0|1|2]\\d)|3[0-1])\\d{3}", "((13[0-9])|(14[5,7])|(15[0-3,5-9])|(17[0,3,5-8])|(18[0-9])|(147))\\d{8}",
          "[1-9]\\d{5}[1-9]\\d{3}((0\\d)|(1[0-2]))(([0|1|2]\\d)|3[0-1])\\d{3}([0-9Xx])", "([1-9]{1})(\\d{11}|\\d{15}|\\d{16}|\\d{17}|\\d{18})", "0\\d{2,3}-\\d{7,8}",
          "([\u4E00-\u9FA5A-Za-z0-9_]+(省|市|区|县|道|路|街|号|弄|条|室)){2,}", "((2[0-4]\\d|25[0-5]|[01]?\\d\\d?)\\.){3}(2[0-4]\\d|25[0-5]|[01]?\\d\\d?)", "[a-zA-Z0-9]+@[a-zA-Z0-9]+(\\.[a-zA-Z0-9]+)+",
          "[1-9]\\d{7}((0\\d)|(1[0-2]))(([0|1|2]\\d)|3[0-1])\\d{3}",
          "[1-9]\\d{5}[1-9]\\d{3}((0\\d)|(1[0-2]))(([0|1|2]\\d)|3[0-1])\\d{3}([0-9Xx])",
          "([1-9]{1})(\\d{11}|\\d{15}|\\d{16}|\\d{17}|\\d{18})",
          "([\u4E00-\u9FA5A-Za-z0-9_]+(省|市|区|县|道|路|街|号|弄|条|室)){2,}",
          "((2[0-4]\\d|25[0-5]|[01]?\\d\\d?)\\.){3}(2[0-4]\\d|25[0-5]|[01]?\\d\\d?)",
          "([A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}"); "([A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}");
          Map map = new HashMap(DEFAULT_MAP_CAPACITY); Map map = new HashMap(DEFAULT_MAP_CAPACITY);
          map.put("姓名", "王海鹰"); map.put("姓名", "王海鹰");
          map.put("身份证号", "372925199008075158"); map.put("身份证号", "372925199101195158");
          map.put("手机号", "15000101879"); map.put("手机号", "15000101879");
          map.put("电话", "021-61341606"); map.put("电话", "021-61341606");
          map.put("邮箱", "wanghaiying@zork.com"); map.put("邮箱", "wanghaiying@zork.com");
          ...@@ -137,6 +158,7 @@ public class MaskUtil implements Serializable { ...@@ -137,6 +158,7 @@ public class MaskUtil implements Serializable {
          map.put("message", "王海鹰,372925199008075158#15000101879"); map.put("message", "王海鹰,372925199008075158#15000101879");
          map.put("messid", "0000011404342B32233DDCDA"); map.put("messid", "0000011404342B32233DDCDA");
          map.put("bsflag", "0000011404342B32233DDCDA"); map.put("bsflag", "0000011404342B32233DDCDA");
          map.put("test", "wanghaiying123");
          map.put("normalFields", "13811110000-110101199003075517-上海市浦东新区张江微电子港-zorkdata@163.com-123456789-wanghaiying123-王海鹰-192.168.1.1-00-50-56-C0-00-08-6227002470170278192"); map.put("normalFields", "13811110000-110101199003075517-上海市浦东新区张江微电子港-zorkdata@163.com-123456789-wanghaiying123-王海鹰-192.168.1.1-00-50-56-C0-00-08-6227002470170278192");
          String[] fieldsWhiteListArray = "messid,fundid,custid,orgid,brhid,secuid,bankcode,market,ordersno,ordergroup,count,poststr,stkcode,bsflag,orderamt,price,qty,bankcode,tacode,ofcode,transacc,taacc".split(","); String[] fieldsWhiteListArray = "messid,fundid,custid,orgid,brhid,secuid,bankcode,market,ordersno,ordergroup,count,poststr,stkcode,bsflag,orderamt,price,qty,bankcode,tacode,ofcode,transacc,taacc".split(",");
          ......
          src/main/resources/application.yml
          View file @ 243313f1
          ...@@ -33,9 +33,9 @@ reg_exp: ...@@ -33,9 +33,9 @@ reg_exp:
          # 手机号正则 # 手机号正则
          mobile: "((13[0-9])|(14[5,7])|(15[0-3,5-9])|(17[0,3,5-8])|(18[0-9])|(147))\\d{8}" mobile: "((13[0-9])|(14[5,7])|(15[0-3,5-9])|(17[0,3,5-8])|(18[0-9])|(147))\\d{8}"
          # 电话号码正则 # 电话号码正则
          phone: "(\\d{3,4}-)?\\d{6,8}" phone: "0\\d{2,3}-\\d{7,8}"
          # 邮箱正则 # 邮箱正则
          email: "\\w+([-+.]\\w+)*@\\w+([-.]\\w+)*\\.\\w+([-.]\\w+)*" email: "[a-zA-Z0-9]+@[a-zA-Z0-9]+(\\.[a-zA-Z0-9]+)+"
          # 身份证号码（15位）正则 # 身份证号码（15位）正则
          id15: "[1-9]\\d{7}((0\\d)|(1[0-2]))(([0|1|2]\\d)|3[0-1])\\d{3}" id15: "[1-9]\\d{7}((0\\d)|(1[0-2]))(([0|1|2]\\d)|3[0-1])\\d{3}"
          # 身份证号码（18位）正则 # 身份证号码（18位）正则
          ......
          • sonarqube @sonarqube ·

            SonarQube analysis reported 153 issues

            • 🚫 20 critical
            • ⚠ 82 major
            • 🔽 49 minor
            • ℹ 2 info

            Watch the comments in this conversation to review them.

            Top 30 extra issues

            Note: The following issues were found on lines that were not modified in the commit. Because these issues can't be reported as line comments, they are summarized here:

            1. 🚫 Move constants to a class or enum. 📘
            2. 🚫 Move constants to a class or enum. 📘
            3. 🚫 Move constants to a class or enum. 📘
            4. 🚫 Add a nested comment explaining why this method is empty, throw an UnsupportedOperationException or complete the implementation. 📘
            5. 🚫 Change this "try" to a try-with-resources. (sonar.java.source not set. Assuming 7 or greater.) 📘
            6. 🚫 Refactor this code to not throw exceptions in finally blocks. 📘
            7. 🚫 Refactor this code to not throw exceptions in finally blocks. 📘
            8. 🚫 Make "patterns" private or transient. 📘
            9. 🚫 Refactor this method to reduce its Cognitive Complexity from 22 to the 15 allowed. 📘
            10. 🚫 Define a constant instead of duplicating this literal "序列化失败" 15 times. 📘
            11. 🚫 Define a constant instead of duplicating this literal " {\n" 7 times. 📘
            12. 🚫 [Define a constant instead of duplicating this literal " "type": \n" 7 times. 📘
            13. 🚫 Define a constant instead of duplicating this literal " "string",\n" 4 times. 📘
            14. 🚫 Define a constant instead of duplicating this literal " "null"\n" 4 times. 📘
            15. 🚫 [Define a constant instead of duplicating this literal " ]\n" 7 times.](https://git.zorkdata.com/wanghaiying/transactionlogmask/blob/243313f1dcd57649e5d99095a290c497c8063b07/src/main/java/com/zorkdata/datamask/util/avro/LogAvroMacroDef.java#L20) 📘
            16. 🚫 Define a constant instead of duplicating this literal " },\n" 6 times. 📘
            17. 🚫 Define a constant instead of duplicating this literal " "null",\n" 3 times. 📘
            18. 🚫 Define a constant instead of duplicating this literal " {\n" 3 times. 📘
            19. 🚫 Define a constant instead of duplicating this literal " "type": "map",\n" 3 times. 📘
            20. 🚫 Define a constant instead of duplicating this literal " }\n" 3 times. 📘
            21. ⚠ Define and throw a dedicated exception instead of using a generic one. 📘
            22. ⚠ Remove this unused "source" private field. 📘
            23. ⚠ Remove this unused "hdfsSrc" private field. 📘
            24. ⚠ Remove this unused "hdfsDest" private field. 📘
            25. ⚠ Remove this unused "core" private field. 📘
            26. ⚠ Remove this unused "date" private field. 📘
            27. ⚠ Remove this unused "startTime" private field. 📘
            28. ⚠ Remove this unused "endTime" private field. 📘
            29. ⚠ Remove this unused "servers" private field. 📘
            30. ⚠ Remove this unused "zookeeper" private field. 📘
            • ... 118 more
          Markdown is supported
          0% or
          You are about to add 0 people to the discussion. Proceed with caution.
          Finish editing this message first!
          Please register or to comment