<dev>

1. 修复当日志路径为“日期/系统简称/日志”这中格式时，启动脱敏任务失败

<dev>
1. 修复当日志路径为“日期/系统简称/日志”这中格式时，启动脱敏任务失败
478d6132 · DeleMing · 0c577ddd · 478d6132 · 478d6132
Commit 478d6132 authored Mar 09, 2021 by DeleMing
2 changed files
--- a/src/main/java/com/zorkdata/desensitization/hadoop/HdfsLogDesensitization.java
+++ b/src/main/java/com/zorkdata/desensitization/hadoop/HdfsLogDesensitization.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.mapred.FileOutputFormat;
 import org.apache.hadoop.mapred.JobConf;

@@ -104,7 +103,7 @@ public class HdfsLogDesensitization implements Serializable {
     * @param hdfsUser hfs用户名 eg: hdfs
     * @return hdfs日志文件的所有文件路径
     */
-    private List<String> filterHdfsLogFiles(String hdfsSrc, String hdfsUri, String hdfsUser) {
+    public List<String> filterHdfsLogFiles(String hdfsSrc, String hdfsUri, String hdfsUser) {
        if (!hdfsSrc.endsWith(GeneralConstants.FILE_SEPARATOR)) {
            hdfsSrc += GeneralConstants.FILE_SEPARATOR;
        }
@@ -146,7 +145,6 @@ public class HdfsLogDesensitization implements Serializable {
        } else {
            log.warn("{} -- {} 时间段内无数据,请注意时间范围", jobConfig.getStartTime(), jobConfig.getEndTime());
        }
-
        return logFiles;
    }


--- a/src/test/java/com/zorkdata/desensitization/hadoop/HdfsLogDesensitizationTest.java
+++ b/src/test/java/com/zorkdata/desensitization/hadoop/HdfsLogDesensitizationTest.java
+package com.zorkdata.desensitization.hadoop;
+
+import com.alibaba.fastjson.JSON;
+import com.zorkdata.desensitization.config.JobConfig;
+import com.zorkdata.desensitization.config.JobInitConfig;
+import com.zorkdata.desensitization.utils.YmlUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * @author: LiaoMingtao
+ * @date: 2021/3/9
+ */
+@Slf4j
+public class HdfsLogDesensitizationTest {
+
+    @Test
+    public void filterHdfsLogFiles() throws Exception {
+        String[] args = new String[]{"--conf", "E:\\Codes\\fork\\transaction-log-desensitization\\src\\main\\resources\\application.yml"};
+        Map<String, String> conf = YmlUtils.getParams(args);
+        JobInitConfig jobInitConfig = new JobInitConfig(conf);
+        JobConfig jobConfig = new JobConfig(jobInitConfig);
+        HdfsLogDesensitization hdfsLogDesensitization = new HdfsLogDesensitization().initJobConfig(jobConfig);
+        List<String> strings = hdfsLogDesensitization
+                .filterHdfsLogFiles(jobConfig.getHdfsSrc(), jobConfig.getHdfsUri(), jobConfig.getHdfsUser());
+        log.info(JSON.toJSONString(strings));
+    }
+}
\ No newline at end of file