Skip to content

T
transaction-log-desensitization
Commit c2c1c39c authored by DeleMing's avatar DeleMing
1 Browse files
Options

<dev> 

1. 调整项目名称
2. 删除多余包
parent c1bda46d
Pipeline #18402 failed with stages
      in 3 minutes and 8 seconds
      Hide whitespace changes
      Inline Side-by-side
      Showing with 1 addition and 276 deletions
      package/assemblies/package.xml deleted 100644 → 0
      View file @ c1bda46d
      <assembly>
      <id>bin</id>
      <!-- 最终打包成一个用于发布的tar.gz文件 -->
      <formats>
      <format>tar.gz</format>
      </formats>
      <fileSets>
      <fileSet>
      <directory>${project.build.scriptSourceDirectory}</directory>
      <outputDirectory/>
      </fileSet>
      <fileSet>
      <directory>${project.basedir}/package/scripts</directory>
      <outputDirectory>${file.separator}bin</outputDirectory>
      <fileMode>0755</fileMode>
      <includes>
      <include>*.sh</include>
      <include>*.bat</include>
      </includes>
      </fileSet>
      <fileSet>
      <directory>${project.basedir}/src/main/resources</directory>
      <outputDirectory>${file.separator}conf</outputDirectory>
      <fileMode>0755</fileMode>
      <includes>
      <include>*.yml</include>
      <include>*.properties</include>
      <include>regular</include>
      </includes>
      </fileSet>
      <!-- 把项目自己编译出来的jar文件，打包进zip文件的lib根目录 -->
      <fileSet>
      <directory>${project.build.directory}</directory>
      <outputDirectory>/lib</outputDirectory>
      <includes>
      <include>*.jar</include>
      </includes>
      </fileSet>
      <!-- 打包环境变量配置文件 -->
      <fileSet>
      <directory>package/env</directory>
      <outputDirectory>${file.separator}</outputDirectory>
      <includes>
      <include>.env</include>
      </includes>
      </fileSet>
      </fileSets>
      </assembly>
      package/scripts/start.sh deleted 100644 → 0
      View file @ c1bda46d
      #!/usr/bin/env bash
      FLINK_TASK_CONF=application.yml
      REGULAR_TASK_NAME=regular
      export BASE_PATH=$(cd `dirname $0`; pwd)
      DEPLOY_PATH=${BASE_PATH%/*}
      if [ ! -d "$DEPLOY_PATH/logs" ]; then
      mkdir -p $DEPLOY_PATH/logs
      fi
      flink run -d -c com.zorkdata.desensitization.TransactionLogDesensitization $DEPLOY_PATH/lib/transaction-log-desensitization-0.1.jar --conf $DEPLOY_PATH/conf/$FLINK_TASK_CONF --regular $DEPLOY_PATH/conf/$REGULAR_TASK_NAME > $DEPLOY_PATH/logs/submit.log &
      package/scripts/start_and_download.sh deleted 100644 → 0
      View file @ c1bda46d
      #!/usr/bin/env bash
      # 必须修改
      CDH_HOST_IP=192.168.70.2
      CDH_HOST_USER=root
      CDH_HOST_PASSWORD=password
      # 与配置文件保持一致
      HDFS_DEST=/tmp/datawarehouse/jzjy/kcbp_biz_log/output1/
      DOWNLOAD_PATH=/tmp/
      JOB_NAME="国泰交易日志脱敏job"
      FLINK_TASK_CONF=application.yml
      REGULAR_TASK_NAME=regular
      SCP_PATH=$DOWNLOAD_PATH$(echo $HDFS_DEST|rev |cut -d '/' -f 2 | rev)/
      LOCAL_IP=$(ip a |grep inet| grep -v inet6 | grep -v 127 | cut -d '/' -f1 | cut -d ' ' -f6)
      export BASE_PATH=$(cd `dirname $0`; pwd)
      DEPLOY_PATH=${BASE_PATH%/*}
      if [ ! -d "$DEPLOY_PATH/logs" ]; then
      mkdir -p $DEPLOY_PATH/logs
      fi
      # 取值配置文件
      IFS=': '
      while read k v
      do
      if [[ "$k" == "hdfs_dest" ]]
      then
      HDFS_DEST=$(echo "$v" | sed -r 's/.*"(.+)".*/\1/')
      fi
      if [[ "$k" == "download_path" ]]
      then
      DOWNLOAD_PATH=$(echo "$v" | sed -r 's/.*"(.+)".*/\1/')
      fi
      if [[ "$k" == "job_name" ]]
      then
      JOB_NAME=$(echo "$v" | sed -r 's/.*"(.+)".*/\1/')
      fi
      if [[ "$k" == "cdh_host_ip" ]]
      then
      CDH_HOST_IP=$(echo "$v" | sed -r 's/.*"(.+)".*/\1/')
      fi
      if [[ "$k" == "cdh_host_user" ]]
      then
      CDH_HOST_USER=$(echo "$v" | sed -r 's/.*"(.+)".*/\1/')
      fi
      if [[ "$k" == "cdh_host_password" ]]
      then
      CDH_HOST_PASSWORD=$(echo "$v" | sed -r 's/.*"(.+)".*/\1/')
      fi
      done < $DEPLOY_PATH/conf/$FLINK_TASK_CONF
      echo $HDFS_DEST
      echo $DOWNLOAD_PATH
      echo $JOB_NAME
      echo $CDH_HOST_IP
      echo $CDH_HOST_USER
      echo $CDH_HOST_PASSWORD
      flink run -d -c com.zorkdata.desensitization.TransactionLogDesensitization $DEPLOY_PATH/lib/transaction-log-desensitization-0.1.jar --conf $DEPLOY_PATH/conf/$FLINK_TASK_CONF --regular $DEPLOY_PATH/conf/$REGULAR_TASK_NAME > $DEPLOY_PATH/logs/submit.log &
      sleep 10
      TASK_STATUS=$(cat $DEPLOY_PATH/logs/submit.log |grep "Job has been submitted with JobID")
      if [[ "$TASK_STATUS" == "" ]]
      then
      echo "提交任务失败"
      exit 8
      fi
      download(){
      expect <<EOF
      set timeout 10
      spawn ssh $CDH_HOST_USER@$CDH_HOST_IP
      expect {
      "yes/no" { send "yes\n";exp_continue }
      "password:" { send "$CDH_HOST_PASSWORD\n" }
      }
      expect "]# " { send "sudo -u hdfs hadoop fs -copyToLocal $HDFS_DEST $DOWNLOAD_PATH\n" }
      expect "]# " {
      send "scp -r $SCP_PATH root@$LOCAL_IP:/tmp/\n"
      expect {
      "yes/no" { send "yes\n";exp_continue }
      "password: " { send "$CDH_HOST_PASSWORD\n" }
      }
      expect "]# " { send "exit\n" }
      }
      expect "]# " { send "exit\n" }
      EOF
      }
      while :
      do
      FLINK_LIST_RUNNING=$(flink list -r )
      FLAG=$(echo $FLINK_LIST_RUNNING | grep "$JOB_NAME")
      if [[ "$FLAG" == "" ]]
      then
      download
      break
      fi
      done
      \ No newline at end of file
      pom.xml
      View file @ c2c1c39c
      ......@@ -78,23 +78,7 @@
      </exclusion>
      </exclusions>
      </dependency>
      <!-- <dependency>-->
      <!-- <groupId>org.apache.flink</groupId>-->
      <!-- <artifactId>flink-shaded-hadoop-2</artifactId>-->
      <!-- <version>2.6.0-10.0</version>-->
      <!-- <scope>provided</scope>-->
      <!-- </dependency>-->
      <!-- <dependency>-->
      <!-- <groupId>org.apache.flink</groupId>-->
      <!-- <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>-->
      <!-- <version>${flink.version}</version>-->
      <!-- </dependency>-->
      <!-- <dependency>-->
      <!-- <groupId>org.apache.flink</groupId>-->
      <!-- <artifactId>flink-connector-kafka_${scala.binary.version}</artifactId>-->
      <!-- <version>${flink.version}</version>-->
      <!-- </dependency>-->
      <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-connector-filesystem_${scala.binary.version}</artifactId>
      ......@@ -284,109 +268,19 @@
      </execution>
      </executions>
      </plugin>
      <!--&lt;!&ndash; 资源整合 &ndash;&gt;
      <plugin>
      <artifactId>maven-resources-plugin</artifactId>
      <version>2.6</version>
      <executions>
      <execution>
      <id>copy-config</id>
      <phase>package</phase>
      <goals>
      <goal>copy-resources</goal>
      </goals>
      <configuration>
      <encoding>${project.build.sourceEncoding}</encoding>
      <outputDirectory>${basedir}/target/transaction-log-deploy/conf
      </outputDirectory>
      <resources>
      <resource>
      <directory>${basedir}/target/classes</directory>
      <excludes>
      <exclude>**/*.java</exclude>
      <exclude>**/*.class</exclude>
      <exclude>**/*.bat</exclude>
      <exclude>**/*.sh</exclude>
      <exclude>**/*.conf</exclude>
      </excludes>
      </resource>
      </resources>
      </configuration>
      </execution>
      <execution>
      <id>copy-script</id>
      <phase>package</phase>
      <goals>
      <goal>copy-resources</goal>
      </goals>
      <configuration>
      <encoding>UTF-8</encoding>
      <outputDirectory>${basedir}/target/transaction-log-deploy/bin
      </outputDirectory>
      <resources>
      <resource>
      <directory>${basedir}/target/classes/bin</directory>
      <filtering>true</filtering>
      <includes>
      <include>*.bat</include>
      <include>*.sh</include>
      <include>*.py</include>
      </includes>
      assembly
      </resource>
      </resources>
      </configuration>
      </execution>
      <execution>
      <id>copy-library</id>
      <phase>package</phase>
      <goals>
      <goal>copy-resources</goal>
      </goals>
      <configuration>
      <outputDirectory>${basedir}/target/transaction-log-deploy/lib
      </outputDirectory>
      <resources>
      <resource>
      <directory>${basedir}/target</directory>
      <includes>
      <include>${project.artifactId}-${project.version}.jar
      </include>
      </includes>
      </resource>
      </resources>
      </configuration>
      </execution>
      </executions>
      </plugin>-->
      <!-- 打包插件 -->
      <plugin>
      <groupId>org.apache.maven.plugins</groupId>
      <artifactId>maven-assembly-plugin</artifactId>
      <version>2.4</version>
      <configuration>
      <!-- <archive>-->
      <!-- <manifest>-->
      <!-- <mainClass>com.zorkdata.desensitization.TransactionLogDesensitization</mainClass>-->
      <!-- </manifest>-->
      <!-- </archive>-->
      <!-- <descriptorRefs>-->
      <!-- <descriptorRef>jar-with-dependencies</descriptorRef>-->
      <!-- </descriptorRefs>-->
      <descriptors>
      <descriptor>src/assemblies/package.xml</descriptor>
      </descriptors>
      <finalName>transaction_log_desensitization</finalName>
      <finalName>transaction-log-desensitization</finalName>
      </configuration>
      <executions>
      <!-- <execution>-->
      <!-- <id>jar-with-dependencies</id>-->
      <!-- <phase>package</phase>-->
      <!-- <goals>-->
      <!-- <goal>single</goal>-->
      <!-- </goals>-->
      <!-- </execution>-->
      <execution>
      <id>make-assembly</id>
      <phase>package</phase>
      ......
      • sonarqube @sonarqube ·

        SonarQube analysis reported 116 issues

        • 🚫 25 critical
        • ⚠ 62 major
        • 🔽 28 minor
        • ℹ 1 info

        Top 30 extra issues

        Note: The following issues were found on lines that were not modified in the commit. Because these issues can't be reported as line comments, they are summarized here:

        1. 🚫 Add a default case to this switch. 📘
        2. 🚫 switch中每个case需要通过break/return等来终止 📘
        3. 🚫 switch块缺少default语句 📘
        4. 🚫 Define a constant instead of duplicating this literal " {\n" 11 times. 📘
        5. 🚫 [Define a constant instead of duplicating this literal " "type": \n" 11 times. 📘
        6. 🚫 Define a constant instead of duplicating this literal " "string",\n" 6 times. 📘
        7. 🚫 Define a constant instead of duplicating this literal " "null"\n" 6 times. 📘
        8. 🚫 [Define a constant instead of duplicating this literal " ]\n" 11 times.](https://git.zorkdata.com/liaomingtao/transaction-log-desensitization/blob/c2c1c39c810193b8e21d983045725a4aa0cf6e3e/src/main/java/com/zorkdata/desensitization/avro/AvroSchemaDef.java#L23) 📘
        9. 🚫 Define a constant instead of duplicating this literal " },\n" 9 times. 📘
        10. 🚫 Define a constant instead of duplicating this literal " "null",\n" 5 times. 📘
        11. 🚫 Define a constant instead of duplicating this literal " {\n" 5 times. 📘
        12. 🚫 Define a constant instead of duplicating this literal " "type": "map",\n" 5 times. 📘
        13. 🚫 Define a constant instead of duplicating this literal " "values": "string"\n" 3 times. 📘
        14. 🚫 Define a constant instead of duplicating this literal " }\n" 5 times. 📘
        15. 🚫 Define a constant instead of duplicating this literal "序列化失败" 13 times. 📘
        16. 🚫 Refactor this method to reduce its Cognitive Complexity from 22 to the 15 allowed. 📘
        17. 🚫 Refactor this method to reduce its Cognitive Complexity from 22 to the 15 allowed. 📘
        18. 🚫 Refactor this method to reduce its Cognitive Complexity from 22 to the 15 allowed. 📘
        19. 🚫 常量【dataFormats】命名应全部大写并以下划线分隔 📘
        20. 🚫 Refactor this method to reduce its Cognitive Complexity from 16 to the 15 allowed. 📘
        21. 🚫 Add a nested comment explaining why this method is empty, throw an UnsupportedOperationException or complete the implementation. 📘
        22. 🚫 Change this "try" to a try-with-resources. (sonar.java.source not set. Assuming 7 or greater.) 📘
        23. 🚫 Refactor this code to not throw exceptions in finally blocks. 📘
        24. 🚫 Refactor this code to not throw exceptions in finally blocks. 📘
        25. 🚫 Change this "try" to a try-with-resources. (sonar.java.source not set. Assuming 7 or greater.) 📘
        26. ⚠ This block of commented-out lines of code should be removed. 📘
        27. ⚠ 及时清理不再使用的代码段或配置信息。 📘
        28. ⚠ Replace this use of System.out or System.err by a logger. 📘
        29. ⚠ Replace this use of System.out or System.err by a logger. 📘
        30. ⚠ String contains no format specifiers. 📘
        • ... 86 more
      Markdown is supported
      0% or
      You are about to add 0 people to the discussion. Proceed with caution.
      Finish editing this message first!
      Please register or to comment