<dev>

1. 回归打包

<dev>
1. 回归打包
f83070c1 · DeleMing · 57a5815e · f83070c1 · f83070c1 · f83070c1
Commit f83070c1 authored Nov 09, 2020 by DeleMing
5 changed files
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -57,4 +57,4 @@ build_release:
  script:
    - 'bash /cicd-script/maven/maven_clean.sh'
    - 'bash /cicd-script/maven/maven_deploy.sh'
-    - 'bash /cicd-script/maven/tag_release.sh transaction-log-desensitization'
+    - 'bash /cicd-script/maven/tag_release.sh transaction_log_desensitization'
--- a/pom.xml
+++ b/pom.xml
@@ -225,7 +225,6 @@
    </dependencies>

    <build>
-        <finalName>${project.name}-${project.version}</finalName>
        <plugins>
            <!-- Java Compiler -->
            <plugin>
@@ -285,7 +284,7 @@
                    </execution>
                </executions>
            </plugin>
-            <!-- 资源整合 -->
+            <!--&lt;!&ndash; 资源整合 &ndash;&gt;
            <plugin>
                <artifactId>maven-resources-plugin</artifactId>
                <version>2.6</version>
@@ -359,7 +358,7 @@
                        </configuration>
                    </execution>
                </executions>
-            </plugin>
+            </plugin>-->
            <!-- 打包插件 -->
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
@@ -375,9 +374,11 @@
                    <!--                        <descriptorRef>jar-with-dependencies</descriptorRef>-->
                    <!--                    </descriptorRefs>-->
                    <descriptors>
-                        <descriptor>package/assemblies/package.xml</descriptor>
+                        <descriptor>src/assemblies/package.xml</descriptor>
                    </descriptors>
+                    <finalName>${project.artifactId}</finalName>
                </configuration>
+
                <executions>
                    <!--                    <execution>-->
                    <!--                        <id>jar-with-dependencies</id>-->

--- a/src/assemblies/package.xml
+++ b/src/assemblies/package.xml
+<assembly>
+    <id>bin</id>
+    <!-- 最终打包成一个用于发布的tar.gz文件 -->
+    <formats>
+        <format>tar.gz</format>
+    </formats>
+
+    <fileSets>
+
+        <fileSet>
+            <directory>${project.build.scriptSourceDirectory}</directory>
+            <outputDirectory/>
+        </fileSet>
+
+        <fileSet>
+            <directory>${project.basedir}/target/classes/bin</directory>
+            <outputDirectory>${file.separator}bin</outputDirectory>
+            <fileMode>0755</fileMode>
+            <includes>
+                <include>*.sh</include>
+                <include>*.bat</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>${project.basedir}/target/classes</directory>
+            <outputDirectory>${file.separator}conf</outputDirectory>
+            <fileMode>0755</fileMode>
+            <includes>
+                <include>*.yml</include>
+                <include>*.properties</include>
+                <include>regular</include>
+            </includes>
+        </fileSet>
+
+        <!-- 把项目自己编译出来的jar文件，打包进zip文件的lib根目录 -->
+        <fileSet>
+            <directory>${project.basedir}/target</directory>
+            <outputDirectory>/lib</outputDirectory>
+            <includes>
+                <include>transaction-log-desensitization-*.jar</include>
+            </includes>
+        </fileSet>
+
+        <!-- 打包环境变量配置文件 -->
+        <fileSet>
+            <directory>package/env</directory>
+            <outputDirectory>${file.separator}</outputDirectory>
+            <includes>
+                <include>.env</include>
+            </includes>
+        </fileSet>
+
+    </fileSets>
+</assembly>
--- a/src/main/resources/bin/start.sh
+++ b/src/main/resources/bin/start.sh
+#!/usr/bin/env bash
+
+FLINK_TASK_CONF=application.yml
+REGULAR_TASK_NAME=regular
+
+export BASE_PATH=$(cd `dirname $0`; pwd)
+DEPLOY_PATH=${BASE_PATH%/*}
+if [ ! -d "$DEPLOY_PATH/logs" ]; then
+  mkdir -p $DEPLOY_PATH/logs
+fi
+
+flink run -d -c com.zorkdata.desensitization.TransactionLogDesensitization $DEPLOY_PATH/lib/transaction-log-desensitization-0.1.jar --conf $DEPLOY_PATH/conf/$FLINK_TASK_CONF --regular $DEPLOY_PATH/conf/$REGULAR_TASK_NAME > $DEPLOY_PATH/logs/submit.log &
--- a/src/main/resources/bin/start_and_download.sh
+++ b/src/main/resources/bin/start_and_download.sh
+#!/usr/bin/env bash
+
+# 必须修改
+CDH_HOST_IP=192.168.70.2
+CDH_HOST_USER=root
+CDH_HOST_PASSWORD=password
+
+# 与配置文件保持一致
+HDFS_DEST=/tmp/datawarehouse/jzjy/kcbp_biz_log/output1/
+DOWNLOAD_PATH=/tmp/
+JOB_NAME="国泰交易日志脱敏job"
+
+FLINK_TASK_CONF=application.yml
+REGULAR_TASK_NAME=regular
+SCP_PATH=$DOWNLOAD_PATH$(echo $HDFS_DEST|rev |cut -d '/' -f 2 | rev)/
+LOCAL_IP=$(ip a |grep inet| grep -v inet6 | grep -v 127 | cut -d '/' -f1 | cut -d ' ' -f6)
+
+export BASE_PATH=$(cd `dirname $0`; pwd)
+DEPLOY_PATH=${BASE_PATH%/*}
+if [ ! -d "$DEPLOY_PATH/logs" ]; then
+  mkdir -p $DEPLOY_PATH/logs
+fi
+
+# 取值配置文件
+IFS=': '
+while read k v
+do
+  if [[ "$k" == "hdfs_dest" ]]
+  then
+    HDFS_DEST=$(echo "$v" | sed -r 's/.*"(.+)".*/\1/')
+  fi
+  if [[ "$k" == "download_path" ]]
+  then
+    DOWNLOAD_PATH=$(echo "$v" | sed -r 's/.*"(.+)".*/\1/')
+  fi
+  if [[ "$k" == "job_name" ]]
+  then
+    JOB_NAME=$(echo "$v" | sed -r 's/.*"(.+)".*/\1/')
+  fi
+  if [[ "$k" == "cdh_host_ip" ]]
+  then
+    CDH_HOST_IP=$(echo "$v" | sed -r 's/.*"(.+)".*/\1/')
+  fi
+  if [[ "$k" == "cdh_host_user" ]]
+  then
+    CDH_HOST_USER=$(echo "$v" | sed -r 's/.*"(.+)".*/\1/')
+  fi
+  if [[ "$k" == "cdh_host_password" ]]
+  then
+    CDH_HOST_PASSWORD=$(echo "$v" | sed -r 's/.*"(.+)".*/\1/')
+  fi
+done < $DEPLOY_PATH/conf/$FLINK_TASK_CONF
+
+echo $HDFS_DEST
+echo $DOWNLOAD_PATH
+echo $JOB_NAME
+echo $CDH_HOST_IP
+echo $CDH_HOST_USER
+echo $CDH_HOST_PASSWORD
+
+
+flink run -d -c com.zorkdata.desensitization.TransactionLogDesensitization $DEPLOY_PATH/lib/transaction-log-desensitization-0.1.jar --conf $DEPLOY_PATH/conf/$FLINK_TASK_CONF --regular $DEPLOY_PATH/conf/$REGULAR_TASK_NAME > $DEPLOY_PATH/logs/submit.log &
+
+sleep 10
+
+TASK_STATUS=$(cat $DEPLOY_PATH/logs/submit.log |grep "Job has been submitted with JobID")
+if [[ "$TASK_STATUS" == "" ]]
+then
+  echo "提交任务失败"
+  exit 8
+fi
+
+download(){
+expect <<EOF
+  set timeout 10
+  spawn ssh $CDH_HOST_USER@$CDH_HOST_IP
+  expect {
+          "yes/no" { send "yes\n";exp_continue }
+          "password:" { send "$CDH_HOST_PASSWORD\n" }
+  }
+  expect "]# " { send "sudo -u hdfs hadoop fs -copyToLocal $HDFS_DEST $DOWNLOAD_PATH\n" }
+  expect "]# " {
+          send "scp -r $SCP_PATH root@$LOCAL_IP:/tmp/\n"
+          expect {
+                  "yes/no" { send "yes\n";exp_continue }
+                  "password: " { send "$CDH_HOST_PASSWORD\n" }
+          }
+          expect "]# " { send "exit\n" }
+  }
+  expect "]# " { send "exit\n" }
+EOF
+}
+
+while :
+do
+  FLINK_LIST_RUNNING=$(flink list -r )
+  FLAG=$(echo $FLINK_LIST_RUNNING | grep "$JOB_NAME")
+  if [[ "$FLAG" == "" ]]
+  then
+    download
+    break
+  fi
+done
\ No newline at end of file