1 #! /bin/bash
2
3 ICMS_PATH=$(cd $(dirname $0); pwd)
4 DEPLOY_PATH=$(cd $(dirname $ICMS_PATH); pwd)
5 JAR_FILE=$DEPLOY_PATH'/dailydata-1.0-SNAPSHOT.jar'
6 INI_FILE=$DEPLOY_PATH'/cnedmp_etl.ini'
7 LOG_FILE=$DEPLOY_PATH'/shellrun.log'
8 CODING='GB2312'
9
10 if [ $# -ne 1 ]; then
11 logging INFO "Ussage: $0 <source file path>"
12 exit 1
13 fi
14
15 #Funtion for Log
16 function logging(){
17 logTime=$(date +"%Y-%m-%d %T")
18 echo "$logTime $1 $2"
19 }
20 source $INI_FILE
21 PASSWORD=`yarn jar $JAR_FILE com.hypers.etl.file.specialFileProcess.BlowFish`
22 SRC_DIR=$1
23 logging INFO "Going to process all the source file under path: $SRC_DIR"
24 cd $SRC_DIR
25 partition=$(echo $SRC_DIR | awk -F/ '{print $(NF-1)}')
26 logging INFO "The partition is $partition"
27 filelist=$(ls $dir)
28 logging INFO "Will handle $filelist under path $SRC_DIR"
29 logging INFO "-----------------------------"
30
31 for filename in $filelist
32 do
33
34 logging INFO "---------------------------"
35 logging INFO "handling file $filename ..."
36
37 PRE_LOCATION=`mysql -h$HOSTNAME -u$USERNAME -p$PASSWORD $DBNAME <<EOF 2> /dev/null | tail -n +2
38 SELECT Pre_Location from $CONFNAME where File_Name='$filename';
39 EOF`
40 year=$(echo $partition | cut -b 1-4)
41 PRE_LOCATION=$PRE_LOCATION/$year
42 logging INFO "get pre file location $PRE_LOCATION for $filename"
43
44 TARG_LOCATION=`mysql -h$HOSTNAME -u$USERNAME -p$PASSWORD $DBNAME <<EOF 2> /dev/null | tail -n +2
45 SELECT Target_Location from $CONFNAME where File_Name='$filename';
46 EOF`
47 TARG_LOCATION=$TARG_LOCATION/$partition
48 logging INFO "get target file location $TARG_LOCATION for $filename"
49
50 TARG_TABLE_NAME=`mysql -h$HOSTNAME -u$USERNAME -p$PASSWORD $DBNAME <<EOF 2> /dev/null | tail -n +2
51 SELECT Trg_Table_Name from $CONFNAME where File_Name='$filename';
52 EOF`
53 logging INFO "get target table name $TARG_TABLE_NAME for $filename"
54
55 hdfs dfs -mkdir -p $PRE_LOCATION && logging INFO "Created $PRE_LOCATION succesfully"
56 hdfs dfs -mkdir -p $TARG_LOCATION && logging INFO "Created $TARG_LOCATION succesfully"
57
58 hdfs dfs -put $filename $PRE_LOCATION && logging INFO "Put $filename onto hdfs prelocation $PRE_LOCATION"
59
60 yarn jar $JAR_FILE com.hypers.etl.file.mr.CLIDriver -i ${PRE_LOCATION}'/'${filename} -o ${TARG_LOCATION} -e ${CODING} -d ${partition} -s ','
61
62 logging INFO "alter table $TARG_TABLE_NAME to add partition"
63 #sudo su hdfs -c"beeline -u jdbc:hive2://${HOSTNAME}:10000/cnedmp -n hdfs -p $HIVE_SERVER2_PASSWORD -e "alter table $TARG_TABLE_NAME add partition dt = '${partition}' location '${TARG_LOCATION}';" "
64
65 done
