使用shell进行etl数据验证

方法如下：

　　整理校验的类型，不同的类型配置文件不一样。

　　　　1：校验数据增量：需要设置表名，增量字段。

　　　　2：非法值校验：设置表名，条件，校验字段，合法值/非法值范围。

　　　　3：自定义校验：设置表名，校验名称，自定义sql。

　　参数解析：

　　　　使用特殊字符作为参数的前缀，后缀；便于在脚本中进行检测和替换。

　　所实现的脚本如下：

　　配置文件：

dm_monitor_list.conf 　　

 record dm_box_office_summary index_date
 record dm_channel_index index_date
 record dm_comment_emotion_summary
 record dm_comment_keyword_summary
 record dm_comment_meterial dt
 record dm_event_meterial index_date
 record dm_event_meterial_comment dt
 record dm_event_summary
 record dm_index index_date
 record dm_main_actor_index index_date
 record dm_movie_comment_summary index_date
 record dm_movie_wish_rating_summary dt
 record dm_voice_meterial dt
 record dm_index_date
 record dm_comment_keyword_base
 record dm_index_base
 record dm_event_meterial_base
 primary_check dm_box_office_summary select concat(movie_id,":",rating_type,":",count(1)) as row_val from dm_box_office_summary where index_date='_##dt##_' group by movie_id,rating_type having count(1) >1
 primary_check dm_channel_index select concat(movie_id,":",count(1)) as row_val from dm_channel_index where datediff('_##dt##_',index_date)=1 group by movie_id having count(1)>1
 primary_cyeck dm_box_office_summary select concat(movie_id,":",index_date,":",value) as row_val from dm_box_office_summary  where index_date='_##dt##_' and value<=0
 primary_check dm_channel_index select concat(movie_id,":",count(1)) as row_val from dm_channel_index where datediff('_##dt##_',index_date)=1 group by movie_id having count(1)>1
 primary_check dm_comment_emotion_summary select concat(movie_id,":",mood_type,":",platform_id,":",channel_id,":",index_date,":",count(1)) as row_val from dm_comment_emotion_summary group by movie_id,mood_type,platform_id,channel_id,index_date having count(1)>1
 primary_check dm_comment_keyword_summary select concat(movie_id,":",mood_type,":",keyword,":",platform_id,":",channel_id,":",index_date,":",count(1)) as row_val from dm_comment_keyword_summary group by movie_id,mood_type,keyword,platform_id,channel_id,index_date having count(1)>1
 primary_check dm_comment_meterial select concat(comment_id,":",count(1)) as row_val from dm_comment_meterial where dt="_##dt##_" group by comment_id having count(1)>1
 primary_check dm_event_meterial select concat(material_url,":",count(1)) as row_val from dm_event_meterial where index_date='_##dt##_' and index_type=1 group by material_url having count(1)>1
 primary_check dm_event_meterial_comment select concat(comment_id,":",count(1)) as row_val from dm_event_meterial_comment where dt='_##dt##_' group by comment_id having count(1)>1
 primary_check dm_event_summary select concat(event_id,":",platform_id,":",channel_id,":",index_date,":",count(1)) as row_val from dm_event_summary group by event_id,platform_id,channel_id,index_date having count(1)>1

　　脚本文件：monitor.sh

 #!/sh/bash
 # 分析表数据量状态
 # .数据的唯一性
 #   电影id唯一
 # .指标的正确行
 #   增量不能小于  ;全量表小
 # .基本状态
 ##  运算的条数、电影数量、空间
 # 日志格式为
 ## tablename    dt  check_type  value   insert_date
 ## check_type :
 ### record 记录值；
 ### movie_num :电影数量 ；
 ### space :所占空间
 ### diff: 昨天和今天的电影差异，使用01 代表今天有昨天没有  代表昨天有今天没有
 ### movie_rep:重复的电影数量
 ### index-* :代表某个指标增量的为负值

 basepath=$(cd `dirname $`;pwd);
 cd $basepath

 source /etc/profile
 source ../../etc/env.ini

 if [[ ! -f "$basepath/monitor_list.conf" ]]; then
     echo "check monitor list file not exists. system exit."
     exit
 fi
 #config
 #分区
 dt=$(date -d "-1 day" "+%Y-%m-%d")
 if [[ $# -eq  ]]; then
     dt=$(date -d "$1" "+%Y-%m-%d")
 fi
 insert_date=$(date "+%Y-%m-%d")
 file_path=$OPERATION_LOG_FILE_PATH
 log_name=monitor_data.log
 log=${file_path}/${log_name}

 cat $basepath/monitor_list.conf | while read line
 do
     check_type=`echo $line | cut -d " " -f `
     table_name=`echo $line |cut -d " " -f `
     profix=$table_name"\t"$insert_date"\t"

     if [[ $check_type == 'dw' ]];then
         DB=$HIVE_DB_DW
         hdfs_path=$HADOOP_DW_DATA_DESC
     elif [[ $check_type == 'ods' ]];then
         DB=$HIVE_DB_ODS_S
         hdfs_path=$HADOOP_ODS_S_DATA_DESC
     fi

     #record
     record=$(spark-sql -e "select count(1) from $DB.$table_name where dt = '$dt';")
     echo -e $profix"record\t"$record >> $log

     #movie_num
     if [[ $table_name == 'dw_weibo_materials' ]];then
         mtime_id="movie_id"
     elif [[ $table_name == 'g4_weibo_materiel_post' ]];then
         mtime_id='x_movie_id'
     else
         mtime_id="mtime_id"
     fi
     if [[ $table_name == 'dw_weibo_user' ]];then
         movie_num=$(hive -e "select count(1) from
             (select mtime_actor_id from $DB.$table_name where dt = '$dt' and source = 'govwb' group by mtime_actor_id) a")
     else
         movie_num=$(spark-sql -e "select count(1) from (select $mtime_id from $DB.$table_name where dt = '$dt' group by $mtime_id) a")
     fi
     echo -e $profix"movie_num\t"$movie_num >> $log

     #space
     if [[ $check_type == 'ods' ]];then
         space=$(hadoop fs -du $hdfs_path/$table_name/$dt)
     else
         space=$(hadoop fs -du $hdfs_path/$table_name/dt=$dt)
     fi
     echo -e $profix"space\t"$space>> $log

     #diff
     if [[ $table_name != 'dw_weibo_user' ]];then
         yesterday=$(date -d "-1 day $dt" "+%Y-%m-%d")
         diff=$(spark-sql -e "
             select concat_ws('|',collect_set(flag)) from (
                 select 'gf' as gf, concat_ws('=',flag,cast(count() as string)) as flag  from (
                     select concat(if(y.$mtime_id is null, , ),if(t.$mtime_id is null,,)) as flag
                     from (select distinct $mtime_id from $DB.$table_name where dt='$dt') t
                     full outer join (select distinct $mtime_id from $DB.$table_name where dt='$yesterday') y
                     on  t.$mtime_id = y.$mtime_id
                 ) a group by flag
             ) b group by gf;")
         echo -e $profix"diff\t"$diff>> $log
     fi
     #movie_rep
     if [[ $check_type == 'dw' ]];then
         movie_rep=$(spark-sql -e "
             select concat_ws('|',collect_set(v)) from (
                 select 'k' as k ,concat_ws('=',id,cast(count() as string)) as v
                 from $DB.$table_name where dt = '$dt'
                 group by id
                 having count()>
             )a  group by k;")
         echo -e $profix"movie_rep\t"$movie_rep>> $log
     fi
     #index-*
     if [[ $table_name == 'dw_comment_statistics' ]];then
         up_day=$(spark-sql -e "select concat('<0:',count(1)) from $DB.$table_name
             where dt = '$dt' and
                 (cast(up_day as int) <
                 or cast(down_day as int) <
                 or cast(vv_day as int ) <
                 or cast(cmts_day as int) <
                 );")
         echo -e $profix"index_day\t"$up_day >> $log
     fi
 done

 #dm
 args_prefix="_##"
 args_suffix="##_"
 cat $basepath/dm_monitor_list.conf | while read line
 do
     check_type=`echo $line | cut -d " " -f `
     table_name=`echo $line |cut -d " " -f `
     echo "表"$table_name
     if [[ $check_type == 'record' ]]; then
         dt_str=`echo $line |cut -d " " -f `
         echo "记录数校验 分区字段"$dt_str
     else
         custom_sql=`echo $line |cut -d " " -f -`
         echo "自定义校验"$check_type
     fi

     profix=$table_name"\t"$insert_date"\t"

     DB=$HIVE_DB_DW
     hdfs_path=$HADOOP_DW_DATA_DESC

     if [[ $check_type == 'record' ]]; then
         record_sql="select count(1) from $DB.$table_name"
         if [[ -n $dt_str ]]; then
             # if [[ $table_name == 'dm_channel_index' ]]; then
                 #     record_sql=$record_sql" where datediff('$dt',$dt_str)=1;"
             # else
             #     record_sql=$record_sql" where $dt_str = '$dt';"
             # fi
             record_sql=$record_sql" where $dt_str = '$dt';"
         else
             record_sql=$record_sql";"
         fi

         echo "执行的语句："$record_sql
         #record
         record=$(hive -e "set hive.mapred.mode = nonstrict;$record_sql")
         #record=$(spark-sql -e "$record_sql")
         echo -e $profix"$check_type\t"$record >> $log
     else
         #custom_sql
         custom_sql=${custom_sql//$args_prefix"dt"$args_suffix/$dt}
         echo "执行的语句："$custom_sql
         invalid_records=$(hive -e "set hive.mapred.mode = nonstrict;use $DB;select concat_ws(\" | \",collect_set(row_val)) from ( $custom_sql ) tmp;")
         echo $invalid_records
         if [[ ! -n $invalid_records || $invalid_records == ''  ]]; then
                 invalid_records=""
         fi
         echo -e $profix"$check_type\t"$invalid_records >> $log
     fi
 done
 # insert hive
 hadoop fs -rm -r $HADOOP_ODS_CONFIG_DATA_DESC/yq_monitor_data_log/dt=$dt

 if [ -f "${file_path}/$log_name" ]; then
     hive -e "
     ALTER TABLE $HIVE_DB_MONITOR.yq_monitor_data_log DROP IF EXISTS PARTITION (dt = '$dt');
     alter table $HIVE_DB_MONITOR.yq_monitor_data_log add partition (dt = '$dt');
     "
 fi
 cd $file_path
 hadoop fs -put $log_name $HADOOP_ODS_CONFIG_DATA_DESC/yq_monitor_data_log/dt=$dt
 mv -f $log_name /home/trash