shell脚本中向hive动态分区插入数据

在hive上建表与普通分区表创建方法一样；

 CREATE  TABLE `dwa_m_user_association_circle`(
   `device_number` string,
   `oppo_number` string,
   `prov_id_oppo` string,
   `area_id_oppo` string,
   `dealer_oppo` string,
   `short_call_nums` bigint,
   `long3_call_nums` bigint,
   `long5_call_nums` bigint,
   `long10_call_nums` bigint,
   `short_total_nums` bigint,
   `long3_total_nums` bigint,
   `long5_total_nums` bigint,
   `long10_total_nums` bigint,
   `area_id` string)
 PARTITIONED BY (
   `month_id` string,
   `prov_id` string,
   `dealer` string)
 ROW FORMAT DELIMITED
   FIELDS TERMINATED BY '|'
   NULL DEFINED AS ''
 STORED AS INPUTFORMAT
   'org.apache.hadoop.mapred.TextInputFormat'
 OUTPUTFORMAT
   'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
 LOCATION
   'hdfs://beh/user/hive/warehouse/all_ana_pro.db/dwa_m_user_association_circle'
 TBLPROPERTIES (
   'numPartitions'='',
   'numFiles'='',
   'transient_lastDdlTime'='',
   'totalSize'='',
   'numRows'='',
   'rawDataSize'='')

在shell脚本中，需设置的参数：

set hive.exec.dynamic.partition=true;                     #开启动态分区，默认是false
set hive.exec.dynamic.partition.mode=nostrict;      #开启允许所有分区都是动态的，否则必须要有静态分区才能使用。

set hive.exec.max.created.files=1000000;             #允许创建的最大文件数，当分区是2个或三个分区时，文件会被分成很多小文件，该设置就是将文件的最大数目设成100w;

 insert overwrite table ALL_ANA_PRO.dwa_m_user_association_circle  PARTITION (month_id=${v_month},prov_id,dealer)
 select device_number,
         oppo_number  ,
         prov_id_oppo ,
         area_id_oppo ,
         dealer_oppo ,
         short_call_nums,
         long3_call_nums,
         long5_call_nums,
         long10_call_nums,
         sum(short_call_nums) over(distribute by device_number)  short_total_nums,
         sum(long3_call_nums ) over(distribute by device_number) long3_total_nums,
         sum(long5_call_nums) over(distribute by  device_number) long5_total_nums,
         sum(long10_call_nums) over(distribute by device_number) long10_total_nums,
         area_id      ,
         prov_id      ,
         dealer
 from
         (SELECT device_number,
                 prov_id      ,
                 area_id      ,
                 dealer       ,
                 oppo_number  ,
                 prov_id_oppo ,
                 area_id_oppo ,
                 dealer_oppo  ,
                 sum(case when t.bill_times < 60 then 1 else 0 end ) short_call_nums,
                 sum(case when t.bill_times >180 then 1 else 0 end ) long3_call_nums,
                 sum(case when t.bill_times >300 then 1 else 0 end ) long5_call_nums,
                 sum(case when t.bill_times >600 then 1 else 0 end ) long10_call_nums
         FROM ( SELECT prov_id      ,
                       area_id      ,
                       device_number,
                       0 dealer     ,
                       prov_id_oppo ,
                       area_id_oppo ,
                       oppo_number  ,
                       dealer_oppo  ,
                       bill_times
         FROM   ALL_ANA_PRO.DWA_M_CALL_RING_BASE t
         WHERE  month_id = '${v_month}'
         AND    ticket_type = 1
         AND    dealer_oppo > -1
         UNION ALL
         SELECT prov_id_oppo prov_id      ,
               area_id_oppo area_id      ,
               oppo_number device_number ,
               dealer_oppo dealer        ,
               prov_id prov_id_oppo      ,
               area_id area_id_oppo      ,
               device_number oppo_number ,
               0 dealer_oppo             ,
               bill_times
         FROM   ALL_ANA_PRO.DWA_M_CALL_RING_BASE t
         WHERE  month_id = '${v_month}'
         AND    ticket_type = 1
         AND    dealer_oppo > -1
            ) t
         GROUP BY
               device_number,
               prov_id      ,
               area_id      ,
               dealer       ,
               oppo_number  ,
               prov_id_oppo ,
               area_id_oppo ,
               dealer_oppo
         ) t;

这里，需要说明的是，向目标表插数：

insert overwrite table ALL_ANA_PRO.dwa_m_user_association_circle  PARTITION (month_id=${v_month},prov_id,dealer)
其中select语句中动态分区prov_id,dealer两个字段一般在最后面；