hive 存储格式及压缩

 -- 设置参数
 set hivevar:target_db_name=db_dw;
 use ${hivevar:target_db_name};

 -- 创建textfile表
 create table file_format_textfile
 row format delimited fields terminated by '\001'
 stored as textfile
 as
 select * from smple_table;

 -- 测试各种压缩的orc表
 create table file_format_orc_zlib
 row format delimited fields terminated by '\001'
 stored as orc tblproperties ("orc.compress"="ZLIB")
 as
 select * from file_format_textfile
 ;

 create table file_format_orc_snappy
 row format delimited fields terminated by '\001'
 stored as orc tblproperties ("orc.compress"="SNAPPY")
 as
 select * from file_format_orc_zlib
 ;

 create table file_format_orc_none
 row format delimited fields terminated by '\001'
 stored as orc tblproperties ("orc.compress"="NONE")
 as
 select * from file_format_orc_zlib
 ;

 create table file_format_orc_default
 row format delimited fields terminated by '\001'
 stored as orc
 as
 select * from file_format_orc_zlib
 ;

 -- 测试各种压缩的parquet表
 create table file_format_parquet_zlib
 row format delimited fields terminated by '\001'
 stored as parquet tblproperties ("parquet.compress"="ZLIB")
 as
 select * from file_format_orc_zlib
 ;

 create table file_format_parquet_snappy
 row format delimited fields terminated by '\001'
 stored as parquet tblproperties ("parquet.compress"="SNAPPY")
 as
 select * from file_format_orc_zlib
 ;

 create table file_format_parquet_none
 row format delimited fields terminated by '\001'
 stored as parquet tblproperties ("parquet.compress"="NONE")
 as
 select * from file_format_orc_zlib
 ;

 create table file_format_parquet_default
 row format delimited fields terminated by '\001'
 stored as parquet
 as
 select * from file_format_orc_zlib
 ;

 -- 测试各种压缩的rcfile表(可能参数没生效，各种压缩后大小一致)
 create table file_format_rcfile_zlib
 row format delimited fields terminated by '\001'
 stored as rcfile tblproperties ("rcfile.compress"="ZLIB")
 as
 select * from file_format_orc_zlib
 ;

 create table file_format_rcfile_snappy
 row format delimited fields terminated by '\001'
 stored as rcfile tblproperties ("rcfile.compress"="SNAPPY")
 as
 select * from file_format_orc_zlib
 ;

 create table file_format_rcfile_none
 row format delimited fields terminated by '\001'
 stored as rcfile tblproperties ("rcfile.compress"="NONE")
 as
 select * from file_format_orc_zlib
 ;

 create table file_format_rcfile_default
 row format delimited fields terminated by '\001'
 stored as rcfile
 as
 select * from file_format_orc_zlib
 ;

-- 查看各种压缩下的格式大小
dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_textfile;

dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_orc_zlib;
dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_orc_snappy;
dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_orc_none;
dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_orc_default;

dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_parquet_zlib;
dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_parquet_snappy;
dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_parquet_none;
dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_parquet_default;

dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_rcfile_zlib;
dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_rcfile_snappy;
dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_rcfile_none;
dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_rcfile_default;

-- 统计数据，原文件见文件中的附件