hadoop-hive学习笔记
create table hive_1(id string,name string ,gender string)
row format delimited fields terminated by ','
stored as TEXTFILE;
load data local inpath '/luozt/hive_001.txt' into table hive_1 ;
create EXTERNAL table hive_2(id string,name string ,gender string)
row format delimited fields terminated by ','
stored as TEXTFILE;
load data inpath '/luo/hive_001.txt' into table hive_2 ;
//查询记录数
count 'hive_1'
//清空表
truncate table log_struct;
//删除表
drop table log_struct;
create table partition_table
(name string ,salary float,gender string,level string)
partitioned by(dt string,dept string)
row format delimited fields terminated by ','
stored as TEXTFILE;
desc partition_table
show partitions partition_table;//查看分区表的分区信息
//给分区表插入数据
load data local inpath '/luozt/par.txt' into table partition_table partition(dt='2014-04-01',dept='yonyu');
添加分区:
alter table partition_table add partition(dt='2014-04-03',dept='yonyou3') location '/user/hive/warehouse/luo.db/partition_table/dt=2014-04-03/dept=yonyou3';
删除分区:
alter table partition_table drop partition(dt='2014-04-03',dept='yonyou4')
select * from partition_table where salary>7600;
//嵌套
from (select name,salary from partition_table)e select e.name,e.salary where e.salary>7600;
//in 的用法
select * from partition_table where salary in(7000,6700);
//case的用法
select name,salary,
case
when salary<6800 then 'L1'
when salary>6800 and salary <8000 then 'L2'
when salary>8100 then 'L3'
else 'L0'
end as salary_level
from partition_table;
//having的用法
select gender,sum(salary) from partition_table group by gender;
//练习join
create table group1 (user string,score int)
row format delimited fields terminated by ','
stored as TEXTFILE;
//
create table group_join (user string,class string)
row format delimited fields terminated by ','
stored as TEXTFILE;
//普通的join
select b.class,a.score from group1 a join group_join b on (a.user=b.user);
//有个表很小时用mapjoin(b) b为小表
select /*+MAPJOIN(b)*/ b.class,a.score from group1 a join group_join b on (a.user=b.user);
//left Semi join
//分组
select user ,sum(score) from group1 group by user; ----分组时select的字段要全部作为group字段
//优化
set hive.map.aggr=true
//order by
create table orderby_test (user string,class string,math int,english int)
row format delimited fields terminated by ','
stored as TEXTFILE;
//默认升序
select * from orderby_test order by math; --desc改为降序,若将set hive.mapred.mode=strict;则要加上limit
//sort by 不受set hive.mapred.mode=strict的影响 可以指定 set mapred.reduce.tasks=<number> sort by 只会在么给reduce上进行排序,reduce输出的数据时有序的,提高全局排序的效率
//union all hive不支持顶层union ,只能将union封装在子查询中,且必须为union的查询输出定义别名
select * from (select count(*) from group1 union all select count(*) from orderby_test) temp;
//记得这种用法
select name,height,mark
from
(select name,height,'0' as mark from a
union all
select name height,'1' as mark from b) t;
//索引
create table index_test(id int,name string) partitioned by (dt string) row format delimited fields terminated by ',' stored as TEXTFILE;
//先创建一个临时表
create table temp(id int,name string ,dt string ) row format delimited fields terminated by ',' stored as TEXTFILE;
//动态分区
set hive.exec.dynamic.partition.mode=nonstrict;
set hive.exec.dynamic.partition=true;
insert overwrite table index_test partition(dt) select id,name,dt from temp;
//索引 创建索引时要有partition 否则不行
create index index1 on table index_test(id) AS 'org.apache.hadoop.hive.ql.index.compact.CompactIndexHandler' with DEFERRED REBUILD;
alter index index1 on index_test rebuild;
show index on index_test;
show partitions index_test;
//Bucket 桶 就是抽样
create table tb_tmp(id int,age int,name string,timeflag bigint) row format delimited fields terminated by ',' ;
create table tb_stu(id int,age int,name string,timeflag bigint) clustered by(id) sorted by (age) into 5 buckets row format delimited fields terminated by ','
insert into table tb_stu select id,age,name,timeflag from tb_tmp;
利用桶进行查询
select * from tb_stu tablesample(bucket 1 out of 5 on id);
//存储类型和复合数据类型
rcfile 是直接load不进去的 要用临时表insert进去
//array
create table log_array(ip string,uid array<bigint>) partitioned by (dt string) row format delimited fields terminated by ',' collection items terminated by '|' stored AS TEXTFILE;
load data local inpath '/luo/log_array.txt' into table log_array partition(dt=20150902);
//查询array中的值
select uid[1] from log_array;
select ip,size(uid) from log_array where dt=20150902;
select ip from log_array where dt=20150902 and array_contains(uid,4732974)
//map
create table log_map(ts string,ip string,type string,logtype string,request Map<string,string>,response Map<string,string>)
row format delimited fields terminated by '#' collection items terminated by '&' Map keys terminated by '=' stored as TEXTFILE;
//查询
select request['src'] from log_map;
//struct
create table log_struct(ip string,user struct<name:string,age:int>)
row format delimited fields terminated by ','
collection items terminated by '#'
stored as TEXTFILE;
数据:192.168.1.1,wow#23
192.168.1.1,wow#23
192.168.1.1,wow#23
192.168.1.1,wow#23
192.168.1.1,wow#23
192.168.1.1,wow#23
select user.name from log_struct;
hadoop-hive学习笔记的更多相关文章
- hive学习笔记之一:基本数据类型
欢迎访问我的GitHub https://github.com/zq2599/blog_demos 内容:所有原创文章分类汇总及配套源码,涉及Java.Docker.Kubernetes.DevOPS ...
- hive学习笔记之三:内部表和外部表
欢迎访问我的GitHub https://github.com/zq2599/blog_demos 内容:所有原创文章分类汇总及配套源码,涉及Java.Docker.Kubernetes.DevOPS ...
- hive学习笔记之四:分区表
欢迎访问我的GitHub https://github.com/zq2599/blog_demos 内容:所有原创文章分类汇总及配套源码,涉及Java.Docker.Kubernetes.DevOPS ...
- hive学习笔记之五:分桶
欢迎访问我的GitHub https://github.com/zq2599/blog_demos 内容:所有原创文章分类汇总及配套源码,涉及Java.Docker.Kubernetes.DevOPS ...
- hive学习笔记之六:HiveQL基础
欢迎访问我的GitHub https://github.com/zq2599/blog_demos 内容:所有原创文章分类汇总及配套源码,涉及Java.Docker.Kubernetes.DevOPS ...
- hive学习笔记之七:内置函数
欢迎访问我的GitHub https://github.com/zq2599/blog_demos 内容:所有原创文章分类汇总及配套源码,涉及Java.Docker.Kubernetes.DevOPS ...
- hive学习笔记之九:基础UDF
欢迎访问我的GitHub https://github.com/zq2599/blog_demos 内容:所有原创文章分类汇总及配套源码,涉及Java.Docker.Kubernetes.DevOPS ...
- hive学习笔记之十:用户自定义聚合函数(UDAF)
欢迎访问我的GitHub 这里分类和汇总了欣宸的全部原创(含配套源码):https://github.com/zq2599/blog_demos 本篇概览 本文是<hive学习笔记>的第十 ...
- hive学习笔记之十一:UDTF
欢迎访问我的GitHub https://github.com/zq2599/blog_demos 内容:所有原创文章分类汇总及配套源码,涉及Java.Docker.Kubernetes.DevOPS ...
- Hadoop入门学习笔记---part4
紧接着<Hadoop入门学习笔记---part3>中的继续了解如何用java在程序中操作HDFS. 众所周知,对文件的操作无非是创建,查看,下载,删除.下面我们就开始应用java程序进行操 ...
随机推荐
- PAT 天梯赛 L2-015. 互评成绩 【排序】
题目链接 https://www.patest.cn/contests/gplt/L2-015 思路 在求和的过程中 标记一下 最大值和最小值,在最后求平均的时候 用总和减去最大值和最小值 去除 (总 ...
- 每天一个Linux命令(54)chkconfig命令
chkconfig命令检查.设置系统的各种服务. (1)用法: 用法: chkconfig [必要参数] [服务] (2)功能: 功能: chkconf ...
- third application :Directions widget
<!DOCTYPE html> <html> <head> <meta http-equiv="Content-Type" content ...
- Redis集群环境搭建
Redis集群cluster环境搭建 描述:本章节主要单服务器搭建集群,在一个服务器上启动多个不同端口的redis服务,非真实环境. 真实环境下redis集群会搭建在多个物理服务器上,并非单一的服务器 ...
- MyBatis SQL 生成方法 增删改查
此类根据JAVA实体BEAN生成MYBATIS的接口SQL(mapper) package com.sicdt.sicsign.bill.service.hessian; import java.la ...
- 嵌入式Qt程序启动参数-qws 不需要X11桌面系统
1 背景 通过串口终端启动arm开发板(linux系统)的Qt应用程序,提示: [root@FORLINX6410]# /opt/qt-4.7.1/demos/textedit/textedit s3 ...
- 爬虫实例之使用requests和Beautifusoup爬取糗百热门用户信息
这次主要用requests库和Beautifusoup库来实现对糗百的热门帖子的用户信息的收集,由于糗百的反爬虫不是很严格,也不需要先登录才能获取数据,所以较简单. 思路,先请求首页的热门帖子获得用户 ...
- iOS_多线程(一)
在学习多线程之前首先搞清楚以下几个问题. 并发:在同一时刻,只有一条指令被执行,多条指令进行快速切换执行. 并行:在同一时刻,多个处理器可以处理多条指令 1.什么是进程? 一个运行的程序就是 ...
- Android编译系统简要介绍【转】
本文转载自:http://blog.csdn.net/luoshengyang/article/details/18466779 在Android源码环境中,我们开发好一个模块后,再写一个Androi ...
- HDFS存档
Hadoop存档 每个文件均按块方式存储,每个块的元数据存储在namenode的内存中,因此Hadoop存储小文件会非常低效.因为大量的小文件会耗尽namenode中的大部分内存.存储小文件所需的磁盘 ...