create table hive_1(id string,name string ,gender string)
row format delimited fields terminated by ','
stored as TEXTFILE;

load data local inpath '/luozt/hive_001.txt' into table hive_1 ;

create EXTERNAL table hive_2(id string,name string ,gender string)
row format delimited fields terminated by ','
stored as TEXTFILE;

load data inpath '/luo/hive_001.txt' into table hive_2 ;
//查询记录数
count 'hive_1'
//清空表
truncate table log_struct;

//删除表
drop table log_struct;

create table partition_table
(name string ,salary float,gender string,level string)
partitioned by(dt string,dept string)
row format delimited fields terminated by ','
stored as TEXTFILE;

desc partition_table

show partitions partition_table;//查看分区表的分区信息
//给分区表插入数据
load data local inpath '/luozt/par.txt' into table partition_table partition(dt='2014-04-01',dept='yonyu');

添加分区:
alter table partition_table add partition(dt='2014-04-03',dept='yonyou3') location '/user/hive/warehouse/luo.db/partition_table/dt=2014-04-03/dept=yonyou3';

删除分区:
alter table partition_table drop partition(dt='2014-04-03',dept='yonyou4')

select * from partition_table where salary>7600;
//嵌套
from (select name,salary from partition_table)e select e.name,e.salary where e.salary>7600;

//in 的用法

select * from partition_table where salary in(7000,6700);

//case的用法
select name,salary,
case
when salary<6800 then 'L1'
when salary>6800 and salary <8000 then 'L2'
when salary>8100 then 'L3'
else 'L0'
end as salary_level
from partition_table;

//having的用法

select gender,sum(salary) from partition_table group by gender;

//练习join
create table group1 (user string,score int)
row format delimited fields terminated by ','
stored as TEXTFILE;

//
create table group_join (user string,class string)
row format delimited fields terminated by ','
stored as TEXTFILE;

//普通的join
select b.class,a.score from group1 a join group_join b on (a.user=b.user);
//有个表很小时用mapjoin(b) b为小表
select /*+MAPJOIN(b)*/ b.class,a.score from group1 a join group_join b on (a.user=b.user);
//left Semi join

//分组
select user ,sum(score) from group1 group by user; ----分组时select的字段要全部作为group字段
//优化
set hive.map.aggr=true

//order by
create table orderby_test (user string,class string,math int,english int)
row format delimited fields terminated by ','
stored as TEXTFILE;

//默认升序
select * from orderby_test order by math; --desc改为降序,若将set hive.mapred.mode=strict;则要加上limit

//sort by 不受set hive.mapred.mode=strict的影响 可以指定 set mapred.reduce.tasks=<number> sort by 只会在么给reduce上进行排序,reduce输出的数据时有序的,提高全局排序的效率

//union all hive不支持顶层union ,只能将union封装在子查询中,且必须为union的查询输出定义别名
select * from (select count(*) from group1 union all select count(*) from orderby_test) temp;

//记得这种用法
select name,height,mark
from
(select name,height,'0' as mark from a
union all
select name height,'1' as mark from b) t;

//索引
create table index_test(id int,name string) partitioned by (dt string) row format delimited fields terminated by ',' stored as TEXTFILE;
//先创建一个临时表
create table temp(id int,name string ,dt string ) row format delimited fields terminated by ',' stored as TEXTFILE;
//动态分区
set hive.exec.dynamic.partition.mode=nonstrict;
set hive.exec.dynamic.partition=true;

insert overwrite table index_test partition(dt) select id,name,dt from temp;

//索引 创建索引时要有partition 否则不行
create index index1 on table index_test(id) AS 'org.apache.hadoop.hive.ql.index.compact.CompactIndexHandler' with DEFERRED REBUILD;

alter index index1 on index_test rebuild;

show index on index_test;
show partitions index_test;

//Bucket 桶 就是抽样

create table tb_tmp(id int,age int,name string,timeflag bigint) row format delimited fields terminated by ',' ;
create table tb_stu(id int,age int,name string,timeflag bigint) clustered by(id) sorted by (age) into 5 buckets row format delimited fields terminated by ','

insert into table tb_stu select id,age,name,timeflag from tb_tmp;

利用桶进行查询
select * from tb_stu tablesample(bucket 1 out of 5 on id);

//存储类型和复合数据类型
rcfile 是直接load不进去的 要用临时表insert进去

//array
create table log_array(ip string,uid array<bigint>) partitioned by (dt string) row format delimited fields terminated by ',' collection items terminated by '|' stored AS TEXTFILE;

load data local inpath '/luo/log_array.txt' into table log_array partition(dt=20150902);
//查询array中的值
select uid[1] from log_array;
select ip,size(uid) from log_array where dt=20150902;
select ip from log_array where dt=20150902 and array_contains(uid,4732974)

//map
create table log_map(ts string,ip string,type string,logtype string,request Map<string,string>,response Map<string,string>)
row format delimited fields terminated by '#' collection items terminated by '&' Map keys terminated by '=' stored as TEXTFILE;

//查询
select request['src'] from log_map;

//struct
create table log_struct(ip string,user struct<name:string,age:int>)
row format delimited fields terminated by ','
collection items terminated by '#'
stored as TEXTFILE;

数据:192.168.1.1,wow#23
192.168.1.1,wow#23
192.168.1.1,wow#23
192.168.1.1,wow#23
192.168.1.1,wow#23
192.168.1.1,wow#23

select user.name from log_struct;

hadoop-hive学习笔记的更多相关文章

  1. hive学习笔记之一:基本数据类型

    欢迎访问我的GitHub https://github.com/zq2599/blog_demos 内容:所有原创文章分类汇总及配套源码,涉及Java.Docker.Kubernetes.DevOPS ...

  2. hive学习笔记之三:内部表和外部表

    欢迎访问我的GitHub https://github.com/zq2599/blog_demos 内容:所有原创文章分类汇总及配套源码,涉及Java.Docker.Kubernetes.DevOPS ...

  3. hive学习笔记之四:分区表

    欢迎访问我的GitHub https://github.com/zq2599/blog_demos 内容:所有原创文章分类汇总及配套源码,涉及Java.Docker.Kubernetes.DevOPS ...

  4. hive学习笔记之五:分桶

    欢迎访问我的GitHub https://github.com/zq2599/blog_demos 内容:所有原创文章分类汇总及配套源码,涉及Java.Docker.Kubernetes.DevOPS ...

  5. hive学习笔记之六:HiveQL基础

    欢迎访问我的GitHub https://github.com/zq2599/blog_demos 内容:所有原创文章分类汇总及配套源码,涉及Java.Docker.Kubernetes.DevOPS ...

  6. hive学习笔记之七:内置函数

    欢迎访问我的GitHub https://github.com/zq2599/blog_demos 内容:所有原创文章分类汇总及配套源码,涉及Java.Docker.Kubernetes.DevOPS ...

  7. hive学习笔记之九:基础UDF

    欢迎访问我的GitHub https://github.com/zq2599/blog_demos 内容:所有原创文章分类汇总及配套源码,涉及Java.Docker.Kubernetes.DevOPS ...

  8. hive学习笔记之十:用户自定义聚合函数(UDAF)

    欢迎访问我的GitHub 这里分类和汇总了欣宸的全部原创(含配套源码):https://github.com/zq2599/blog_demos 本篇概览 本文是<hive学习笔记>的第十 ...

  9. hive学习笔记之十一:UDTF

    欢迎访问我的GitHub https://github.com/zq2599/blog_demos 内容:所有原创文章分类汇总及配套源码,涉及Java.Docker.Kubernetes.DevOPS ...

  10. Hadoop入门学习笔记---part4

    紧接着<Hadoop入门学习笔记---part3>中的继续了解如何用java在程序中操作HDFS. 众所周知,对文件的操作无非是创建,查看,下载,删除.下面我们就开始应用java程序进行操 ...

随机推荐

  1. PAT 天梯赛 L2-015. 互评成绩 【排序】

    题目链接 https://www.patest.cn/contests/gplt/L2-015 思路 在求和的过程中 标记一下 最大值和最小值,在最后求平均的时候 用总和减去最大值和最小值 去除 (总 ...

  2. 每天一个Linux命令(54)chkconfig命令

        chkconfig命令检查.设置系统的各种服务.     (1)用法:     用法:  chkconfig  [必要参数]  [服务]     (2)功能:     功能:  chkconf ...

  3. third application :Directions widget

    <!DOCTYPE html> <html> <head> <meta http-equiv="Content-Type" content ...

  4. Redis集群环境搭建

    Redis集群cluster环境搭建 描述:本章节主要单服务器搭建集群,在一个服务器上启动多个不同端口的redis服务,非真实环境. 真实环境下redis集群会搭建在多个物理服务器上,并非单一的服务器 ...

  5. MyBatis SQL 生成方法 增删改查

    此类根据JAVA实体BEAN生成MYBATIS的接口SQL(mapper) package com.sicdt.sicsign.bill.service.hessian; import java.la ...

  6. 嵌入式Qt程序启动参数-qws 不需要X11桌面系统

    1 背景 通过串口终端启动arm开发板(linux系统)的Qt应用程序,提示: [root@FORLINX6410]# /opt/qt-4.7.1/demos/textedit/textedit s3 ...

  7. 爬虫实例之使用requests和Beautifusoup爬取糗百热门用户信息

    这次主要用requests库和Beautifusoup库来实现对糗百的热门帖子的用户信息的收集,由于糗百的反爬虫不是很严格,也不需要先登录才能获取数据,所以较简单. 思路,先请求首页的热门帖子获得用户 ...

  8. iOS_多线程(一)

    在学习多线程之前首先搞清楚以下几个问题. 并发:在同一时刻,只有一条指令被执行,多条指令进行快速切换执行. 并行:在同一时刻,多个处理器可以处理多条指令 1.什么是进程?      一个运行的程序就是 ...

  9. Android编译系统简要介绍【转】

    本文转载自:http://blog.csdn.net/luoshengyang/article/details/18466779 在Android源码环境中,我们开发好一个模块后,再写一个Androi ...

  10. HDFS存档

    Hadoop存档 每个文件均按块方式存储,每个块的元数据存储在namenode的内存中,因此Hadoop存储小文件会非常低效.因为大量的小文件会耗尽namenode中的大部分内存.存储小文件所需的磁盘 ...