GreenPlum 大数据平台--外部表(三)

一,外部表介绍

　　Greenplum 在数据加载上有一个明显的优势,就是支持数据的并发加载,gpfdisk是并发加载的工具,数据库中对应的就是外部表

　　所谓外部表，就是在数据库中只有表定义、没有数据，数据都存放在数据库之外的数据文件。greenplum可以对一个外部表执行正常的DML操作，当读取数据的时候，数据库从数据文件中加载数据。外部表支持在segment上并发地告诉从gpfdist导入数据，由于是从segment上导入数据，所以效率很高。

　　结构图:

　　外部表需要指定gpfdist的IP和端口，还要有详细的目录地址，文件名支持通配符匹配。可以编写多个gpfdist地址，但是总数不能超过总的segment数量，否则会报错。

　　GPDB提供两种外部表：可读外部表用于数据装载、可写外部表用于数据卸载。外部表可基于文件、亦可基于WEB，这两种都能实现可读、可写。

当一个查询使用一个常规的外部表，该外部表被认为是可重读的，因为在该查询期间数据是静态的。而对于WEB外部表，数据是不可重读的，因为在该查询的执行期间数据可能会发生变化。

　　　　可写外部表用以从数据库表中选择记录并输出到文件、命名管道或其他可执行程序。比如，可以从GPDB中卸载数据并发送到一个可执行程序，该程序连接到其他数据库或者ETL工具并装载数据到其他地方。可写外部表还可以用于输出到GPDB的并行MapReduce计算。

　　　　可写外部表被定义后，数据即可从数据库表中被选择并插入到该可写外部表。可写外部表只允许INSERT操作 – SELECT、 UPDATE、 DELETE或TRUNCATE是不允许的。可写外部表输出数据到一个可执行程序，该程序要能够接受流输入数据。

　　　　在创建外部表的时候，可以指定分隔符、err表、指定允许出错的数据条数，以及源文件的编码等信息。

二,外部表语法

CREATE [READABLE] EXTERNAL TABLE table_name

    ( column_name data_type [, ...] | LIKE other_table )

      LOCATION ('file://seghost[:port]/path/file' [, ...])

        | ('gpfdist://filehost[:port]/file_pattern[#transform]'

        | ('gpfdists://filehost[:port]/file_pattern[#transform]'

            [, ...])

        | ('gphdfs://hdfs_host[:port]/path/file')

      FORMAT 'TEXT'

            [( [HEADER]

               [DELIMITER [AS] 'delimiter' | 'OFF']

               [NULL [AS] 'null string']

               [ESCAPE [AS] 'escape' | 'OFF']

               [NEWLINE [ AS ] 'LF' | 'CR' | 'CRLF']

               [FILL MISSING FIELDS] )]

           | 'CSV'

            [( [HEADER]

               [QUOTE [AS] 'quote']

               [DELIMITER [AS] 'delimiter']

               [NULL [AS] 'null string']

               [FORCE NOT NULL column [, ...]]

               [ESCAPE [AS] 'escape']

               [NEWLINE [ AS ] 'LF' | 'CR' | 'CRLF']

               [FILL MISSING FIELDS] )]

           | 'AVRO'

           | 'PARQUET'

           | 'CUSTOM' (Formatter=<formatter specifications>)

     [ ENCODING 'encoding' ]

     [ [LOG ERRORS [INTO error_table]] SEGMENT REJECT LIMIT count

       [ROWS | PERCENT] ]

CREATE [READABLE] EXTERNAL WEB TABLE table_name

   ( column_name data_type [, ...] | LIKE other_table )

      LOCATION ('http://webhost[:port]/path/file' [, ...])

    | EXECUTE 'command' [ON ALL

                          | MASTER

                          | number_of_segments

                          | HOST ['segment_hostname']

                          | SEGMENT segment_id ]

      FORMAT 'TEXT'

            [( [HEADER]

               [DELIMITER [AS] 'delimiter' | 'OFF']

               [NULL [AS] 'null string']

               [ESCAPE [AS] 'escape' | 'OFF']

               [NEWLINE [ AS ] 'LF' | 'CR' | 'CRLF']

               [FILL MISSING FIELDS] )]

           | 'CSV'

            [( [HEADER]

               [QUOTE [AS] 'quote']

               [DELIMITER [AS] 'delimiter']

               [NULL [AS] 'null string']

               [FORCE NOT NULL column [, ...]]

               [ESCAPE [AS] 'escape']

               [NEWLINE [ AS ] 'LF' | 'CR' | 'CRLF']

               [FILL MISSING FIELDS] )]

           | 'CUSTOM' (Formatter=<formatter specifications>)

     [ ENCODING 'encoding' ]

     [ [LOG ERRORS [INTO error_table]] SEGMENT REJECT LIMIT count

       [ROWS | PERCENT] ]

CREATE WRITABLE EXTERNAL TABLE table_name

    ( column_name data_type [, ...] | LIKE other_table )

     LOCATION('gpfdist://outputhost[:port]/filename[#transform]'

      | ('gpfdists://outputhost[:port]/file_pattern[#transform]'

          [, ...])

      | ('gphdfs://hdfs_host[:port]/path')

      FORMAT 'TEXT'

               [( [DELIMITER [AS] 'delimiter']

               [NULL [AS] 'null string']

               [ESCAPE [AS] 'escape' | 'OFF'] )]

          | 'CSV'

               [([QUOTE [AS] 'quote']

               [DELIMITER [AS] 'delimiter']

               [NULL [AS] 'null string']

               [FORCE QUOTE column [, ...]] ]

               [ESCAPE [AS] 'escape'] )]

           | 'AVRO'

           | 'PARQUET'

           | 'CUSTOM' (Formatter=<formatter specifications>)

    [ ENCODING 'write_encoding' ]

    [ DISTRIBUTED BY (column, [ ... ] ) | DISTRIBUTED RANDOMLY ]

CREATE WRITABLE EXTERNAL WEB TABLE table_name

    ( column_name data_type [, ...] | LIKE other_table )

    EXECUTE 'command' [ON ALL]

    FORMAT 'TEXT'

               [( [DELIMITER [AS] 'delimiter']

               [NULL [AS] 'null string']

               [ESCAPE [AS] 'escape' | 'OFF'] )]

          | 'CSV'

               [([QUOTE [AS] 'quote']

               [DELIMITER [AS] 'delimiter']

               [NULL [AS] 'null string']

               [FORCE QUOTE column [, ...]] ]

               [ESCAPE [AS] 'escape'] )]

           | 'CUSTOM' (Formatter=<formatter specifications>)

    [ ENCODING 'write_encoding' ]

    [ DISTRIBUTED BY (column, [ ... ] ) | DISTRIBUTED RANDOMLY ]

三,创建外部表

　　01,语法

gpfdist [-d directory] [-p http_port] [-l log_file] [-t timeout]

[-S] [-w time] [-v | -V] [-s] [-m max_length] [--ssl certificate_path]

gpfdist -? | --help

gpfdist --version

　　02,启动进程

--创建gpdist进程

[gpadmin@greenplum02 ~]$ mkdir script

[gpadmin@greenplum02 ~]$ nohup gpfdist -d /home/gpadmin/script/ -p 8081 -l /home/gpadmin/script/gpfdist.log &

[] 6904

[gpadmin@greenplum02 ~]$ nohup: ignoring input and appending output to ‘nohup.out’

[gpadmin@greenplum02 ~]$ ss -lntup|grep 8081

tcp    LISTEN     0      128      :::8081                 :::*                   users:(("gpfdist",pid=6904,fd=6))

---配置读取文件

[gpadmin@greenplum02 script]$ cat test.txt

Prague,Jan,101,4875.33

Rome,Mar,87,1557.39

Bangalore,May,317,8936.99

Beijing,Jul,411,11600.67

San Francisco,Sept,156,6846.34

Paris,Nov,159,7134.56

San Francisco,Jan,113,5397.89

Prague,Dec,333,9894.77

Bangalore,Jul,271,8320.55

Beijing,Dec,100,4248.41

q

[gpadmin@greenplum02 script]$ pwd

/home/gpadmin/script

--后面的是错误信息

　　03,创建外部表

create external table public.test

(

country varchar(128),

name varchar(128),

id int,

sale varchar(128)

)

location ('gpfdist://192.168.0.222:8081/test.txt')

format 'text'

(delimiter ',' null as '' escape 'off')

encoding 'utf8'

log error segment reject limit 3 rows;

--- location   文件所在位置，可以直接是本地路径、gpfdist地址、gpfdists地址、gphdfs地址。

--- format     文本类型

--- delimiter  分隔符

--- encoding    编码

--- log error into  错误数据表，记录错误数据，会自动创建。一般都是tablename_err格式，例如t1_err。

--- segment reject limit  错误数据的条数/百分比(rows/percent)，超过设置值会报错。最小值是2。用来确保数据的完整性。

结果:
postgres=# create external table public.test99(country varchar(128),name varchar(128),id int,sale varchar(128))location ('gpfdist://192.168.0.222:8081/test.txt')format 'text'(delimiter ',' null as '' escape 'off')encoding 'utf8'log errors segment reject limit 3 rows;

CREATE EXTERNAL TABLE

postgres=# SELECT * from public.test99

postgres-# ;

NOTICE:  Found 2 data formatting errors (2 or more input rows). Rejected related input data.

    country    | name | id  |   sale

---------------+------+-----+----------

 Prague        | Jan  | 101 | 4875.33

 Rome          | Mar  |  87 | 1557.39

 Bangalore     | May  | 317 | 8936.99

 Beijing       | Jul  | 411 | 11600.67

 San Francisco | Sept | 156 | 6846.34

 Paris         | Nov  | 159 | 7134.56

 San Francisco | Jan  | 113 | 5397.89

 Prague        | Dec  | 333 | 9894.77

 Bangalore     | Jul  | 271 | 8320.55

 Beijing       | Dec  | 100 | 4248.41

(10 rows)

postgres=# SELECT * from test99;

NOTICE:  Found 2 data formatting errors (2 or more input rows). Rejected related input data.

    country    | name | id  |   sale

---------------+------+-----+----------

 Prague        | Jan  | 101 | 4875.33

 Rome          | Mar  |  87 | 1557.39

 Bangalore     | May  | 317 | 8936.99

 Beijing       | Jul  | 411 | 11600.67

 San Francisco | Sept | 156 | 6846.34

 Paris         | Nov  | 159 | 7134.56

 San Francisco | Jan  | 113 | 5397.89

 Prague        | Dec  | 333 | 9894.77

 Bangalore     | Jul  | 271 | 8320.55

 Beijing       | Dec  | 100 | 4248.41

(10 rows)

　　04,数据装载

insert into table select * from table_ext;

内部表<----外部表