plpython 中文分词Windows 版

windows 下安装
版本匹配
python-3.4.3.amd64.msi
postgresql-10.1-2-windows-x64.exe

create language plpython3u;
select * from pg_language ;

psql -h 127.0.0.1 -p 5432 postgres postgres

python setup.py install

jieba.cut 方法接受三个输入参数: 需要分词的字符串；cut_all 参数用来控制是否采用全模式；HMM 参数用来控制是否使用 HMM 模型
jieba.cut_for_search 方法接受两个参数：需要分词的字符串；是否使用 HMM 模型。该方法适合用于搜索引擎构建倒排索引的分词，粒度比较细
待分词的字符串可以是 unicode 或 UTF-8 字符串、GBK 字符串。注意：不建议直接输入 GBK 字符串，可能无法预料地错误解码成 UTF-8
jieba.cut 以及 jieba.cut_for_search 返回的结构都是一个可迭代的 generator，可以使用 for 循环来获得分词后得到的每一个词语(unicode)，或者用
jieba.lcut 以及 jieba.lcut_for_search 直接返回 list
jieba.Tokenizer(dictionary=DEFAULT_DICT) 新建自定义分词器，可用于同时使用不同词典。jieba.dt 为默认分词器，所有全局分词相关函数都是该分词器的映射。

# encoding=utf-8
import jieba

seg_list = jieba.cut("我来到北京清华大学", cut_all=True)
print("Full Mode: " + "/ ".join(seg_list)) # 全模式

seg_list = jieba.cut("我来到北京清华大学", cut_all=False)
print("Default Mode: " + "/ ".join(seg_list)) # 精确模式

seg_list = jieba.cut("他来到了网易杭研大厦") # 默认是精确模式
print(", ".join(seg_list))

seg_list = jieba.cut_for_search("小明硕士毕业于中国科学院计算所，后在日本京都大学深造") # 搜索引擎模式
print(", ".join(seg_list))

--方法体内 需要写python语法
create or replace function to_tsv(i_state int,i_text text) returns tsvector as $$
import jieba
if i_state == 1:
seg_list = jieba.cut(i_text, cut_all=True)
elif i_state == 2:
seg_list = jieba.cut(i_text, cut_all=False)
elif i_state == 3:
seg_list = jieba.cut_for_search(i_text)
return(" ".join(seg_list))
$$ language plpython3u;

cd E:\PostgreSQL_common\pgsql\jieba-master

python setup.py install

psql -h 127.0.0.1 -p 5432 postgres postgres

drop language plpython3u;

create language plpython3u;

--python2使用
create language plpythonu;

create or replace function to_tsv(i_text text) returns tsvector as $$
import jieba
seg_list = jieba.cut(i_text, cut_all=True);
return("".join(seg_list))
$$ language plpython3u;

-- python 2.7.5使用
create or replace function to_tsv(i_text text) returns tsvector as $$
import jieba
seg_list = jieba.cut(i_text, cut_all=True);
return("".join(seg_list))
$$ language plpythonu;

select to_tsv('我爱北京天安门');

create or replace function to_tsv(i_text text) returns tsvector as $$
import jieba
seg_list = jieba.cut(i_text, cut_all=True);
return("".join(seg_list))
$$ language plpython3u;

create or replace function to_tsv(i_text text) returns tsvector as $$
$$ language plpython3u;

from /opt/python343/lib/python3.4/site-packages/jieba import cut

把“import *****”改为“from 文件路径 import *****”；

create or replace function to_tsv(i_text text) returns tsvector as $$
$$ language plpython3u;

create table ICC_T_ADDRESS
(
GUID VARCHAR(64),
ADDRESS VARCHAR(128),
NAME VARCHAR(128),
SHORTNAME VARCHAR(64),
HISNAME VARCHAR(128),
SZM VARCHAR(64),
PINYIN VARCHAR(64),
X numeric(10,6),
Y numeric(10,6),
TYPE VARCHAR(64),
LINKPHONE VARCHAR(64),
tsvnameaddress tsvector
)

create index idx_tsv_address on ICC_T_ADDRESS using gin(tsvnameaddress);//创建gin索引

create trigger tsvupdate after insert
on ICC_T_ADDRESS for each row execute procedure
tsv_update_trigger();

create or replace function tsv_update_trigger() returns trigger as $$
declare
begin
update ICC_T_ADDRESS set tsvnameaddress = to_tsv(name || address) where guid=NEW.guid;
return null;
end;
$$ language plpgsql;

create or replace function tsv_update_trigger() returns trigger as $$
declare
begin
update ICC_T_ADDRESS set tsvnameaddress = to_tsv(COALESCE(name,'') || COALESCE(address,'')) where guid=NEW.guid;
return null;
end;
$$ language plpgsql;

insert into ICC_T_ADDRESS(guid,NAME,ADDRESS) values(1,'西北风面馆','黑龙江省');
insert into ICC_T_ADDRESS(guid,NAME,ADDRESS) values(2,'和兴小学','黑龙江省');
insert into ICC_T_ADDRESS(guid,NAME,ADDRESS) values(3,'师范附小','黑龙江省');

select guid,name,address,tsvnameaddress from ICC_T_ADDRESS where tsvnameaddress @@ to_tsquery('振兴|街');
select guid,name,address,tsvnameaddress from ICC_T_ADDRESS where tsvnameaddress @@ '振兴|街';
select guid,name,address,tsvnameaddress,ts_rank(tsvnameaddress, to_tsquery('振兴')) from ICC_T_ADDRESS;

select guid,name,address,tsvnameaddress,ts_rank(tsvnameaddress, to_tsquery('三')) from ICC_T_ADDRESS order by ts_rank(tsvnameaddress, to_tsquery('三')) desc;

SELECT t.rank_cd as score,t.address,t.name,t.shortname,t.hisname,t.szm,t.pinyin,t.x,t.y,t.type,t.linkphone,t.guid
from(select t.* ,"row_number"() over(ORDER BY t.rank_cd desc ) as rownum from ( SELECT ts_rank(tsvnameaddress, to_tsquery(to_tsv_text('富桥综合市场'))) as rank_cd,* from icc_t_address ) t ) t
order by score desc

create or replace function to_tsv_text(i_text text) returns text as $$
import jieba
seg_list = jieba.cut(i_text, cut_all=True);
return("|".join(seg_list))
$$ language plpython3u;

select t2.* from(select t.*,
row_number() over (order by (ts_rank(tsvnameaddress,to_bigram_tsquery('富桥综合市场', '|'))) desc) as rownum
from icc_t_address t
where tsvnameaddress@@to_bigram_tsquery('富桥综合市场', '|')) t2

SELECT t2.rank_cd as score,t2.address,t2.name,t2.shortname,t2.hisname,
t2.szm,t2.pinyin,t2.x,t2.y,t2.type,t2.linkphone,t2.guid
from (SELECT ts_rank(tsvnameaddress, to_tsquery(t1.tsv_text)) as rank_cd,*
from (select * from icc_t_address,(select to_tsv_text('富桥综合市场') as tsv_text) t) t1
where tsvnameaddress@@to_tsquery(t1.tsv_text) order by rank_cd desc) t2