elasticsearch 导入基础数据并索引之 geo_point

elasticsearch 中的地理信息存储, 有geo_point形式和geo_shape两种形式

此篇只叙述geo_point,

地理位置需要声明为特殊的类型, 不显示在mapping中定义的话, 需要

{
    "pin" : {
        "location" : {
            "lat" : 40.12,
            "lon" : -71.34
        },
        "tag" : ["food", "family"],
        "text" : "my favorite family restaurant"
    }
}

如果仍然要显示的在mapping中定义, 则需要将其声明为 geo_point格式

{
    "pin" : {
        "properties" : {
            "location" : {
                "type" : "geo_point"
            }
        }
    }
}

es的类型有: string, long, date, geo_point, 以后知道了在补充, text(for binary),

　　range(integer_range, float_range, long_range, double_range, date_range)

　　boolean, geo_point, geo_shape, ip, keyword, nested, token_count.. 可以参见这儿

多说一句: location的数据存放有3种形式:

　　

    1), location: lat + "," + lon　　　　// 最开始用的这个, 但是做 geoHashCellQuery查询测试时, 报错了
　　2) location: {

　　　　　　"lat": ...,
　　　　　　"lon": ...　　　　
　　　　} 　　　　　　　　　　　　　　// 这个是我使用的导入方式
　　3), location: [lon, lat]　　　　　　// 这个没用, 没测试, 没发言权     

1, 导入查询数据, 使用的建立mapping的方式, 因为需要声明ik分词器

package com.iwhere.geosearch;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.junit.Before;
import org.junit.Test;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.yaml.snakeyaml.tokens.StreamStartToken;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;

/**
 * 从json文件中导入数据到es中
 * @author 231
 *
 */
public final class ImportData {

    private TransportClient client;

    @Test
    public void test1() throws Exception {
        createMapping("test", "catchModel");
        importData("test", "catchModel");
        System.out.println("success");
    }

    /**
     * 插入https://www.elastic.co/blog/geo-location-and-search的测试数据
     * @throws Exception
     */
    @Test
    public void test2() throws Exception {
        String index = "geo";
        String type = "test";
        BulkRequestBuilder prepareBulk = client.prepareBulk();
        for (int i = 0; i < 50; i++) {
            XContentBuilder source = getJson(40 + i, -71.34 + i, "my favorite family restaurant");
            prepareBulk.add(client.prepareIndex(index, type).setSource(source));
        }
        BulkResponse response = prepareBulk.execute().actionGet();
    }

    public XContentBuilder getJson(double lat, double lon, String text) throws IOException {
        return XContentFactory.jsonBuilder()
                .startObject()
                    .startObject("pin")
                        .startObject("location").field("lat", lat).field("lon", lon).endObject()
                        .field("tag", "food", "family")
                        .field("text", text)
                    .endObject()
                .endObject();
    }

    /**
     * 导入数据
     * @throws Exception
     */
    public void importData(String index, String type) throws Exception {
        BufferedReader br = new BufferedReader(new FileReader(new File("D://catchModel.json")));
        StringBuilder sb = new StringBuilder();
        String line = null;
        while((line = br.readLine()) != null) {
            sb.append(line);
        }

        BulkRequestBuilder prepareBulk = client.prepareBulk();
        JSONArray parseArray = JSON.parseArray(sb.toString());
        for (Object object : parseArray) {
            // 强转为map, 否则报错  the number of object passed must be even
            Map<String, Object> source = (Map<String, Object>) object;
//            IndexResponse response = client.prepareIndex(index, type).setSource(source).get();
            XContentBuilder xContentBuilder = XContentFactory.jsonBuilder()
                .startObject()
                    .field("taskName", source.get("taskName"))
                    .field("sessionId", source.get("sessionId"))
                    .field("geoNum", source.get("geoNum"))
                    .field("geoLevel", source.get("geoLevel"))
                    .field("createTime", source.get("createTime"))
                    .field("endTime", source.get("endTime"))
                    .startObject("location").field("lat", source.get("lbLat"))
                            .field("lon", source.get("lbLng"))
//                    .field("location", source.get("lbLat") + "," + source.get("lbLng"))
//                            XContentFactory.jsonBuilder()
//                                        .startObject()
//                                            .field("lat", source.get("lbLat"))
//                                            .field("lon", source.get("lblng"))
//                                        .endObject())
                    .endObject();
            prepareBulk.add(client.prepareIndex(index, type).setSource(xContentBuilder));
        }
        BulkResponse response = prepareBulk.get();
        System.out.println(response);
    }

    /**
     * 创建mapping, 添加ik分词器等, 相当于创建数据库表
     * 索引库名: indices
     * 类型      : mappingType
     * field("indexAnalyzer", "ik"): 字段分词ik索引
     * field("searchAnalyzer", "ik"): ik分词查询
     * @throws Exception
     */
    public void createMapping(String indices, String type) throws Exception {

        // 创建index
        Map<String, Object> settings = new HashMap<>();
        settings.put("number_of_shards", 4);    // 分片数量
        settings.put("number_of_replicas", 0);    // 复制数量, 导入时最好为0, 之后2-3即可
        settings.put("refresh_interval", "10s");// 刷新时间

        CreateIndexRequestBuilder prepareCreate = client.admin().indices().prepareCreate(indices);
        prepareCreate.setSettings(settings);

        // 创建mapping
        XContentBuilder mapping = XContentFactory.jsonBuilder()
            .startObject()
                .startObject(type)
//                    .startObject("_ttl")//有了这个设置,就等于在这个给索引的记录增加了失效时间,
//                    //ttl的使用地方如在分布式下,web系统用户登录状态的维护.
//                        .field("enabled", true)//默认的false的
//                        .field("default", "5m")//默认的失效时间,d/h/m/s 即天/小时/分钟/秒
//                        .field("store", "yes")
//                        .field("index", "not_analyzed")
//                    .endObject()
//                     .startObject("_timestamp")//这个字段为时间戳字段.即你添加一条索引记录后,自动给该记录增加个时间字段(记录的创建时间),搜索中可以直接搜索该字段.
//                        .field("enabled", true)
//                        .field("store", "no")
//                        .field("index", "not_analyzed")
//                    .endObject()
                .startObject("properties")
                .startObject("taskName").field("type", "string").field("analyzer", "ik").field("searchAnalyzer", "ik").endObject()
                .startObject("sessionId").field("type", "string").endObject()
                .startObject("geoNum").field("type", "string").endObject()
                .startObject("grandPaGeoNum").field("type", "string").endObject()
                .startArray("sonGeoNum").endArray()
                .startObject("geoLevel").field("type", "long").endObject()
                .startObject("state").field("type", "long").endObject()
                .startObject("createTime").field("type", "date").endObject()
                .startObject("endTime").field("type", "date").endObject()
                .startObject("location")
                    .field("type", "geo_point").field("geohash_prefix", true).field("geohash_precision", "1km").endObject()/*.field("lat_lon", true)*/.endObject()
            .endObject().endObject();
        System.out.println(mapping.string());
//        PutMappingResponse actionGet = client.admin().|indices().preparePutMapping(indices).setType(indices).setSource(mapping).execute().actionGet();
        prepareCreate.addMapping(type, mapping);
        CreateIndexResponse response = prepareCreate.execute().actionGet();
        System.out.println(response);
    }

    @Before
    public void before() {
        ClassPathXmlApplicationContext context = new ClassPathXmlApplicationContext("classpath:spring/applicationContxt-escluster.xml");
        client = context.getBean(TransportClient.class);
    }
}

2, 地理位置查询

package com.iwhere.geosearch;

import java.net.InetSocketAddress;

import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.geo.GeoDistance;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.geo.ShapeRelation;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.unit.DistanceUnit;
import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder;
import org.elasticsearch.index.query.GeoDistanceQueryBuilder;
import org.elasticsearch.index.query.GeoDistanceRangeQueryBuilder;
import org.elasticsearch.index.query.GeoPolygonQueryBuilder;
import org.elasticsearch.index.query.GeoShapeQueryBuilder;
import org.elasticsearch.index.query.GeohashCellQuery.Builder;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.Before;
import org.junit.Test;

/**
 * 使用dsl查询
 * @author 231
 */
public class JavaESGEO {

    private TransportClient client;

    /**
     * Caused by: [test] QueryParsingException[Field [location] is not a geo_shape]
　　　* 报错, 没运行出来
     */
    @Test
    public void testGeoShapeQuery() {
        GeoShapeQueryBuilder geoShapeQuery = QueryBuilders.geoShapeQuery("location", ShapeBuilder.newMultiPoint()
                    .point(0, 0)
                    .point(0, 10)
                    .point(10, 10)
                    .point(10, 0)
                    .point(0, 0)
                    , ShapeRelation.WITHIN);
        System.out.println(geoShapeQuery);

        SearchResponse response = client.prepareSearch("geo")
                        .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
                        .setQuery(geoShapeQuery).get();
    }

    /**
     * Caused by: java.lang.IllegalStateException: Shape with name [AVqw3mb-kOe4Yke4p-lh] found but missing shape field
     */
    @Test
    public void testGeoShapeQuery1() {
        GeoShapeQueryBuilder queryBuilder = QueryBuilders.geoShapeQuery(
                "model.location",     // field
                "AVqxrMyikOe4Yke4p_Wx", // id of document
                "catchModel", ShapeRelation.WITHIN)    // type, relation
            .indexedShapeIndex("test")    // name of index
            .indexedShapePath("location");    // filed specified as path
        SearchResponse response = client.prepareSearch()
                .setQuery(queryBuilder).execute().actionGet();
        String string = response.getHits().getHits().toString();
        System.out.println(string);
    }

    /**
     * 使用 BoundingBoxQuery进行查询
     */
    @Test
    public void testGeoBoundingBoxQuery( ){
        GeoBoundingBoxQueryBuilder queryBuilder = QueryBuilders.geoBoundingBoxQuery("location")
            .topRight(40.0, 117)
            .bottomLeft(39.9, 116);
        SearchResponse searchResponse = client.prepareSearch("test")
                .setQuery(queryBuilder).get();
        System.out.println(searchResponse);
        System.err.println(searchResponse.getHits().totalHits());
    }

    /**
     * distance query  查询
     */
    @Test
    public void testDistanceQuery() {
        GeoDistanceQueryBuilder queryBuilder = QueryBuilders.geoDistanceQuery("location")
            .point(40, 116.5)
            .distance(20, DistanceUnit.KILOMETERS)
            .optimizeBbox("memory")
            .geoDistance(GeoDistance.ARC);
        SearchResponse response = client.prepareSearch("geo", "test")
                .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
                .setQuery(queryBuilder).execute().actionGet();
        System.out.println(response);
        System.err.println(response.getHits().totalHits());
    }

    /**
     * 环形查询
     */
    @Test
    public void testDistanceRangeQuery() {
        GeoDistanceRangeQueryBuilder queryBuilder = QueryBuilders.geoDistanceRangeQuery("location")
                            .point(40,  116.5)            // 中心点
                            .from("20km")                // 内环
                            .to("25km")                    //外环
                            .includeLower(true)            // 包含上届
                            .includeUpper(true)            // 包含下届
                            .optimizeBbox("memory")        // 边界框
                            .geoDistance(GeoDistance.SLOPPY_ARC);
        SearchResponse response = client.prepareSearch("test")
            .setSearchType(SearchType.DFS_QUERY_AND_FETCH)
            .setQuery(queryBuilder).execute().actionGet();
        System.out.println(response);
        System.out.println(response.getHits().totalHits());
    }

    /**
     * 多边形查询
     */
    @Test
    public void testPolygonQuery() {
        GeoPolygonQueryBuilder queryBuilder = QueryBuilders.geoPolygonQuery("location")
                                .addPoint(39, 116)
                                .addPoint(39, 117)
                                .addPoint(40, 117);
        SearchResponse response = client.prepareSearch("test", "geo")
            .setQuery(queryBuilder).get();
        System.out.println(response);
        System.err.println(response.getHits().totalHits());
    }

    /**
     * geoHash查询
     * 要使用, 需要先开启
     *  "location": {
          "type":               "geo_point",
          "geohash_prefix":     true,
          "geohash_precision":  "1km" // 精度, 可在mapping中指定, 也可在代码中指定
     */
    @Test
    public void testGeoHashCellQuery() {
        Builder precision = QueryBuilders.geoHashCellQuery("location",
                new GeoPoint(39.9, 116))
            .neighbors(true)
            .precision(3);
        SearchResponse response = client.prepareSearch("test")
                    .setQuery(precision).get();
        System.out.println(response);
        System.err.println(response.getHits().totalHits());
    }

    @Before
    public void testBefore() {
        Settings settings = Settings.settingsBuilder().put("cluster.name", "wenbronk_escluster")
                    .put("client.transport.sniff", true).build();
        client = TransportClient.builder().settings(settings).build()
                 .addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress("192.168.50.37", 9300)));
        System.out.println("success to connect escluster");
    }
}

其他配置信息见: spring整合java一章

基础数据从mongodb中拷贝来的, 在github有一个小量的数据