亿级别记录的mongodb批量导入Es的java代码完整实现

针对mongodb亿级别或者十亿级别的模糊查询，效率不高，解决方式是使用Es查询，这样就需要把数据导入的ES中

完整的代码实现如下所示：(仅供参考)

import java.io.IOException;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.codec.binary.Base64;
import org.apache.http.HttpHost;
import org.bson.types.ObjectId;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;

import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.MongoClient;
import com.mongodb.MongoException;

public class Test {

    public static void main(String[] args) throws IOException {
        int pageSize=10000;

        try {
            MongoClient mongo = new MongoClient("localhost", 27017);

            /**** Get database ****/
            // if database doesn't exists, MongoDB will create it for you
            DB db = mongo.getDB("www");

            /**** Get collection / table from 'testdb' ****/
            // if collection doesn't exists, MongoDB will create it for you
            DBCollection table = db.getCollection("person");

             RestHighLevelClient client = new RestHighLevelClient(
                        RestClient.builder(
                                new HttpHost("localhost", 9200, "http")));
            DBCursor dbObjects;
            Long cnt=table.count();
            System.out.println(table.getStats().toString());
            Long page=getPageSize(cnt,pageSize);
            ObjectId lastIdObject=null;
            Long start=System.currentTimeMillis();
            long ss=start;
            for(Long i=0L;i<page;i++) {
                start=System.currentTimeMillis();
                dbObjects=getCursorForCollection(table, lastIdObject, pageSize);
                System.out.println("第"+(i+1)+"次查询，耗时:"+(System.currentTimeMillis()-start)+" 毫秒");
                List<DBObject> objs=dbObjects.toArray();
                start=System.currentTimeMillis();
                batchInsertToEsSync(client,objs,"person","doc");
                lastIdObject=(ObjectId) objs.get(objs.size()-1).get("_id");
                System.out.println("第"+(i+1)+"次插入，耗时:"+(System.currentTimeMillis()-start)+" 毫秒");
            }
            System.out.println("耗时:"+(System.currentTimeMillis()-ss)/1000+"秒");
        } catch (UnknownHostException e) {
            e.printStackTrace();
        } catch (MongoException e) {
            e.printStackTrace();
        }

    }

    public static void batchInsertToEsSync(RestHighLevelClient client,List<DBObject> objs,String tableName,String type) throws IOException {
        BulkRequest bulkRequest=new BulkRequest();
        for(DBObject obj:objs) {
            IndexRequest req = new IndexRequest(tableName, type);
            Map<String,Object> map=new HashMap<>();
            for(String key:obj.keySet()) {
                if("_id".equalsIgnoreCase(key)) {
                    map.put("id", obj.get(key));
                }else {
                    String valStr="";
                    Object val=obj.get(key);
                    if(val!=null) {
                        valStr=Base64.encodeBase64String(val.toString().getBytes());
                    }
                    map.put(key, valStr);
                }
            }
            req.id(map.get("id").toString());
            req.source(map, XContentType.JSON);
            bulkRequest.add(req);
        }
        BulkResponse bulkResponse=client.bulk(bulkRequest);
        for (BulkItemResponse bulkItemResponse : bulkResponse) {
            if (bulkItemResponse.isFailed()) {
                System.out.println(bulkItemResponse.getId()+","+bulkItemResponse.getFailureMessage());
            }
        }
    }

    public static DBCursor getCursorForCollection(DBCollection collection,ObjectId lastIdObject,int pageSize) {
        DBCursor dbObjects=null;
        if(lastIdObject==null) {
            lastIdObject=(ObjectId) collection.findOne().get("_id");
        }
        BasicDBObject query=new BasicDBObject();
        query.append("_id",new BasicDBObject("$gt",lastIdObject));
        BasicDBObject sort=new BasicDBObject();
        sort.append("_id",1);
        dbObjects=collection.find(query).limit(pageSize).sort(sort);
        return dbObjects;
    }

    public static Long getPageSize(Long cnt,int pageSize) {
        return cnt%pageSize==0?cnt/pageSize:cnt/pageSize+1;
    }