开发环境

spring 4.3.7 + springBoot 1.5.2 + dubbo 2.6.5 + mongoDB 4.0.0

连接池配置

mongo-pool.properties

spring.data.mongodb.address=127.0.0.1:,127.0.0.1:,127.0.0.1:
spring.data.mongodb.replica-set=mySet
spring.data.mongodb.database=test
spring.data.mongodb.username=test
spring.data.mongodb.password=test # Configure spring.data.mongodbDB Pool
spring.data.mongodb.min-connections-per-host=
spring.data.mongodb.max-connections-per-host=
spring.data.mongodb.threads-allowed-to-block-for-connection-multiplier=
spring.data.mongodb.server-selection-timeout=
spring.data.mongodb.max-wait-time=
spring.data.mongodb.max-connection-idel-time=
spring.data.mongodb.max-connection-life-time=
spring.data.mongodb.connect-timeout=
spring.data.mongodb.socket-timeout=
spring.data.mongodb.socket-keep-alive=false
spring.data.mongodb.ssl-enabled=false
spring.data.mongodb.ssl-invalid-host-name-allowed=false
spring.data.mongodb.always-use-m-beans=false
spring.data.mongodb.heartbeat-socket-timeout=
spring.data.mongodb.heartbeat-connect-timeout=
spring.data.mongodb.min-heartbeat-frequency=
spring.data.mongodb.heartbeat-frequency=
spring.data.mongodb.local-threshold=
spring.data.mongodb.authentication-database=admin

MongoSettingsProperties

import java.util.List;

import lombok.Data;

import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.PropertySource;
import org.springframework.stereotype.Component; @Data
@Component
@PropertySource(value = "classpath:mongo-pool.properties")
@ConfigurationProperties(prefix = "spring.data.mongodb")
public class MongoSettingsProperties { private List<String> address;
private String replicaSet;
private String database;
private String username;
private String password;
private Integer minConnectionsPerHost = ;
private Integer maxConnectionsPerHost = ;
private Integer threadsAllowedToBlockForConnectionMultiplier = ;
private Integer serverSelectionTimeout = ;
private Integer maxWaitTime = ;
private Integer maxConnectionIdleTime = ;
private Integer maxConnectionLifeTime = ;
private Integer connectTimeout = ;
private Integer socketTimeout = ;
private Boolean socketKeepAlive = false;
private Boolean sslEnabled = false;
private Boolean sslInvalidHostNameAllowed = false;
private Boolean alwaysUseMBeans = false;
private Integer heartbeatConnectTimeout = ;
private Integer heartbeatSocketTimeout = ;
private Integer minHeartbeatFrequency = ;
private Integer heartbeatFrequency = ;
private Integer localThreshold = ;
private String authenticationDatabase; }

MongoConfig

import java.util.ArrayList;
import java.util.List; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.mongodb.MongoDbFactory;
import org.springframework.data.mongodb.core.SimpleMongoDbFactory; import com.mongodb.MongoClient;
import com.mongodb.MongoClientOptions;
import com.mongodb.MongoCredential;
import com.mongodb.ServerAddress; @Configuration
public class MongoConfig {
// 覆盖默认的MongoDbFactory
@Bean
MongoDbFactory mongoDbFactory(MongoSettingsProperties mongoSettingsProperties) {
// 客户端配置(连接数、副本集群验证)
MongoClientOptions.Builder builder = new MongoClientOptions.Builder();
builder.connectionsPerHost(mongoSettingsProperties.getMaxConnectionsPerHost());
builder.minConnectionsPerHost(mongoSettingsProperties.getMinConnectionsPerHost());
if (mongoSettingsProperties.getReplicaSet() != null) {
builder.requiredReplicaSetName(mongoSettingsProperties.getReplicaSet());
}
builder.threadsAllowedToBlockForConnectionMultiplier(
mongoSettingsProperties.getThreadsAllowedToBlockForConnectionMultiplier());
builder.serverSelectionTimeout(mongoSettingsProperties.getServerSelectionTimeout());
builder.maxWaitTime(mongoSettingsProperties.getMaxWaitTime());
builder.maxConnectionIdleTime(mongoSettingsProperties.getMaxConnectionIdleTime());
builder.maxConnectionLifeTime(mongoSettingsProperties.getMaxConnectionLifeTime());
builder.connectTimeout(mongoSettingsProperties.getConnectTimeout());
builder.socketTimeout(mongoSettingsProperties.getSocketTimeout());
builder.sslEnabled(mongoSettingsProperties.getSslEnabled());
builder.sslInvalidHostNameAllowed(mongoSettingsProperties.getSslInvalidHostNameAllowed());
builder.alwaysUseMBeans(mongoSettingsProperties.getAlwaysUseMBeans());
builder.heartbeatFrequency(mongoSettingsProperties.getHeartbeatFrequency());
builder.minHeartbeatFrequency(mongoSettingsProperties.getMinHeartbeatFrequency());
builder.heartbeatConnectTimeout(mongoSettingsProperties.getHeartbeatConnectTimeout());
builder.heartbeatSocketTimeout(mongoSettingsProperties.getHeartbeatSocketTimeout());
builder.localThreshold(mongoSettingsProperties.getLocalThreshold()); MongoClientOptions mongoClientOptions = builder.build(); // MongoDB地址列表
List<ServerAddress> serverAddresses = new ArrayList<>();
for (String address : mongoSettingsProperties.getAddress()) {
String[] hostAndPort = address.split(":");
String host = hostAndPort[];
Integer port = Integer.parseInt(hostAndPort[]);
ServerAddress serverAddress = new ServerAddress(host, port);
serverAddresses.add(serverAddress);
}
System.out.println("serverAddresses:" + serverAddresses.toString()); // 连接认证
List<MongoCredential> mongoCredentialList = new ArrayList<>();
if (mongoSettingsProperties.getUsername() != null) {
mongoCredentialList.add(MongoCredential.createScramSha1Credential(
mongoSettingsProperties.getUsername(),
mongoSettingsProperties.getAuthenticationDatabase() != null ? mongoSettingsProperties
.getAuthenticationDatabase() : mongoSettingsProperties.getDatabase(),
mongoSettingsProperties.getPassword().toCharArray()));
}
System.out.println("mongoCredentialList:" + mongoCredentialList.toString()); // 创建客户端和Factory
MongoClient mongoClient = new MongoClient(serverAddresses,mongoCredentialList, mongoClientOptions);
MongoDbFactory mongoDbFactory = new SimpleMongoDbFactory(mongoClient,mongoSettingsProperties.getDatabase()); return mongoDbFactory;
}
}

查询百万数据

第一种:

        DBObject query = new BasicDBObject(); //setup the query criteria 设置查询条件
query.put("method", method);
query.put("ctime", (new BasicDBObject("$gte", bTime)).append("$lt", eTime)); logger.debug("query: {}", query); DBObject fields = new BasicDBObject(); //only get the needed fields. 设置需要获取哪些域
fields.put("_id", );
fields.put("uId", );
fields.put("ctime", ); DBCursor dbCursor = mongoTemplate.getCollection("collectionName").find(query, fields); while (dbCursor.hasNext()){
DBObject object = dbCursor.next();
logger.debug("object: {}", object);
//do something.
}

第二种:

Query query = new Query();
 query.addCriteria(Criteria.where("_id").is(orderId));

mongoTemplate.find(query, this.getEntityClass(), collectionName);

对于数据量巨大的情况下,第二种方法使用mongoTemplate.find()方法返回一个列表,如果不分页的话恐怕比较麻烦。第一种方法,使用游标逐个获取数据,同时可以指定只获取哪些域,而不是全部获取回来。

记录:

插入40万 耗时大约10秒

查询70万总数 耗时3毫秒

MongoDB设置连接池操作百万级以上数据的更多相关文章

  1. java操作mongodb(连接池)(转)

    原文链接: java操作mongodb(连接池) Mongo的实例其实就是一个数据库连接池,这个连接池里默认有10个链接.我们没有必要重新实现这个链接池,但是我们可以更改这个连接池的配置.因为Mong ...

  2. .Net连接字符串设置连接池大小显著提高数据库速度

    在访问mysql数据库时,如果在连接字符串中设置使用连接池,同时设置连接池大小,经测试,可以显著提高访问数据库时的速度. 连接字符串: connectionStrings>    <add ...

  3. 在Spark中通过Scala + Mongodb实现连接池

    How to implement connection pool in spark https://github.com/YulinGUO/BigDataTips/blob/master/spark/ ...

  4. 关于处理百万级大批量数据的mysql运行几个重要点

    处理大批量百万级的数据几点重要知识点: 一:设置php运行的内存配置 ini_set("memory_limit","1200M"); 在php.ini中有如下 ...

  5. Golang 连接 MongoDB使用连接池

    可以免费试用 MongoDB ,500MB 平时做测试没有问题啦,连接数据库可能因为网络有点慢,但是我们是测试啊,不在乎这点吧~ 这是怎么申请试用版的博客,感谢这位大佬.注册好用起来很方便~ 传送门 ...

  6. redis的连接方法|连接池|操作

    1.先看下redis的连接 import redis # 连接服务端 r = redis.Redis(host="127.0.0.1",port=6379) #获取所有的key值 ...

  7. redis连接池操作

    /** * @类描述 redis 工具 * @功能名 POJO * @author zxf * @date 2014年11月25日 */public final class RedisUtil { p ...

  8. 一种利用ADO连接池操作MySQL的解决方案(VC++)

    VC++连接MySQL数据库 常用的方式有三种:ADO.mysql++,mysql API ; 本文只讲述ADO的连接方式. 为什么要使用连接池? 对于简单的数据库应用,完全可以先创建一个常连接(此连 ...

  9. Python3 多线程(连接池)操作MySQL插入数据

    1.主要模块DBUtils : 允许在多线程应用和数据库之间连接的模块套件Threading : 提供多线程功能 2.创建连接池PooledDB 基本参数: mincached : 最少的空闲连接数, ...

随机推荐

  1. java常见报错及解决

    Java常见报错信息: Java 常见异常种类 Java Exception: 1.Error  2.Runtime Exception 运行时异常 3.Exception  4.throw 用户自定 ...

  2. [STM32F103]串口UART配置

    l 串口时钟使能,GPIO时钟使能: RCC_APB2PeriphClockCmd(); l 串口复位: USART_DeInit(); 这一步不是必须的 l GPIO端口模式设置: GPIO_Ini ...

  3. node启动本地服务器

    //下载依赖 cnpm i http cnpm i path cnpm i fsvar http = require('http'); var path = require('path'); var ...

  4. react基础学习 一

    1. 搭建环境 安装react脚手架  >npm install create-react-app -g 创建文件            >create-react-app 项目名称 启动 ...

  5. IDEA 常用插件

    Maven Helperp3pauto filling Java call argumentsCamelCaseVisualVM LauncherTranslation.ignoreGenerateA ...

  6. python-day19 Django模板,路由分发,ORM

    @获取文件所有数据 request.FILES: request.POST.get('fafafa')#拿到文件名: user = request.POST.get('user',None)#用get ...

  7. MongoDB(3)--Java 操作 MongoDB 的 API

    引入mongoDb 依赖 <dependency> <groupId>org.mongodb</groupId> <artifactId>mongo-j ...

  8. Jquery如何序列化form表单数据为JSON对象

    jquery提供的serialize方法能够实现. $("#searchForm").serialize();但是,观察输出的信息,发现serialize()方法做的是将表单中的数 ...

  9. Django之Form验证

    view.py 配置 from django import forms from django.forms import fields # Create your views here. class ...

  10. Python module ---- getopt

    optlist, argv = getopt.getopt(argv, 'cdsarRvn:e:E:f:F:w:O:m:M:o:g:') 模块getopt用于分割了main函数的参数,参数args一般 ...