MQTT 开源代理mosquitto的网络层封装相当sucks

最近学习MQTT协议，选择了当前比较流行的MQTT Broker “mosquitto”，但是在阅读代码过程中发现其网络底层库封装的相当差劲。

对于MQTT协议的变长头长度的读取上，基本上采取每次一个byte的方式进行读取判断，对于系统调用read的高代价来讲，真的是相当的浪费，也难怪其不能作为高并发的服务器进行处理。

当然mosquitto需要优化的地方还很多：

1. 使用poll而不是使用epoll （可能是处于跨平台考虑，如果linux下可以使用epoll替换），同时的就是刚才提到的 byte 读取网络数据

2. 订阅树的管理上，对于大量的请求断开或者重练效率比较低

3. 空闲空间管理机制优化和数据包发送方式的修改

4. 内存管理上malloc new 没有使用mem pool机制，在大并发情况下，内存管理容易出现问题

5. 锁遍地飞，如果采用reactor_

但是从另一个方面讲，mosquitto作为开源的实现，思路上还是比较清晰，为mqtt服务器开发提供了比较完备的参考，这也就是它的价值所在了。

#ifdef WITH_BROKER

int _mosquitto_packet_read(struct mosquitto_db *db, struct mosquitto *mosq)

#else

int _mosquitto_packet_read(struct mosquitto *mosq)

#endif

{

uint8_t byte;

ssize_t read_length;

int rc = 0;

if(!mosq) return MOSQ_ERR_INVAL;

if(mosq->sock == INVALID_SOCKET) return MOSQ_ERR_NO_CONN;

if(mosq->state == mosq_cs_connect_pending){

return MOSQ_ERR_SUCCESS;

}

/* This gets called if pselect() indicates that there is network data

* available - ie. at least one byte.  What we do depends on what data we

* already have.

* If we've not got a command, attempt to read one and save it. This should

* always work because it's only a single byte.

* Then try to read the remaining length. This may fail because it is may

* be more than one byte - will need to save data pending next read if it

* does fail.

* Then try to read the remaining payload, where 'payload' here means the

* combined variable header and actual payload. This is the most likely to

* fail due to longer length, so save current data and current position.

* After all data is read, send to _mosquitto_handle_packet() to deal with.

* Finally, free the memory and reset everything to starting conditions.

*/

if(!mosq->in_packet.command){

read_length = _mosquitto_net_read(mosq, &byte, 1);

if(read_length == 1){

mosq->in_packet.command = byte;

#ifdef WITH_BROKER

#  ifdef WITH_SYS_TREE

g_bytes_received++;

#  endif

/* Clients must send CONNECT as their first command. */

if(!(mosq->bridge) && mosq->state == mosq_cs_new && (byte&0xF0) != CONNECT) return MOSQ_ERR_PROTOCOL;

#endif

}else{

if(read_length == 0) return MOSQ_ERR_CONN_LOST; /* EOF */

#ifdef WIN32

errno = WSAGetLastError();

#endif

if(errno == EAGAIN || errno == COMPAT_EWOULDBLOCK){

return MOSQ_ERR_SUCCESS;

}else{

switch(errno){

case COMPAT_ECONNRESET:

return MOSQ_ERR_CONN_LOST;

default:

return MOSQ_ERR_ERRNO;

}

}

}

}

/* remaining_count is the number of bytes that the remaining_length

* parameter occupied in this incoming packet. We don't use it here as such

* (it is used when allocating an outgoing packet), but we must be able to

* determine whether all of the remaining_length parameter has been read.

* remaining_count has three states here:

*   0 means that we haven't read any remaining_length bytes

*   <0 means we have read some remaining_length bytes but haven't finished

*   >0 means we have finished reading the remaining_length bytes.

*/

if(mosq->in_packet.remaining_count <= 0){

do{

read_length = _mosquitto_net_read(mosq, &byte, 1);

if(read_length == 1){

mosq->in_packet.remaining_count--;

/* Max 4 bytes length for remaining length as defined by protocol.

* Anything more likely means a broken/malicious client.

*/

if(mosq->in_packet.remaining_count < -4) return MOSQ_ERR_PROTOCOL;

#if defined(WITH_BROKER) && defined(WITH_SYS_TREE)

g_bytes_received++;

#endif

mosq->in_packet.remaining_length += (byte & 127) * mosq->in_packet.remaining_mult;

mosq->in_packet.remaining_mult *= 128;

}else{

if(read_length == 0) return MOSQ_ERR_CONN_LOST; /* EOF */

#ifdef WIN32

errno = WSAGetLastError();

#endif

if(errno == EAGAIN || errno == COMPAT_EWOULDBLOCK){

return MOSQ_ERR_SUCCESS;

}else{

switch(errno){

case COMPAT_ECONNRESET:

return MOSQ_ERR_CONN_LOST;

default:

return MOSQ_ERR_ERRNO;

}

}

}

}while((byte & 128) != 0);

/* We have finished reading remaining_length, so make remaining_count

* positive. */

mosq->in_packet.remaining_count *= -1;

if(mosq->in_packet.remaining_length > 0){

mosq->in_packet.payload = _mosquitto_malloc(mosq->in_packet.remaining_length*sizeof(uint8_t));

if(!mosq->in_packet.payload) return MOSQ_ERR_NOMEM;

mosq->in_packet.to_process = mosq->in_packet.remaining_length;

}

}

while(mosq->in_packet.to_process>0){

read_length = _mosquitto_net_read(mosq, &(mosq->in_packet.payload[mosq->in_packet.pos]), mosq->in_packet.to_process);

if(read_length > 0){

#if defined(WITH_BROKER) && defined(WITH_SYS_TREE)

g_bytes_received += read_length;

#endif

mosq->in_packet.to_process -= read_length;

mosq->in_packet.pos += read_length;

}else{

#ifdef WIN32

errno = WSAGetLastError();

#endif

if(errno == EAGAIN || errno == COMPAT_EWOULDBLOCK){

if(mosq->in_packet.to_process > 1000){

/* Update last_msg_in time if more than 1000 bytes left to

* receive. Helps when receiving large messages.

* This is an arbitrary limit, but with some consideration.

* If a client can't send 1000 bytes in a second it

* probably shouldn't be using a 1 second keep alive. */

pthread_mutex_lock(&mosq->msgtime_mutex);

mosq->last_msg_in = mosquitto_time();

pthread_mutex_unlock(&mosq->msgtime_mutex);

}

return MOSQ_ERR_SUCCESS;

}else{

switch(errno){

case COMPAT_ECONNRESET:

return MOSQ_ERR_CONN_LOST;

default:

return MOSQ_ERR_ERRNO;

}

}

}

}

/* All data for this packet is read. */

mosq->in_packet.pos = 0;

#ifdef WITH_BROKER

#  ifdef WITH_SYS_TREE

g_msgs_received++;

if(((mosq->in_packet.command)&0xF5) == PUBLISH){

g_pub_msgs_received++;

}

#  endif

rc = mqtt3_packet_handle(db, mosq);

#else

rc = _mosquitto_packet_handle(mosq);

#endif

/* Free data and reset values */

_mosquitto_packet_cleanup(&mosq->in_packet);

pthread_mutex_lock(&mosq->msgtime_mutex);

mosq->last_msg_in = mosquitto_time();

pthread_mutex_unlock(&mosq->msgtime_mutex);

return rc;

}