glusterfs 内存管理方式

glusterfs中的内存管理方式：

　　首先来看看glusterfs的内存管理结构吧：

 struct mem_pool {
         struct list_head  list;
         int               hot_count;
         int               cold_count;
         gf_lock_t         lock;
         unsigned long     padded_sizeof_type;
         void             *pool;
         void             *pool_end;
         int               real_sizeof_type;
         uint64_t          alloc_count;
         uint64_t          pool_misses;
         int               max_alloc;
         int               curr_stdalloc;
         int               max_stdalloc;
         char             *name;
         struct list_head  global_list;
 };

管理结构的信息量很简单，核心的数据项是list，每个要分配的内存块被一个双向链表串连起来管理。

　　接下来是创建内存池的接口：

 struct mem_pool *
 mem_pool_new_fn (unsigned long sizeof_type,
                  unsigned long count, char *name)
 {
         struct mem_pool  *mem_pool = NULL;
         unsigned long     padded_sizeof_type = ;
         void             *pool = NULL;
         int               i = ;
         int               ret = ;
         struct list_head *list = NULL;
         glusterfs_ctx_t  *ctx = NULL;

         if (!sizeof_type || !count) {
                 gf_log_callingfn ("mem-pool", GF_LOG_ERROR, "invalid argument");
                 return NULL;
         }
         padded_sizeof_type = sizeof_type + GF_MEM_POOL_PAD_BOUNDARY;

         mem_pool = GF_CALLOC (sizeof (*mem_pool), , gf_common_mt_mem_pool);
         if (!mem_pool)
                 return NULL;

         ret = gf_asprintf (&mem_pool->name, "%s:%s", THIS->name, name);
         if (ret < )
                 return NULL;

         if (!mem_pool->name) {
                 GF_FREE (mem_pool);
                 return NULL;
         }

         LOCK_INIT (&mem_pool->lock);
         INIT_LIST_HEAD (&mem_pool->list);
         INIT_LIST_HEAD (&mem_pool->global_list);

         mem_pool->padded_sizeof_type = padded_sizeof_type;
         mem_pool->cold_count = count;
         mem_pool->real_sizeof_type = sizeof_type;

         pool = GF_CALLOC (count, padded_sizeof_type, gf_common_mt_long);
         if (!pool) {
                 GF_FREE (mem_pool->name);
                 GF_FREE (mem_pool);
                 return NULL;
         }

         for (i = ; i < count; i++) {
                 list = pool + (i * (padded_sizeof_type));
                 INIT_LIST_HEAD (list);
                 list_add_tail (list, &mem_pool->list);
         }

         mem_pool->pool = pool;
         mem_pool->pool_end = pool + (count * (padded_sizeof_type));

         /* add this pool to the global list */
         ctx = THIS->ctx;
         if (!ctx)
                 goto out;

         list_add (&mem_pool->global_list, &ctx->mempool_list);

 out:
         return mem_pool;
 }

在第19行中申请了一个mem_pool内存管理结构，在初始化这个结构体后，40行申请了真正要使用的内存pool并把用mem_pool->list链表串起来。之后再记录内存池的开始和结束地址（53-54），再把这个结构加入全局管理。

再看一下申请后的内存是如何使用的呢？

 void *
 mem_get (struct mem_pool *mem_pool)
 {
         struct list_head *list = NULL;
         void             *ptr = NULL;
         int             *in_use = NULL;
         struct mem_pool **pool_ptr = NULL;

         if (!mem_pool) {
                 gf_log_callingfn ("mem-pool", GF_LOG_ERROR, "invalid argument");
                 return NULL;
         }

         LOCK (&mem_pool->lock);
         {
                 mem_pool->alloc_count++;
                 if (mem_pool->cold_count) {
                         list = mem_pool->list.next;
                         list_del (list);

                         mem_pool->hot_count++;
                         mem_pool->cold_count--;

                         if (mem_pool->max_alloc < mem_pool->hot_count)
                                 mem_pool->max_alloc = mem_pool->hot_count;

                         ptr = list;
                         in_use = (ptr + GF_MEM_POOL_LIST_BOUNDARY +
                                   GF_MEM_POOL_PTR);
                         *in_use = ;

                         goto fwd_addr_out;
                 }

                 /* This is a problem area. If we've run out of
                  * chunks in our slab above, we need to allocate
                  * enough memory to service this request.
                  * The problem is, these individual chunks will fail
                  * the first address range check in __is_member. Now, since
                  * we're not allocating a full second slab, we wont have
                  * enough info perform the range check in __is_member.
                  *
                  * I am working around this by performing a regular allocation
                  * , just the way the caller would've done when not using the
                  * mem-pool. That also means, we're not padding the size with
                  * the list_head structure because, this will not be added to
                  * the list of chunks that belong to the mem-pool allocated
                  * initially.
                  *
                  * This is the best we can do without adding functionality for
                  * managing multiple slabs. That does not interest us at present
                  * because it is too much work knowing that a better slab
                  * allocator is coming RSN.
                  */
                 mem_pool->pool_misses++;
                 mem_pool->curr_stdalloc++;
                 if (mem_pool->max_stdalloc < mem_pool->curr_stdalloc)
                         mem_pool->max_stdalloc = mem_pool->curr_stdalloc;
                 ptr = GF_CALLOC (, mem_pool->padded_sizeof_type,
                                  gf_common_mt_mem_pool);
                 gf_log_callingfn ("mem-pool", GF_LOG_DEBUG, "Mem pool is full. "
                                   "Callocing mem");

                 /* Memory coming from the heap need not be transformed from a
                  * chunkhead to a usable pointer since it is not coming from
                  * the pool.
                  */
         }
 fwd_addr_out:
         pool_ptr = mem_pool_from_ptr (ptr);
         *pool_ptr = (struct mem_pool *)mem_pool; //保存分配者地址
         ptr = mem_pool_chunkhead2ptr (ptr);
         UNLOCK (&mem_pool->lock);

         return ptr;
 }

从17行到33行可以看出，当需要内存时，glusterfs从mem_pool->list中分配内存。关键是：当内存不足时，mem_pool如何处理呢？55-63行处理这个问题：当内存不足时，它向系统申请了内存，并处理了内存的管理信息后，直接将内存返回给调用者。

最后看看内存的释放过程：

 void
 mem_put (void *ptr)
 {
         struct list_head *list = NULL;
         int    *in_use = NULL;
         void   *head = NULL;
         struct mem_pool **tmp = NULL;
         struct mem_pool *pool = NULL;

         if (!ptr) {
                 gf_log_callingfn ("mem-pool", GF_LOG_ERROR, "invalid argument");
                 return;
         }

         list = head = mem_pool_ptr2chunkhead (ptr);
         tmp = mem_pool_from_ptr (head); //取出分配者地址
         if (!tmp) {
                 gf_log_callingfn ("mem-pool", GF_LOG_ERROR,
                                   "ptr header is corrupted");
                 return;
         }

         pool = *tmp;
         if (!pool) {
                 gf_log_callingfn ("mem-pool", GF_LOG_ERROR,
                                   "mem-pool ptr is NULL");
                 return;
         }
         LOCK (&pool->lock);
         {

                 switch (__is_member (pool, ptr))
                 {
                 case :
                         in_use = (head + GF_MEM_POOL_LIST_BOUNDARY +
                                   GF_MEM_POOL_PTR);
                         if (!is_mem_chunk_in_use(in_use)) {
                                 gf_log_callingfn ("mem-pool", GF_LOG_CRITICAL,
                                                   "mem_put called on freed ptr %p of mem "
                                                   "pool %p", ptr, pool);
                                 break;
                         }
                         pool->hot_count--;
                         pool->cold_count++;
                         *in_use = ;
                         list_add (list, &pool->list);
                         break;
                 case -:
                         /* For some reason, the address given is within
                          * the address range of the mem-pool but does not align
                          * with the expected start of a chunk that includes
                          * the list headers also. Sounds like a problem in
                          * layers of clouds up above us. ;)
                          */
                         abort ();
                         break;
                 case :
                         /* The address is outside the range of the mem-pool. We
                          * assume here that this address was allocated at a
                          * point when the mem-pool was out of chunks in mem_get
                          * or the programmer has made a mistake by calling the
                          * wrong de-allocation interface. We do
                          * not have enough info to distinguish between the two
                          * situations.
                          */
                         pool->curr_stdalloc--;
                         GF_FREE (list);
                         break;
                 default:
                         /* log error */
                         break;
                 }
         }
         UNLOCK (&pool->lock);
 }

在switch语句中，在case 1中处理了内存池分配的过程。在case 0中处理内存不足的情况，从这里看出，glusterfs直接将内存释放了，正好与分配的过程完美的结合。