redis源码解读--内存分配zmalloc

目录

• 主要函数
• void *zmalloc(size_t size)
• void *zcalloc(size_t size)
• void *zrealloc(void *ptr, size_t size)
• void zfree(void *ptr)
• char *zstrdup(const char *s)
• size_t zmalloc_used_memory(void)
• void zmalloc_set_oom_handler(void (*oom_handler)(size_t))
• size_t zmalloc_get_rss(void)
• int zmalloc_get_allocator_info(size_t *allocated, size_t *active, size_t *resident)
• size_t zmalloc_get_smap_bytes_by_field(char *field, long pid)
• size_t zmalloc_get_private_dirty(long pid)
• size_t zmalloc_get_memory_size(void)
• 总结

主要函数

• void *zmalloc(size_t size);

• void *zcalloc(size_t size);

• void *zrealloc(void *ptr, size_t size);

• void zfree(void *ptr);

• char *zstrdup(const char *s);

• size_t zmalloc_used_memory(void);

• void zmalloc_set_oom_handler(void (*oom_handler)(size_t));

• size_t zmalloc_get_rss(void);

• int zmalloc_get_allocator_info(size_t *allocated, size_t *active, size_t *resident);

• size_t zmalloc_get_private_dirty(long pid);

• size_t zmalloc_get_smap_bytes_by_field(char *field, long pid);

• size_t zmalloc_get_memory_size(void);

• void zlibc_free(void *ptr);

void *zmalloc(size_t size)

void *zmalloc(size_t size) {
    void *ptr = malloc(size+PREFIX_SIZE);

    if (!ptr) zmalloc_oom_handler(size);
#ifdef HAVE_MALLOC_SIZE
    update_zmalloc_stat_alloc(zmalloc_size(ptr));
    return ptr;
#else
    *((size_t*)ptr) = size;
    update_zmalloc_stat_alloc(size+PREFIX_SIZE);
    return (char*)ptr+PREFIX_SIZE;
#endif
}

参数size是需要分配的空间大小。事实上我们需要分配的空间大小为size+PREFIX_SIZE。PREFIX_SIZE是根据平台的不同和HAVE_MALLOC_SIZE宏定义控制的。如果malloc()函数调用失败，就会调用zmalloc_oom_handler()函数来打印异常，并且会终止函数，zmalloc_oom_handler其实是一个函数指针，真正调用的函数是zmalloc_default_oom(),zmalloc_default_oom()函数源码如下:

static void zmalloc_default_oom(size_t size) {
    fprintf(stderr, "zmalloc: Out of memory trying to allocate %zu bytes\n",
        size);
    fflush(stderr);
    abort();
}

static void (*zmalloc_oom_handler)(size_t) = zmalloc_default_oom;

内存分配成功之后，会依据HAVE_MALLOC_SIZE的控制对前八个字节操作，用以记录分配内存的长度，会在update_zmalloc_stat_alloc()宏定义函数中更新used_memory这个静态变量的值，update_zmalloc_stat_alloc()源码如下:

#define update_zmalloc_stat_alloc(__n) do { \
    size_t _n = (__n); \
    if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1)); \
    atomicIncr(used_memory,__n); \
} while(0)

if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1));

这一行是为了将不为sizeof(long)的_n对sizeof(long)补齐

atomicIncr(used_memory,__n);会调用__atomic_add_fetch(&var,(count),__ATOMIC_RELAXED);用于保证更新used_memory变量的操作是一个原子操作。

void *zcalloc(size_t size)

void *zcalloc(size_t size) {
    void *ptr = calloc(1, size+PREFIX_SIZE);

    if (!ptr) zmalloc_oom_handler(size);
#ifdef HAVE_MALLOC_SIZE
    update_zmalloc_stat_alloc(zmalloc_size(ptr));
    return ptr;
#else
    *((size_t*)ptr) = size;
    update_zmalloc_stat_alloc(size+PREFIX_SIZE);
    return (char*)ptr+PREFIX_SIZE;
#endif
}

zcalloc函数和zmalloc函数处理的思路很是相似，就不做太多的解释了

void *zrealloc(void *ptr, size_t size)

void *zrealloc(void *ptr, size_t size) {
    // 如果没有定义HAVE_MALLOC_SIZE，就说明PREFIX_SIZE宏定义不为0，那么ptr并不是该段内存真正的开始地址
#ifndef HAVE_MALLOC_SIZE
    void *realptr;
#endif
    size_t oldsize;
    void *newptr;

    if (ptr == NULL) return zmalloc(size);
// 根据HAVE_MALLOC_SIZE宏定义，oldsize,newptr获取方式不一样，以及更新used_memory的细节
#ifdef HAVE_MALLOC_SIZE
    oldsize = zmalloc_size(ptr);
    newptr = realloc(ptr,size);
    if (!newptr) zmalloc_oom_handler(size);

    update_zmalloc_stat_free(oldsize);
    update_zmalloc_stat_alloc(zmalloc_size(newptr));
    return newptr;
#else
    realptr = (char*)ptr-PREFIX_SIZE;
    oldsize = *((size_t*)realptr);
    newptr = realloc(realptr,size+PREFIX_SIZE);
    if (!newptr) zmalloc_oom_handler(size);

    *((size_t*)newptr) = size;
    update_zmalloc_stat_free(oldsize+PREFIX_SIZE);
    update_zmalloc_stat_alloc(size+PREFIX_SIZE);
    return (char*)newptr+PREFIX_SIZE;
#endif
}

大致思路就是根据新的size进行重新分配内存，并且对used_memory变量进行更新。只不过获取原内存大小方式不一样，根据HAVE_MALLOC_SIZE进行区分。

void zfree(void *ptr)

void zfree(void *ptr) {
#ifndef HAVE_MALLOC_SIZE
    void *realptr;
    size_t oldsize;
#endif

    if (ptr == NULL) return;
#ifdef HAVE_MALLOC_SIZE
    update_zmalloc_stat_free(zmalloc_size(ptr));
    free(ptr);
#else
    realptr = (char*)ptr-PREFIX_SIZE;
    oldsize = *((size_t*)realptr);
    update_zmalloc_stat_free(oldsize+PREFIX_SIZE);
    free(realptr);
#endif
}

其实zfree函数和zrealloc函数做法差不到太多，都是对oldsize和realptr对HAVE_MALLOC_SIZE有无声明分别进行操作。

char *zstrdup(const char *s)

char *zstrdup(const char *s) {
    size_t l = strlen(s)+1;
    char *p = zmalloc(l);

    memcpy(p,s,l);
    return p;
}

该函数是创建一个字符串副本

size_t zmalloc_used_memory(void)

size_t zmalloc_used_memory(void) {
    size_t um;
    atomicGet(used_memory,um);
    return um;
}

获取used_memory变量的值，主要保证原子操作(在atomicGet(used_memory,um);中保证)

void zmalloc_set_oom_handler(void (*oom_handler)(size_t))

void zmalloc_set_oom_handler(void (*oom_handler)(size_t)) {
    zmalloc_oom_handler = oom_handler;
}

主要用来设置内存分配失败处理函数指针zmalloc_oom_handler的值

size_t zmalloc_get_rss(void)

size_t zmalloc_get_rss(void) {
    int page = sysconf(_SC_PAGESIZE);
    size_t rss;
    char buf[4096];
    char filename[256];
    int fd, count;
    char *p, *x;

    snprintf(filename,256,"/proc/%d/stat",getpid());
    if ((fd = open(filename,O_RDONLY)) == -1) return 0;
    if (read(fd,buf,4096) <= 0) {
        close(fd);
        return 0;
    }
    close(fd);

    p = buf;
    count = 23; /* RSS is the 24th field in /proc/<pid>/stat */
    while(p && count--) {
        p = strchr(p,' ');
        if (p) p++;
    }
    if (!p) return 0;
    x = strchr(p,' ');
    if (!x) return 0;
    *x = '\0';

    rss = strtoll(p,NULL,10);
    rss *= page;
    return rss;
}

返回驻留集大小

int zmalloc_get_allocator_info(size_t *allocated, size_t *active, size_t *resident)

#if defined(USE_JEMALLOC)
int zmalloc_get_allocator_info(size_t *allocated,
                               size_t *active,
                               size_t *resident) {
    uint64_t epoch = 1;
    size_t sz;
    *allocated = *resident = *active = 0;
    /* Update the statistics cached by mallctl. */
    sz = sizeof(epoch);
    je_mallctl("epoch", &epoch, &sz, &epoch, sz);
    sz = sizeof(size_t);
    /* Unlike RSS, this does not include RSS from shared libraries and other non
     * heap mappings. */
    je_mallctl("stats.resident", resident, &sz, NULL, 0);
    /* Unlike resident, this doesn't not include the pages jemalloc reserves
     * for re-use (purge will clean that). */
    je_mallctl("stats.active", active, &sz, NULL, 0);
    /* Unlike zmalloc_used_memory, this matches the stats.resident by taking
     * into account all allocations done by this process (not only zmalloc). */
    je_mallctl("stats.allocated", allocated, &sz, NULL, 0);
    return 1;
}
#else
int zmalloc_get_allocator_info(size_t *allocated,
                               size_t *active,
                               size_t *resident) {
    *allocated = *resident = *active = 0;
    return 1;
}
#endif

获取分配器的信息，主要在使用jemalloc前提下使用，获取jemalloc分配的信息，详细信息可在http://jemalloc.net/jemalloc.3.html查阅

size_t zmalloc_get_smap_bytes_by_field(char *field, long pid)

#if defined(HAVE_PROC_SMAPS)
size_t zmalloc_get_smap_bytes_by_field(char *field, long pid) {
    char line[1024];
    size_t bytes = 0;
    int flen = strlen(field);
    FILE *fp;

    if (pid == -1) {
        // /proc/pid/smaps反应了运行时的进程的内存影响，系统的运行时库(so)，堆，栈信息均可在其中看到。
        fp = fopen("/proc/self/smaps","r");
    } else {
        char filename[128];
        snprintf(filename,sizeof(filename),"/proc/%ld/smaps",pid);
        fp = fopen(filename,"r");
    }

    if (!fp) return 0;
    while(fgets(line,sizeof(line),fp) != NULL) {
        if (strncmp(line,field,flen) == 0) {
            char *p = strchr(line,'k');
            if (p) {
                *p = '\0';
                bytes += strtol(line+flen,NULL,10) * 1024;
            }
        }
    }
    fclose(fp);
    return bytes;
}
#else
size_t zmalloc_get_smap_bytes_by_field(char *field, long pid) {
    ((void) field);
    ((void) pid);
    return 0;
}
#endif

获取/proc/pid/smaps中某一个field的字节大小

size_t zmalloc_get_private_dirty(long pid)

size_t zmalloc_get_private_dirty(long pid) {
    return zmalloc_get_smap_bytes_by_field("Private_Dirty:",pid);
}

获取Rss中已改写的私有页面页面大小

size_t zmalloc_get_memory_size(void)

size_t zmalloc_get_memory_size(void) {
#if defined(__unix__) || defined(__unix) || defined(unix) || \
    (defined(__APPLE__) && defined(__MACH__))
#if defined(CTL_HW) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM64))
    int mib[2];
    mib[0] = CTL_HW;
#if defined(HW_MEMSIZE)
    mib[1] = HW_MEMSIZE;            /* OSX. --------------------- */
#elif defined(HW_PHYSMEM64)
    mib[1] = HW_PHYSMEM64;          /* NetBSD, OpenBSD. --------- */
#endif
    int64_t size = 0;               /* 64-bit */
    size_t len = sizeof(size);
    if (sysctl( mib, 2, &size, &len, NULL, 0) == 0)
        return (size_t)size;
    return 0L;          /* Failed? */

#elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
    /* FreeBSD, Linux, OpenBSD, and Solaris. -------------------- */
    return (size_t)sysconf(_SC_PHYS_PAGES) * (size_t)sysconf(_SC_PAGESIZE);

#elif defined(CTL_HW) && (defined(HW_PHYSMEM) || defined(HW_REALMEM))
    /* DragonFly BSD, FreeBSD, NetBSD, OpenBSD, and OSX. -------- */
    int mib[2];
    mib[0] = CTL_HW;
#if defined(HW_REALMEM)
    mib[1] = HW_REALMEM;        /* FreeBSD. ----------------- */
#elif defined(HW_PHYSMEM)
    mib[1] = HW_PHYSMEM;        /* Others. ------------------ */
#endif
    unsigned int size = 0;      /* 32-bit */
    size_t len = sizeof(size);
    if (sysctl(mib, 2, &size, &len, NULL, 0) == 0)
        return (size_t)size;
    return 0L;          /* Failed? */
#else
    return 0L;          /* Unknown method to get the data. */
#endif
#else
    return 0L;          /* Unknown OS. */
#endif
}

获取物理内存的字节数

总结

看了redis内存分配的源码后，其实没有相信中的那么难以理解，或许只是心理上的作用，当然也说明redis源码写得真的是好，让我这种渣渣都能轻而易举的看懂，并且注释也很少，这里的函数几乎都是对glibc的malloc中的函数进行了一层包装，并且维护了一个叫做used_memory的全局变量，并且每一次对全局变量的操作都是原子操作。也对一些常用的函数进行了封装，例如：获取rss的大小，获取/proc/pid/smaps文件中某一field占用字节数的大小，获取物理内存字节数等等，总的来说，收益匪浅，没想到内存操作可以做到这样简单。