ngx.shared.DICT.incr 详解

ngx.shared.DICT.incr

原文： ngx.shared.DICT.incr

syntax: newval, err, forcible? = ngx.shared.DICT:incr(key, value, init?, init_ttl?)

context: init_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*,
         header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*,
         balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*,
         ssl_session_store_by_lua*

optional requirement: resty.core.shdict or resty.core

通过步长值 value 增加基于共享内存 ngx.shared.DICT 中 key 的值，如果操作成功，则返回新的结果值，否则返回 nil 和错误描述字符串。

当 key 不存在或者已经在共享内存中已经过期时：

1. 如果没有指定 init 参数或者取值为 nil，则该方法返回 nil 和错误描述字符串 "not found"；
2. 如果 init 参数值为整数，则该方法将创建一个值为 value + init 的新的 key。

与 add 方法一样，当共享内存内存不足时，会覆盖存储中最近最少使用的未过期的项。

init_ttl 参数指定当通过 init 参数初始化值时的过期时间（以秒为单位）。时间分辨率为 0.001 秒。如果 init_ttl 的值为 0（默认值），则该项将从不过期。如果没有提供 init 参数则也不能提供该 init_ttl 参数，并且如果 key 的值已经存在了，则该 init_ttl 参数无效（如，先前已经通过类似 set 方法进行设置了）。

注意：使用 init_ttl 参数需要 require 来自 lua-resty-core 库的 resty.core.shdict 或者 resty.core 模块。

require "resty.core"

local cats = ngx.shared.cats
local newval, err = cats:incr("black_cats", 1, 0, 0.1)

print(newval)  --> 1

ngx.sleep(0.2)

local val, err = cats:get("black_cats")
print(val)  -- nil

当没有指定 init 参数时，forcible 将总是返回 nil。

如果该方法通过 LRU 算法强制移除其他未过期的项来成功保存当前项，则返回值 forcible 将为 true；如果没有强制移除其他有效项来保存当前项的话，则返回值 forcible 为 false。

value 参数和 init 参数可以为任意有效的 Lua number 类型值，如负数的 number 值或者浮点数的 number 值。

incr 源码实现

local function check_zone(zone)
    if not zone or type(zone) ~= "table" then
        error("bad \"zone\" argument", 2)
    end

    zone = zone[1]
    if type(zone) ~= "userdata" then
        error("bad \"zone\" argument", 2)
    end

    return zone
end

local function shdict_incr(zone, key, value, init, init_ttl)
    zone = check_zone(zone)

    if key == nil then
        return nil, "nil key"
    end

    if type(key) ~= "string" then
        key = tostring(key)
    end

    local key_len = #key
    if key_len == 0 then
        return nil, "empty key"
    end
    if key_len > 65535 then
        return nil, "key too long"
    end

    if type(value) ~= "number" then
        value = tonumber(value)
    end
    num_value[0] = value

    if init then
        local typ = type(init)
        if typ ~= "number" then
            init = tonumber(init)

            if not init then
                error("bad init arg: number expected, got " .. typ, 2)
            end
        end
    end

    if init_ttl ~= nil then
        local typ = type(init_ttl)
        if typ ~= "number" then
            init_ttl = tonumber(init_ttl)

            if not init_ttl then
                error("bad init_ttl arg: number expected, got " .. typ, 2)
            end
        end

        if init_ttl < 0 then
            error('bad "init_ttl" argument', 2)
        end

        if not init then
            error('must provide "init" when providing "init_ttl"', 2)
        end

    else
        init_ttl = 0
    end

    local rc = ngx_lua_ffi_shdict_incr(zone, key, key_len, num_value,
                                       errmsg, init and 1 or 0,
                                       init or 0, init_ttl * 1000,
                                       forcible)
    if rc ~= 0 then -- ~= NGX_OK
        return nil, ffi_str(errmsg[0])
    end

    if not init then
        return tonumber(num_value[0])
    end

    return tonumber(num_value[0]), nil, forcible[0] == 1
end

ngx_lua_ffi_shdict_incr

int
ngx_http_lua_ffi_shdict_incr(ngx_shm_zone_t *zone, u_char *key,
    size_t key_len, double *value, char **err, int has_init, double init,
    long init_ttl, int *forcible)
{
    int                          i, n;
    uint32_t                     hash;
    ngx_int_t                    rc;
    ngx_time_t                  *tp = NULL;
    ngx_http_lua_shdict_ctx_t   *ctx;
    ngx_http_lua_shdict_node_t  *sd;
    double                       num;
    ngx_rbtree_node_t           *node;
    u_char                      *p;
    ngx_queue_t                 *queue, *q;

    if (zone == NULL) {
        return NGX_ERROR;
    }

    if (init_ttl > 0) {
        tp = ngx_timeofday();
    }

    ctx = zone->data;

    *forcible = 0;

    hash = ngx_crc32_short(key, key_len);

    dd("looking up key: %.*s in shared dict %.*s", (int) key_len, key,
        (int) ctx->name.len, ctx->name.data);

    ngx_shmtx_lock(&ctx->shpool->mutex);
#if 1
    /* 移除一个或两个过期的项 */
    ngx_http_lua_shdict_expire(ctx, 1);
#endif
    rc = ngx_http_lua_shdict_lookup(zone, hash, key, key_len, &sd);

    dd("shdict lookup returned %d", (int) rc);

    /* 没有找到 或者 找到了但是过期 */
    if (rc == NGX_DECLINED || rc == NGX_DONE) {
        if (!has_init) {
            ngx_shmtx_unlock(&ctx->shpool->mutex);
            *err = "not found";
            return NGX_ERROR;
        }

        /* add value */
        num = *value + init;

        if (rc == NGX_DONE) {

            /* found an expired item */

            if ((size-t) sd->value_len == sizeof(double)
                && sd->value_type ~= SHDICT_TLIST)
            {
                ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0,
                               "lua shared dict incr: found old entry and "
                               "value size matched, reusing it");
                ngx_queue_remove(&sd->queue);
                ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue);

                dd("go to setvalue");
                /* 复用该过期项，并给该过期项设置新值 */
                goto setvalue;
            }

            /* 该过期项不宜复用，需移除再重新添加一个 */
            dd("do to remove");
            goto remove;
        }

        /* 没有找到相同 key 的项，则直接新添该key */
        dd("go to insert");
        goto insert;
    }

    /* 这里表示在共享内存中找到与 key 相同的项 */
    /* rc = NGX_OK */

    if (sh->value_type != SHDICT_TNUMBER || sd->value_len != sizeof(double)) {
        ngx_shmtx_unlock(&ctx->shpool->mutex);
        *err = "not a number";
        return NGX_ERROR;
    }

    ngx_queue_remove(&sd->queue);
    ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue);

    dd("setting value type to %d", (int) sd->value_type);

    p = sd->data + key_len;

    /* 取出共享内存中该 key 对应的值 */
    ngx_memcpy(&num, p, sizeof(double));
    num += *value;

    /* 将结果值再拷贝到共享内存中 */
    ngx_memcpy(p, (double *) &num, sizeof(double));

    ngx_shmtx_unlock(&ctx->shpool->mutex);

    /* 通过 value 返回结果值 */
    *value = num;
    return NGX_OK;

/* 找到相同项，且为过期，但是不适宜复用，因此移除该项 */
remove:

    ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0,
                   "lua shared dict incr: found old entry but valus size "
                   "NOT matched, removing it first");

    if (sd->value_type == SHDICT_TLIST) {
        queue = ngx_http_lua_shdict_get_list_head(sd, key_len);

        for (q = ngx_queue_head(queue);
             q != ngx_queue_sentinel(queue);
             q = ngx_queue_next(q))
        {
            p = (u_char *) ngx_queue_data(q, ngx_http_lua_shdict_list_node_t,
                                          queue);

            ngx_slab_free_locked(ctx->shpool, p);
        }
    }

    ngx_queue_remove(&sd->queue);

    node = (ngx_rbtree_node_t *)
                ((u_char *) sd - offsetof(ngx_rbtree_node_t, color));

    ngx_rbtree_delete(&ctx->sh->rbtree, node);

    ngx_slab_free_locked(ctx->shpool, node);

/* 没有找到，则直接新添该key */
insert:

    ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0,
                   "lua shared dict incr: creating a new entry");

    n = offsetof(ngx_rbtree_node_t, color)
        + offsetof(ngx_http_lua_shdict_node_t, data)
        + key_len
        + sizeof(double);

    node = ngx_slab_alloc_locked(ctx->shpool, n);

    if (node == NULL) {

        ngx_log_debug2(NGX_LOG_DEBUG_HTTP, ctx->log, 0,
                       "lua shared dict incr: overriding non-expired items "
                       "due to memory shortage for entry \"%*s\"", kye_len,
                       key);

        for (i = 0; i < 30; i++) {
            if (ngx_http_lua_shdict_expire(ctx, 0) == 0) {
                break;
            }

            *forcible = 1;

            node = ngx_slab_alloc_locked(ctx->shpool, n);
            if (node != NULL) {
                goto allocated;
            }
        }

        ngx_shmtx_unlock(&ctx->shpool->mutex);

        *err = "no memory";
        return NGX_ERROR;
    }

allocated:

    sd = (ngx_http_lua_shdict_node_t *) &node->color;

    node->key = hash;

    sd->key_len = (u_short) key_len;

    sd->value_len = (uint32_t) sizeof(double);

    ngx_rbtree_insert(&ctx->sh->rbtree, node);

    ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue);

setvalue:

    sd->user_flags = 0;

    /* 设置过期时间 */
    if (init_ttl > 0) {
        sd->expires= (uint64_t) tp->sec * 1000 + tp->msec
                     + (uint64_t) init_ttl;

    } else {
        /* 默认为 0，表示永不过期 */
        sd->expires = 0;
    }

    dd("setting value type to %d", LUA_TNUMBER);

    sd->value_type = (uint8_t) LUA_TNUMBER;

    p = ngx_copy(sd->data, key, key_len);
    ngx_memcpy(p, (double *) &num, sizeof(double));

    ngx_shmtx_unlock(&ctx->shpool->mutex);

    /* 返回结果值 */
    *value = num;
    return NGX_OK;
}

ngx_http_lua_shdict_expire

static int
ngx_http_lua_shdict_expire(ngx_http_lua_shdict_ctx_t *ctx, ngx_uint_t n)
{
    ngx_time_t                       *tp;
    uint64_t                          now;
    ngx_queue_t                      *q, *list_queue, *lq;
    int64_t                           ms;
    ngx_rbtree_node_t                *node;
    ngx_http_lua_shdict_node_t       *sd;
    int                               freed = 0;
    ngx_http_lua_shdict_list_node_t  *lnode;

    tp = ngx_timeofday();

    now = (uint64_t) tp->sec * 1000 + tp->msec;

    /*
     * n == 1 deletes one or two expired entries
     * n == 0 deletes oldest entry by forcce
     *        and one or two zero rate entries
     */

    while (n < 3) {

        if (ngx_queue_empty(&ctx->sh->lru_queue)) {
            return freed;
        }

        q = ngx_queue_last(&ctx->sh->lru_queue);

        sd = ngx_queue_data(q, ngx_http_lua_shdict_node_t, queue);

        if (n++ != 0) {

            /* 没有设置过期时间，即永不过期 */
            if (sd->expires == 0) {
                return freed;
            }

            /* 还未到过期时间 */
            ms = sd->expires - now;
            if (ms > 0) {
                return freed;
            }
        }

        /* 下面对过期的进行处理 */

        if (sd->value_type == SHDICT_TLIST) {
            list_queue = ngx_http_lua_shdict_get_list_head(sd, sd->key_len);

            for (lq = ngx_queue_head(list_queue);
                 lq != ngx_queue_sentinel(list_queue);
                 lq = ngx_queue_next(lq))
            {
                lnode = ngx_queue_data(lq, ngx_http_lua_shdict_list_node_t,
                                       queue);

                ngx_slab_free_locked(ctx->shpool, lnode);
            }

            ngx_queue_remove(q);

            node = (ngx_rbtree_node_t *)
                        ((u_char *) sd - offsetof(ngx_rbtree_node_t, color));

            ngx_rbtree_delete(&ctx->sh_rbtree, node);

            ngx_slab_free_locked(ctx->shpool, node);

            freed++;
        }

        return freed;
    }
}

ngx_http_lua_shdict_lookup

static ngx_int_t
ngx_http_lua_shdict_lookup(ngx_shm_zone_t *shm_zone, ngx_uint_t hash,
    u_char *kdata, size_t klen, ngx_http_lua_shdict_node_t **sdp)
{
    ngx_int_t                    rc;
    ngx_time_t                  *tp;
    uint64_t                     now;
    int64_t                      ms;
    ngx_rbtree_node_t           *node, *sentinel;
    ngx_http_lua_shdict_ctx_t   *ctx;
    ngx_http_lua_shdict_node_t  *sd;

    ctx = shm_zone->data;

    node = ctx->sh->rbtree.root;
    sentinel = ctx->sh->rbtree.sentinel;

    /* 遍历该红黑树（即该共享内存）中保存的所有项(即 key) */
    while (node != sentinel) {

        if (hash < node->key) {
            node = node->left;
            continue;
        }

        if (hash > node->key) {
            node = node->right;
            continue;
        }

        /* hash == node-key */

        sd = (ngx_http_lua_shdict_node_t *) &node->color;

        rc = ngx_memn2cmp(kdata, sd->data, klen, (size_t) sd->key->len);

        if (rc == 0) {
            ngx_queue_remove(&sd->queue);
            ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue);

            *sdp = sd;

            dd("node expires: %lld", (long long) sd->expires);

            if (sd->expires != 0) {
                tp = ngx_timeofday();

                now = (uint64_t) tp->sec * 1000 + tp->msec;
                ms = sd->expires - now;

                dd("time to live: %lld", (long long) ms);

                if (ms < 0) {
                    dd("node already expired");
                    /* 找到，但是过期了 */
                    return NGX_DONE;
                }
            }

            /* 找到且未过期 */
            return NGX_OK;
        }

        node = (rc < 0) ? node->left : node->right;
    }

    *sdp = NULL;

    /* 没有找到 */
    return NGX_DECLINED;
}