LevelDB源码分析-Bloom Filter

Bloom Filter

bloom filter是leveldb中用来在一个block中检测key是否存在的工具，主要在BloomFilterPolicy类中实现：

class BloomFilterPolicy : public FilterPolicy
{
  private:
    size_t bits_per_key_;
    size_t k_;

  public:
    explicit BloomFilterPolicy(int bits_per_key)
        : bits_per_key_(bits_per_key)
    {
        // ...
    }

    // ...

    virtual void CreateFilter(const Slice *keys, int n, std::string *dst) const
    {
        // ...
    }

    virtual bool KeyMayMatch(const Slice &key, const Slice &bloom_filter) const
    {
        // ...
    }
};

其中，bits_per_key_表示m/n，即bloom filter中包含的位数组长度与sstable中key的数量的比值。k_为hash函数的个数。

接下来看BloomFilterPolicy的构造函数，因为bloom filter的FPR（false positive rate）为(1-e^(-kn/m))^k，当k为log(2)(m/n)时，FPR最小，因此这里用bits_per_key_乘以0.69来求得k_的值，当然最后还是要将k限定在1-30的范围内：

        // We intentionally round down to reduce probing cost a little bit
        k_ = static_cast<size_t>(bits_per_key * 0.69); // 0.69 =~ ln(2)
        if (k_ < 1)
            k_ = 1;
        if (k_ > 30)
            k_ = 30;

通过调用CreateFilter函数创建一个新的bloom filter。这个函数首先计算出需要多少字节来存储这个新的bloom filter：

        // Compute bloom filter size (in both bits and bytes)
        size_t bits = n * bits_per_key_;

        // For small n, we can see a very high false positive rate.  Fix it
        // by enforcing a minimum bloom filter length.
        if (bits < 64)
            bits = 64;

        size_t bytes = (bits + 7) / 8;
        bits = bytes * 8;

接下来函数将存储bloom filter的字符串dst进行扩展，使其能够存储刚刚计算出的新增的bytes个字节，dst中原来存储的是之前得到的bloom filter，同时将当前正在构建的这个bloom filter使用的hash函数的个数存入dst的末尾：

        const size_t init_size = dst->size();
        dst->resize(init_size + bytes, 0);
        dst->push_back(static_cast<char>(k_)); // Remember # of probes in filter

最后依次取出key，通过double hash来计算出key的k_个hash后的结果，并将值存入dst指向的字符串中：

        char *array = &(*dst)[init_size];
        for (int i = 0; i < n; i++)
        {
            // Use double-hashing to generate a sequence of hash values.
            // See analysis in [Kirsch,Mitzenmacher 2006].
            uint32_t h = BloomHash(keys[i]);
            const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits
            for (size_t j = 0; j < k_; j++)
            {
                const uint32_t bitpos = h % bits;
                array[bitpos / 8] |= (1 << (bitpos % 8));
                h += delta;
            }
        }

这里double hash的算法为：

Hi = h1(x) + ih2(x)
h2(x) = (h1(x) >> 17) | (h1(x) << 15)

通过调用KeyMayMatch函数可以判断一个key是不是在bloom filter表示的key集合中。这个函数首先得到给定bloom filter的一些相关信息，比如bit数、hash函数的个数（生成一个bloom filter时，hash函数的个数被编码在bloom filter的最后一个字节）等，如果hash函数的个数大于30，则直接返回true：

        const size_t len = bloom_filter.size();
        if (len < 2)
            return false;

        const char *array = bloom_filter.data();
        const size_t bits = (len - 1) * 8;

        // Use the encoded k so that we can read filters generated by
        // bloom filters created using different parameters.
        const size_t k = array[len - 1];
        if (k > 30)
        {
            // Reserved for potentially new encodings for short bloom filters.
            // Consider it a match.
            return true;
        }

然后通过依次计算key的k个hash函数的值，判断这个key是否在bloom filter表示的key集合中：

        uint32_t h = BloomHash(key);
        const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits
        for (size_t j = 0; j < k; j++)
        {
            const uint32_t bitpos = h % bits;
            if ((array[bitpos / 8] & (1 << (bitpos % 8))) == 0)
                return false;
            h += delta;
        }
        return true;

LevelDB还封装了FilterBlockBuilder类来构建在sstable中存储bloom filter的block：

// A FilterBlockBuilder is used to construct all of the filters for a
// particular Table.  It generates a single string which is stored as
// a special block in the Table.
//
// The sequence of calls to FilterBlockBuilder must match the regexp:
//      (StartBlock AddKey*)* Finish
class FilterBlockBuilder
{
  public:
    explicit FilterBlockBuilder(const FilterPolicy *);

    void StartBlock(uint64_t block_offset);
    void AddKey(const Slice &key);
    Slice Finish();

  private:
    void GenerateFilter();

    const FilterPolicy *policy_;
    std::string keys_;            // Flattened key contents
    std::vector<size_t> start_;   // Starting index in keys_ of each key
    std::string result_;          // Filter data computed so far
    std::vector<Slice> tmp_keys_; // policy_->CreateFilter() argument
    std::vector<uint32_t> filter_offsets_;

    // ...
};

keys_用来存储需要添加进bloom filter的key，每个key都简单的连接在keys_的后面，start_存储每个key在keys_中的偏移量，result_为当前已经计算得到的bloom filter的值，tmp_keys_用于在调用CreateFilter函数时传递参数，filter_offsets_用于存储构建的每一个bloom filter在当前filter block中的偏移量。

StartBlock函数在开始构建bloom filter前调用。因为bloom filter的构建策略是每2k个字节就构建一个bloom filter，所以StartBlock函数会首先判断当前sstable中尚未构建bloom filter的数据是否已经达到了2k，如果达到，就调用GenerateFilter()函数为sstable中尚未构建bloom filter的数据构建新的bloom filter。

void FilterBlockBuilder::StartBlock(uint64_t block_offset)
{
    uint64_t filter_index = (block_offset / kFilterBase);
    assert(filter_index >= filter_offsets_.size());
    while (filter_index > filter_offsets_.size())
    {
        GenerateFilter();
    }
}

其中GenerateFilter函数为：

void FilterBlockBuilder::GenerateFilter()

函数首先计算出需要添加进bloom filter中的key的数量，并将key存入tmp_keys_以便传参：

    const size_t num_keys = start_.size();
    if (num_keys == 0)
    {
        // Fast path if there are no keys for this filter
        filter_offsets_.push_back(result_.size());
        return;
    }

    // Make list of keys from flattened key structure
    start_.push_back(keys_.size()); // Simplify length computation
    tmp_keys_.resize(num_keys);
    for (size_t i = 0; i < num_keys; i++)
    {
        const char *base = keys_.data() + start_[i];
        size_t length = start_[i + 1] - start_[i];
        tmp_keys_[i] = Slice(base, length);
    }

接下来将新的bloom filter在filter block中的偏移量存入filter_offsets_，然后调用BloomFilterPolicy类的CreateFilter函数构建新的bloom filter：

    // Generate filter for current set of keys and append to result_.
    filter_offsets_.push_back(result_.size());
    policy_->CreateFilter(&tmp_keys_[0], static_cast<int>(num_keys), &result_);

AddKey函数比较简单，只是将key加在keys_的后面，然后在start_中添加这个key的偏移量：

void FilterBlockBuilder::AddKey(const Slice &key)
{
    Slice k = key;
    start_.push_back(keys_.size());
    keys_.append(k.data(), k.size());
}

Finish函数完成一个filter block的构建

Slice FilterBlockBuilder::Finish()

首先，当FilterBlockBuilder中还有key尚未构建bloom filter时，调用GenerateFilter函数将其构建为一个bloom filter：

    if (!start_.empty())
    {
        GenerateFilter();
    }

然后将每个bloom filter在result_中的偏移量编码后加入result_：

    // Append array of per-filter offsets
    const uint32_t array_offset = result_.size();
    for (size_t i = 0; i < filter_offsets_.size(); i++)
    {
        PutFixed32(&result_, filter_offsets_[i]);
    }

最后将描述bloom filter在result_中的偏移量的数组在result_中的偏移量编码后存入result_，这里还要存储kFilterBaseLg（sstable中每1 << kFilterBaseLg个字节构建一个bloom filter）：

    PutFixed32(&result_, array_offset);
    result_.push_back(kFilterBaseLg); // Save encoding parameter in result
    return Slice(result_);

LevelDB同时还封装了一个FilterBlockReader类用于filter block的读取：

class FilterBlockReader
{
  public:
    // REQUIRES: "contents" and *policy must stay live while *this is live.
    FilterBlockReader(const FilterPolicy *policy, const Slice &contents);
    bool KeyMayMatch(uint64_t block_offset, const Slice &key);

  private:
    const FilterPolicy *policy_;
    const char *data_;   // Pointer to filter data (at block-start)
    const char *offset_; // Pointer to beginning of offset array (at block-end)
    size_t num_;         // Number of entries in offset array
    size_t base_lg_;     // Encoding parameter (see kFilterBaseLg in .cc file)
};

其中data_指向bloom filter的起始位置，offset_指向描述bloom filter在filter block中的偏移量的数组的起始位置，num_表示offset_数组的元素个数，base_lg_即为之前提到的kFilterBaseLg。

这里调用KeyMayMatch函数来实现对给定key是否在给定block中的检查：

bool FilterBlockReader::KeyMayMatch(uint64_t block_offset, const Slice &key)
{

}

函数首先根据给定block的block_offset找到该block对应的是filter block中的第几个bloom filter：

    uint64_t index = block_offset >> base_lg_;

然后得到对应的bloom filter的起始位置和结束位置，并调用BloomFilterPolicy类的KeyMayMatch函数进行检查：

    if (index < num_)
    {
        uint32_t start = DecodeFixed32(offset_ + index * 4);
        uint32_t limit = DecodeFixed32(offset_ + index * 4 + 4);
        if (start <= limit && limit <= static_cast<size_t>(offset_ - data_))
        {
            Slice filter = Slice(data_ + start, limit - start);
            return policy_->KeyMayMatch(key, filter);
        }
        else if (start == limit)
        {
            // Empty filters do not match any keys
            return false;
        }
    }
    return true; // Errors are treated as potential matches

238 Love u