package engine

import (
    "github.com/huichen/wukong/types"
    "sync/atomic"
)

type indexerAddDocumentRequest struct {
    document    *types.DocumentIndex
    forceUpdate bool
}

type indexerLookupRequest struct {
    countDocsOnly       bool
    tokens              []string
    labels              []string
    docIds              map[uint64]bool
    options             types.RankOptions
    rankerReturnChannel chan rankerReturnRequest
    orderless           bool
}

type indexerRemoveDocRequest struct {
    docId       uint64
    forceUpdate bool
}

func (engine *Engine) indexerAddDocumentWorker(shard int) {
    for {
        request := <-engine.indexerAddDocChannels[shard]
        engine.indexers[shard].AddDocumentToCache(request.document, request.forceUpdate)
        if request.document != nil {
            atomic.AddUint64(&engine.numTokenIndexAdded,
                uint64(len(request.document.Keywords)))
            atomic.AddUint64(&engine.numDocumentsIndexed, 1)
        }
        if request.forceUpdate {
            atomic.AddUint64(&engine.numDocumentsForceUpdated, 1)
        }
    }
}

func (engine *Engine) indexerRemoveDocWorker(shard int) {
    for {
        request := <-engine.indexerRemoveDocChannels[shard]
        engine.indexers[shard].RemoveDocumentToCache(request.docId, request.forceUpdate)
        if request.docId != 0 {
            atomic.AddUint64(&engine.numDocumentsRemoved, 1)
        }
        if request.forceUpdate {
            atomic.AddUint64(&engine.numDocumentsForceUpdated, 1)
        }
    }
}

func (engine *Engine) indexerLookupWorker(shard int) {
    for {
        request := <-engine.indexerLookupChannels[shard]

        var docs []types.IndexedDocument
        var numDocs int
        if request.docIds == nil {
            docs, numDocs = engine.indexers[shard].Lookup(request.tokens, request.labels, nil, request.countDocsOnly)
        } else {
            docs, numDocs = engine.indexers[shard].Lookup(request.tokens, request.labels, request.docIds, request.countDocsOnly)
        }

        if request.countDocsOnly {
            request.rankerReturnChannel <- rankerReturnRequest{numDocs: numDocs}
            continue
        }

        if len(docs) == 0 {
            request.rankerReturnChannel <- rankerReturnRequest{}
            continue
        }

        if request.orderless {
            var outputDocs []types.ScoredDocument
            for _, d := range docs {
                outputDocs = append(outputDocs, types.ScoredDocument{
                    DocId: d.DocId,
                    TokenSnippetLocations: d.TokenSnippetLocations,
                    TokenLocations:        d.TokenLocations})
            }
            request.rankerReturnChannel <- rankerReturnRequest{
                docs:    outputDocs,
                numDocs: len(outputDocs),
            }
            continue
        }

        rankerRequest := rankerRankRequest{
            countDocsOnly:       request.countDocsOnly,
            docs:                docs,
            options:             request.options,
            rankerReturnChannel: request.rankerReturnChannel,
        }
        engine.rankerRankChannels[shard] <- rankerRequest
    }
}

indexer_worker.go的更多相关文章

随机推荐

  1. canvas元素

    一.canvas元素的基础知识 canvas元素是html5中新增的一个重要的元素,专门用来绘制图形.在页面上放置了一个canvas元素,就相当于在页面上放置了一块"画布",可以在 ...

  2. RunTime运行时在iOS中的应用之UITextField占位符placeholder

    RunTime运行时机制 runtime是一套比较底层的纯C语言API, 属于1个C语言库, 包含了很多底层的C语言API. 在我们平时编写的Objective-C代码中, 程序运行过程时, 其实最终 ...

  3. centos下 redmind2.6安装

    1.下载安装redmind有关软件 cd /tmp wget http://cache.ruby-lang.org/pub/ruby/2.1/ruby-2.1.5.tar.gz wget http:/ ...

  4. 最优Django环境配置

    2 最优Django环境配置 本章描述了我们认为对于中等和高级Django使用者来说最优的本地环境配置 2.1 统一使用相同的数据库引擎 一个常见的开发者错误是在本地开发环境中使用SQLite3,而在 ...

  5. 下载Github上某个项目的子文件夹和单个文件

    preface Github下的项目可能很大,里面有很多的子文件夹,我们可能只需要使用某个子目录下的资源,可以不用下载完整的repo就能使用. 例如,我想下载这个repo中的字典文件:https:// ...

  6. DDGScreenShot —图片加各种滤镜高逼格操作

    写在前面 图片加各种滤镜操作,当然苹果给开发者提供了相关的api和封装, 大部分开发者感觉这是这是晦涩难懂的,接下来就让我们来了解一下, 其实也没有那么深不可测. 代码如下(每一步已经解释的很详细) ...

  7. java线程之创建线程类

    1.extends Thread方法 class Person extends Thread { int sum1 = 50; // 含参构造器 public Person(String name) ...

  8. java8完全解读一

    java8完全解读 java8完全解读前言java8的一些新特性1.为什么要用java8?1.1首先想到的逻辑应该是如下1.2使用策略模式来解这个问题1.3使用策略模式和内部类来解决问题1.4使用策略 ...

  9. spring中配置quartz调用两次及项目日志log4j不能每天生成日志解决方法

    在quartz中配置了一个方法运行时会连续调用两次,是因为加载两次,只需在tomcat的server.xml中修改配置 <Host name="www.xx.cn" appB ...

  10. Java Collections类

    集合类是什么? List列表.Set集合.Map映射.工具类(Iterator迭代器.Enumeration枚举类.Arrays和Collections). Set 接口继承 Collection,但 ...