stop_token.go

package engine

import (

    "bufio"

    "log"

    "os"

)

type StopTokens struct {

    stopTokens map[string]bool

}

// 从stopTokenFile中读入停用词，一个词一行

// 文档索引建立时会跳过这些停用词

func (st *StopTokens) Init(stopTokenFile string) {

    st.stopTokens = make(map[string]bool)

    if stopTokenFile == "" {

        return

    }

    file, err := os.Open(stopTokenFile)

    if err != nil {

        log.Fatal(err)

    }

    defer file.Close()

    scanner := bufio.NewScanner(file)

    for scanner.Scan() {

        text := scanner.Text()

        if text != "" {

            st.stopTokens[text] = true

        }

    }

}

func (st *StopTokens) IsStopToken(token string) bool {

    _, found := st.stopTokens[token]

    return found

}