golang之操作kafka

安装第三方包:

go get github.com/IBM/sarama

生产者实例:

package main

import (
   "fmt"
   "github.com/IBM/sarama"
)

func main() {

   //1.生产者配置
   config := sarama.NewConfig()
   config.Producer.RequiredAcks = sarama.WaitForAll          //ACK,发送完数据需要leader和follow都确认
   config.Producer.Partitioner = sarama.NewRandomPartitioner //分区,新选出一个分区
   config.Producer.Return.Successes = true                   //确认,成功交付的消息将在success channel返回

   //2.连接Kafka

   client, err := sarama.NewSyncProducer([]string{"127.0.0.1:9092"}, config)
   if err != nil {
      fmt.Println("Producer error", err)
      return
   }

   defer client.Close()

   //3.封装消息

   msg := &sarama.ProducerMessage{}
   msg.Topic = "log"
   msg.Value = sarama.StringEncoder("this is test log")

   //4.发送消息
   pid, offset, err := client.SendMessage(msg)

   if err != nil {
      fmt.Println("send faild", err)
   }
   fmt.Printf("pid:%v offset:%v\n", pid, offset)
}

消费者:

package main

import (
    "fmt"

   // "github.com/Shopify/sarama"  // 该包已经转移到了IBM/sarama

　　"github.com/IBM/sarama"

)

// kafka consumer

func main() {
    consumer, err := sarama.NewConsumer([]string{"127.0.0.1:9092"}, nil)
    if err != nil {
        fmt.Printf("fail to start consumer, err:%v\n", err)
        return
    }
    partitionList, err := consumer.Partitions("web_log") // 根据topic取到所有的分区
    if err != nil {
        fmt.Printf("fail to get list of partition:err%v\n", err)
        return
    }
    fmt.Println(partitionList)
    for partition := range partitionList { // 遍历所有的分区
        // 针对每个分区创建一个对应的分区消费者
        pc, err := consumer.ConsumePartition("web_log", int32(partition), sarama.OffsetNewest)
        if err != nil {
            fmt.Printf("failed to start consumer for partition %d,err:%v\n", partition, err)
            return
        }
        defer pc.AsyncClose()
        // 异步从每个分区消费信息
        go func(sarama.PartitionConsumer) {
            for msg := range pc.Messages() {
                fmt.Printf("Partition:%d Offset:%d Key:%v Value:%v", msg.Partition, msg.Offset, msg.Key, msg.Value)
            }
        }(pc)
    }
}

完整文件内容:

package main

import (
    "context"
    "fmt"
    "github.com/IBM/sarama"
    "log"
    "math"
    "os"
    "os/signal"
    "sync"
    "syscall"
)

// 更多参考:https://kpretty.tech/archives/gokafkaclient

var addrs = []string{"172.29.97.140:9092"}
var topic = "log"

func producer() {
    //    生产者配置
    config := sarama.NewConfig()
    config.Producer.RequiredAcks = sarama.WaitForAll          // ACK
    config.Producer.Partitioner = sarama.NewRandomPartitioner // 分区
    // 异步回调(两个channel, 分别是成功和错误)
    config.Producer.Return.Successes = true // 确认
    config.Producer.Return.Errors = true

    sarama.Logger = log.New(os.Stdout, "[Sarama]", log.LstdFlags)

    // 连接kafka
    // 同步
    client, err := sarama.NewSyncProducer(addrs, config)
    // 异步
    //client, err := sarama.NewAsyncProducer(addrs, config)
    if err != nil {
        fmt.Println("producer error", err)
        return
    }

    defer func() {
        _ = client.Close()
    }()

    // 封装消息
    msg := &sarama.ProducerMessage{
        Topic: topic,
        Value: sarama.StringEncoder("this is test log"),
    }

    pid, offset, err := client.SendMessage(msg)
    if err != nil {
        fmt.Println("send failed", err)
        return
    }

    fmt.Printf("pid:%v offset:%v \n", pid, offset)
}

func consumer() {
    consumer, err := sarama.NewConsumer(addrs, nil)
    if err != nil {
        fmt.Printf("fail to start consumer, err:%v \n", err)
        return
    }
    partitionList, err := consumer.Partitions(topic) // 通过topic获取所有分区
    if err != nil {
        fmt.Printf("fail to get partition list, err:%v\n", err)
        return
    }

    fmt.Println(partitionList)
    for partition := range partitionList { // 遍历所有分区
        pc, err := consumer.ConsumePartition(topic, int32(partition), sarama.OffsetNewest)
        if err != nil {
            fmt.Printf("failed to start consumer for partition %d, err:%v\n", partition, err)
            return
        }
        defer pc.AsyncClose()
        go func(sarama.PartitionConsumer) {
            for msg := range pc.Messages() {
                // 当设置了key的时候,不为空
                fmt.Printf("Partition:%d Offset:%d Key:%s Value:%s\n", msg.Partition, msg.Offset, string(msg.Key), string(msg.Value))
            }
        }(pc)
    }
    //time.Sleep(5 * time.Second)
    select {}
}

func groupConsumer() {
    groupId := "sarama-consumer"
    config := sarama.NewConfig()
    // 关闭自动提交 和 初始化策略(oldest|newest)
    config.Consumer.Offsets.AutoCommit.Enable = false
    config.Consumer.Offsets.Initial = sarama.OffsetOldest
    sarama.NewConsumerGroup(addrs, groupId, config)
}

func main() {
    // 生产者
    //producer()
    // 消费者(只能连续读取,中断期间会丢失数据)
    //consumer()

    // 消费者
    //groupConsumer()
    SimpleConsumer()
}

var groupID = "sarama-consumer"
var asyncOffset chan struct{}
var wg sync.WaitGroup

const defaultOffsetChannelSize = math.MaxInt

func SimpleConsumer() {
    brokers := addrs
    // 消费者配置
    config := sarama.NewConfig()
    // 关闭自动提交
    config.Consumer.Offsets.AutoCommit.Enable = false
    config.Consumer.Offsets.Initial = sarama.OffsetOldest
    // 开启日志
    logger := log.New(os.Stdout, "[Sarama] ", log.LstdFlags)
    sarama.Logger = logger
    consumer, err := sarama.NewConsumerGroup(brokers, groupID, config)
    if err != nil {
        panic(err)
    }
    defer func() { _ = consumer.Close() }()
    // 搞一个上下文用于终止消费者
    ctx, cancelFunc := context.WithCancel(context.Background())
    // 监听终止信号
    go func() {
        logger.Println("monitor signal")
        quit := make(chan os.Signal, 1)
        signal.Notify(quit, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
        <-quit
        logger.Println("stop consumer")
        cancelFunc()
    }()
    // 消费数据
    err = consumer.Consume(ctx, []string{topic}, &Consumer{})
    if err != nil {
        panic(err)
    }
    // 等待所有偏移量都提交完毕再退出
    logger.Println("当前存在未提交的偏移量")
    wg.Wait()
}

type Consumer struct{}

func (c *Consumer) Setup(session sarama.ConsumerGroupSession) error {
    // 初始化异步提交的channel
    asyncOffset = make(chan struct{}, defaultOffsetChannelSize)
    wg.Add(1)
    // 异步提交偏移量
    go func() {
        for range asyncOffset {
            session.Commit()
        }
        wg.Done()
    }()
    return nil
}

func (c *Consumer) Cleanup(_ sarama.ConsumerGroupSession) error {
    // 关闭通道
    close(asyncOffset)
    return nil
}

func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error {
EXIT:
    for {
        select {
        case message := <-claim.Messages():
            log.Printf("Message claimed: key= %s, value = %s, timestamp = %v, topic = %s", string(message.Key), string(message.Value), message.Timestamp, message.Topic)
            // 标记消息，并不是提交偏移量
            session.MarkMessage(message, "")
            // 异步提交
            asyncOffset <- struct{}{}
        case <-session.Context().Done():
            log.Println("cancel consumer")
            break EXIT
        }
    }
    return nil
}

更深入使用:

• https://kpretty.tech/archives/gokafkaclient