py kafka
# https://github.com/confluentinc/confluent-kafka-python/blob/master/examples/consumer.py
#生产者
import json
from kafka import KafkaProducer
from confluent_kafka import Producer
msg_dict = {
"interval": 10,
"producer": {
"name": "producer 1",
"host": "10.10.10.1",
"user": "root",
"password": "root"
},
"cpu": "33.5%",
"mem": "77%",
"msg": "Hello kafka",
"data": "测试",
}
def test():
producer = KafkaProducer(bootstrap_servers='127.0.0.1:9092') # 连接kafka
msg = "Hello World".encode('utf-8') # 发送内容,必须是bytes类型
# msg = {"data": 1}
# producer.send('mytopic', json.dumps(msg_dict).encode("utf-8")) # 发送的topic为test
producer.send('mytopic', json.dumps(msg_dict)) # 发送的topic为test
producer.close()
# p = Producer({'bootstrap.servers': 'mybroker1,mybroker2'})
p = Producer({'bootstrap.servers': '127.0.0.1:9092,mybroker2'})
def delivery_report(err, msg):
""" Called once for each message produced to indicate delivery result.
Triggered by poll() or flush(). """
if err is not None:
print('Message delivery failed: {}'.format(err))
else:
print('Message delivered to {} [{}]'.format(msg.topic(), msg.partition()))
# some_data_source = [str(i) + "*" for i in range(100)]
some_data_source = [json.dumps(msg_dict) for i in range(5)]
for data in some_data_source:
# Trigger any available delivery report callbacks from previous produce() calls
p.poll(0.1)
# 异步通信 triggered(触发) delivered(递送)
# Asynchronously produce a message, the delivery report callback
# will be triggered from poll() above, or flush() below, when the message has
# been successfully delivered or failed permanently.
p.produce('mytopic', data.encode('utf-8'), callback=delivery_report)
# Wait for any outstanding messages to be delivered and delivery report
# callbacks to be triggered. 等待任何未完成的消息被传递,并触发传递报告回调。
p.flush()
if __name__ == '__main__':
# test()
pass
# 消费者
import json
from kafka import KafkaConsumer
from confluent_kafka.avro import AvroProducer, AvroConsumer
from confluent_kafka import Consumer, KafkaError
def one():
consumer = KafkaConsumer('mytopic', bootstrap_servers=['127.0.0.1:9092'])
for msg in consumer:
# print(type(msg.value))
# print(str(msg.value, encoding="utf8"))
print(msg.value.decode())
recv = "%s:%d:%d: key=%s value=%s" % (msg.topic, msg.partition, msg.offset, msg.key, msg.value.decode("utf-8"))
print("...")
print(recv)
bootstrap_servers = '127.0.0.1:9092,xxx'
def two():
client = AvroProducer({
'bootstrap.servers': bootstrap_servers,
'schema.registry.url': "",
})
value = {"test": "value"}
key = {"test": "key"}
client.produce(topic="mytopic", value=value, key=key)
client.flush()
def three():
c = Consumer({
# 'bootstrap.servers': 'mybroker',
'bootstrap.servers': bootstrap_servers,
'group.id': 'mygroup2',
'auto.offset.reset': 'earliest'
})
c.subscribe(['mytopic'])
while True:
msg = c.poll(1.0)
if msg is None:
continue
if msg.error():
print("Consumer error: {}".format(msg.error()))
continue
print(type(msg.value()))
print(msg.value())
print(json.loads(msg.value().decode('utf-8')))
print('Received message: {}'.format(msg.value().decode('utf-8')))
c.close()
if __name__ == '__main__':
# one()
# two()
three()
py kafka的更多相关文章
- Storm sql 简单测试
准备工作: 1.安装Kafka,启动,以及创建相应的topic 1.启动kafka bin/kafka-server-start.sh config/server.properties > /d ...
- [Spark] 07 - Spark Streaming Programming
Streaming programming 一.编程套路 编写Streaming程序的套路 创建DStream,也就定义了输入源. 对DStream进行一些 “转换操作” 和 "输出操作&q ...
- kafka Py客户端
1.pip install kafka-python 2.Producer.py from kafka import KafkaProducer producer = KafkaProducer(bo ...
- Ambari 不能配置 Kafka 监听host的问题
问题:Ambari下Kafka多IP监听配置 环境:Ambari 1.7.0 , Hadoop 2.2 Kafka 0.8.1.2.2.0.0 现象: Ambari 中是不能配置Kafka的host. ...
- flume+kafka+smart数据接入实施手册
1. 概述 本手册主要介绍了,一个将传统数据接入到Hadoop集群的数据接入方案和实施方法.供数据接入和集群运维人员参考. 1.1. 整体方案 Flume作为日志收集工具,监控一个文件目录或者一 ...
- 检测zookeeper和kafka是否正常
cd $(dirname $) source ~/.bash_profile count_zoo=`ps -ef | grep "config/zookeeper.properties&qu ...
- kafka学习(四)-Topic & Partition
topic中partition存储分布 Topic在逻辑上可以被认为是一个queue.每条消费都必须指定它的topic,可以简单理解为必须指明把这条消息放进哪个queue里.为了使得 Kafka的吞吐 ...
- 使用spark-streaming实时读取Kafka数据统计结果存入MySQL
在这篇文章里,我们模拟了一个场景,实时分析订单数据,统计实时收益. 场景模拟 我试图覆盖工程上最为常用的一个场景: 1)首先,向Kafka里实时的写入订单数据,JSON格式,包含订单ID-订单类型-订 ...
- 生产环境一键创建kafka集群
前段时间公司的一个kafka集群出现了故障,由于之前准备不足,当时处理的比较慌乱.如:由于kafka的集群里topic数量较多,并且每个topic的分区数量和副本数量都不是一样的,如果按部就班的一个一 ...
随机推荐
- MyBatis 与 Hibernate
MyBatis 是一个优秀的基于 Java 的持久层框架,它内部封装了 JDBC,使开发者只需关注 SQL 语句本身,而不用再花费精力去处理诸如注册驱动.创建 Connection.配置 Statem ...
- ML_Review_LDA(Ch5)
Note sth about LDA(Linear Discriminant Analysis) 这篇来说说LDA的复习,LDA在第二次作业的博客中也提及了,但是那是作业思考,所以决定再开一篇只说LD ...
- Qtcreator远程调试出现“The selected build of GDB does not support Python scripting.It cannot be used .."
版权声明:本文为博主原创文章,未经博主允许不得转载. https://blog.csdn.net/aristolto/article/details/77370853 之前使用的是Qt4.7后来换 ...
- docker - nginx+php+php-mysql(扩展)
Docker 安装 Nginx(https://www.runoob.com/docker/docker-install-nginx.html) Docker 安装 PHP(https://www.r ...
- 【Oracle/Java】以Insert ALL方式向表中插入百万条记录,耗时9分17秒
由于按一千条一插程序长期无反应,之后改为百条一插方式,运行完发现插入百万记录需要9m17s,虽然比MySQL效率差,但比单条插入已经好不少了. 对Oracle的批量插入语法不明的请参考:https:/ ...
- 两个ESP8266一个作为服务器一个作为客户端实现互相通讯
两个ESP8266一个作为服务器一个作为客户端实现互相通讯
- Leetcode: Longest Palindromic Substring && Summary: Palindrome
Given a string s, find the longest palindromic substring in s. You may assume that the maximum lengt ...
- PHP中的符号 ->、=> 和 :: 分别表示什么意思?
php新手经常碰到的问题,->.=> 和 :: 这三个家伙是什么分别都是做什么的啊!看着就很晕. 没关系,下面我们做一下详细的解释,如果你有C++,Perl基础,你会发现这些家伙和他们里面 ...
- mysql 1577、1548错误 解决方案
mysql 1577.1548错误 解决方案 1.mysql版本: 5.5.12 2.问题原因: 使用Navicat导出数据库的提示 :1577 – Cannot proceed because sy ...
- python中 将数字转化为人民币的形式
def fn(args): """ 将金额转化为人民币模式,带逗号分隔,保留小数点两位,四舍五入 :param args: :return: ""&q ...