1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
|
#!/usr/bin/env python
import threading, logging, time, collections
from kafka.client import KafkaClient
from kafka.consumer import SimpleConsumer
from kafka.producer import SimpleProducer
msg_size = 524288
class Producer(threading.Thread):
daemon = True
big_msg = "1" * msg_size
def run(self):
client = KafkaClient("localhost:9092")
producer = SimpleProducer(client)
self.sent = 0
while True:
producer.send_messages('my-topic', self.big_msg)
self.sent += 1
class Consumer(threading.Thread):
daemon = True
def run(self):
client = KafkaClient("localhost:9092")
consumer = SimpleConsumer(client, "test-group", "my-topic",
max_buffer_size = None,
)
self.valid = 0
self.invalid = 0
for message in consumer:
if len(message.message.value) == msg_size:
self.valid += 1
else:
self.invalid += 1
def main():
threads = [
Producer(),
Consumer()
]
for t in threads:
t.start()
time.sleep(10)
print 'Messages sent: %d' % threads[0].sent
print 'Messages recvd: %d' % threads[1].valid
print 'Messages invalid: %d' % threads[1].invalid
if __name__ == "__main__":
logging.basicConfig(
format='%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s',
level=logging.DEBUG
)
main()
|
