1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
|
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2024 Confluent Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# A simple example demonstrating use of AvroDeserializer.
import argparse
from confluent_kafka.schema_registry.rules.encryption.encrypt_executor import \
FieldEncryptionExecutor
from confluent_kafka.schema_registry.rules.encryption.localkms.local_driver import \
LocalKmsDriver
from confluent_kafka.schema_registry.rules.encryption.hcvault.hcvault_driver import \
HcVaultKmsDriver
from confluent_kafka.schema_registry.rules.encryption.gcpkms.gcp_driver import \
GcpKmsDriver
from confluent_kafka.schema_registry.rules.encryption.azurekms.azure_driver import \
AzureKmsDriver
from confluent_kafka.schema_registry.rules.encryption.awskms.aws_driver import \
AwsKmsDriver
from confluent_kafka import Consumer
from confluent_kafka.serialization import SerializationContext, MessageField
from confluent_kafka.schema_registry import SchemaRegistryClient
from confluent_kafka.schema_registry.avro import AvroDeserializer
class User(object):
"""
User record
Args:
name (str): User's name
favorite_number (int): User's favorite number
favorite_color (str): User's favorite color
"""
def __init__(self, name=None, favorite_number=None, favorite_color=None):
self.name = name
self.favorite_number = favorite_number
self.favorite_color = favorite_color
def dict_to_user(obj, ctx):
"""
Converts object literal(dict) to a User instance.
Args:
obj (dict): Object literal(dict)
ctx (SerializationContext): Metadata pertaining to the serialization
operation.
"""
if obj is None:
return None
return User(name=obj['name'],
favorite_number=obj['favorite_number'],
favorite_color=obj['favorite_color'])
def main(args):
# Register the KMS drivers and the field-level encryption executor
AwsKmsDriver.register()
AzureKmsDriver.register()
GcpKmsDriver.register()
HcVaultKmsDriver.register()
LocalKmsDriver.register()
FieldEncryptionExecutor.register()
topic = args.topic
# When using Data Contract rules, a schema should not be passed to the
# AvroDeserializer. The schema is fetched from the Schema Registry.
schema_str = None
sr_conf = {'url': args.schema_registry}
schema_registry_client = SchemaRegistryClient(sr_conf)
rule_conf = None
# KMS credentials can be passed as follows
# rule_conf = {'secret.access.key': 'xxx', 'access.key.id': 'yyy'}
# Alternatively, the KMS credentials can be set via environment variables
avro_deserializer = AvroDeserializer(schema_registry_client,
schema_str,
dict_to_user,
rule_conf=rule_conf)
consumer_conf = {'bootstrap.servers': args.bootstrap_servers,
'group.id': args.group,
'auto.offset.reset': "earliest"}
consumer = Consumer(consumer_conf)
consumer.subscribe([topic])
while True:
try:
# SIGINT can't be handled when polling, limit timeout to 1 second.
msg = consumer.poll(1.0)
if msg is None:
continue
user = avro_deserializer(msg.value(), SerializationContext(msg.topic(), MessageField.VALUE))
if user is not None:
print("User record {}: name: {}\n"
"\tfavorite_number: {}\n"
"\tfavorite_color: {}\n"
.format(msg.key(), user.name,
user.favorite_number,
user.favorite_color))
except KeyboardInterrupt:
break
consumer.close()
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="AvroDeserializer example")
parser.add_argument('-b', dest="bootstrap_servers", required=True,
help="Bootstrap broker(s) (host[:port])")
parser.add_argument('-s', dest="schema_registry", required=True,
help="Schema Registry (http(s)://host[:port]")
parser.add_argument('-t', dest="topic", default="example_serde_avro",
help="Topic name")
parser.add_argument('-g', dest="group", default="example_serde_avro",
help="Consumer group")
main(parser.parse_args())
|
