Buckets:

download
raw
2.92 kB
import os
from datetime import datetime, timedelta, timezone
import csp
from csp_gateway import (
AddChannelsToGraphOutput,
Gateway,
GatewayChannels,
GatewayModule,
GatewayStruct,
KafkaConfiguration,
ReadWriteKafka,
)
def create_kafka_config():
group_id = "foo"
broker = "kafka-broker:9093"
auth = True
user_lower = os.environ.get("USER").lower()
user_principal = os.environ.get("USER_PRINCIPAL_NAME")
# NOTE: This file is used for verifying the Kafka broker
ssl_ca_location = "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"
# NOTE: This will have to be set up by the user.
kerberos_keytab = f"/home/{user_lower}/.keytab/{user_principal.split('@')[0]}.keytab.rc4-hmac"
return KafkaConfiguration(
group_id=group_id,
broker=broker,
sasl_kerberos_keytab=kerberos_keytab,
sasl_kerberos_principal=user_principal,
ssl_ca_location=ssl_ca_location,
auth=auth,
)
class MyStruct(GatewayStruct):
foo: str
class GWC(GatewayChannels):
my_proclamation: csp.ts[MyStruct] = None
class MyCallToTheWorld(GatewayModule):
my_data: csp.ts[MyStruct]
def connect(self, channels: GWC):
channels.set_channel(GWC.my_proclamation, self.my_data)
if __name__ == "__main__":
# In this example, the read and write functionalities for the
# ReadWriteKafka GatewayModule are shown. This allows users
# to use Kafka to send and receive Gateway channel ticks on
# Gateway Strcuts.
# When ran, this file will print a struct called MyStruct,
# with foo="HELLO WORLD!!" many times. One ReadWriteKafka instance
# writes the struct to Kafka from the channel, and the other reads it
# from Kafka and populates the channel.
encoding_with_engine_timestamps = True
set_module = MyCallToTheWorld(my_data=csp.const(MyStruct(foo="HELLO WORLD!!")))
channels = [GWC.my_proclamation]
kafka_write_module = ReadWriteKafka(
config=create_kafka_config(),
publish_channel_to_topic_and_key={GWC.my_proclamation: {"kafka_test": "kafka_read_write_example"}},
encoding_with_engine_timestamps=encoding_with_engine_timestamps,
)
kafka_read_module = ReadWriteKafka(
config=create_kafka_config(),
subscribe_channel_to_topic_and_key={GWC.my_proclamation: {"kafka_test": "kafka_read_write_example"}},
encoding_with_engine_timestamps=encoding_with_engine_timestamps,
)
gateway = Gateway(
modules=[
set_module,
kafka_read_module,
kafka_write_module,
AddChannelsToGraphOutput(),
],
channels=GWC(),
channels_model=GWC,
)
out = csp.run(
gateway.graph,
starttime=datetime.now(timezone.utc),
endtime=timedelta(seconds=2),
realtime=True,
)
print(f"{out[GWC.my_proclamation] = }")

Xet Storage Details

Size:
2.92 kB
·
Xet hash:
1dd25152f9a3a1317fe54544cdbdf3de0b5e8c22544cb8c9a96470e0364e6ad6

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.