Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /Point72_csp-gateway /examples /filedrop_example.py
| from datetime import datetime, timezone | |
| from typing import Any | |
| import csp | |
| from csp_gateway import ( | |
| AddChannelsToGraphOutput, | |
| FileDropType, | |
| Gateway, | |
| GatewayChannels, | |
| GatewayModule, | |
| GatewayStruct, | |
| ReadFileDrop, | |
| ReadFileDropConfiguration, | |
| ) | |
| class MySubStruct(GatewayStruct): | |
| foo: str | |
| class MyStruct(GatewayStruct): | |
| foo: str | |
| class GWC(GatewayChannels): | |
| data: csp.ts[MyStruct] = None | |
| list_data: csp.ts[list[MyStruct]] = None | |
| dict_data: dict[str, csp.ts[MyStruct]] = None | |
| def dynamic_keys(self) -> dict[str, list[Any]] | None: | |
| """Define dynamic dictionary keys by field, driven by data from the channels.""" | |
| keys = ["a", "b", "c"] | |
| return { | |
| GWC.dict_data: keys, | |
| } | |
| class PrintModule(GatewayModule): | |
| def connect(self, channels): | |
| data = channels.get_channel(GWC.data) | |
| csp.print("Data", data) | |
| list_data = channels.get_channel(GWC.list_data) | |
| csp.print("ListData", list_data) | |
| dict_data = channels.get_channel(GWC.dict_data) | |
| csp.print("DictData", dict_data) | |
| if __name__ == "__main__": | |
| # In this example, the functionality of ReadFileDrop is shown | |
| # When ran, this program will wait for: | |
| # Any new json files in ./json_fd/ | |
| # Any new parquet files in ./parquet_fd/ | |
| # | |
| # The files going into the above directories should represent MyStruct | |
| # as a json (for json_fd) and as a parquet (for parquet fd). | |
| # When a new file is found, the program will read that file and convert | |
| # it into the underlying structs. The PrintModule will print out any structs | |
| # that are created | |
| # An example json file might look as: | |
| # [ | |
| # { "foo": "s1" }, | |
| # { "foo": "s2" }, | |
| # ] | |
| # | |
| # This will result in 2 structs: MyStruct(foo="s1") & MyStruct(foo="s2") | |
| encoding_with_engine_timestamps = True | |
| channels = [GWC.data, GWC.list_data] | |
| print("Waiting for files in ./json_fd/ & ./parquet_fd/") | |
| fd_module = ReadFileDrop( | |
| configs=[ | |
| # JSON | |
| ReadFileDropConfiguration(dir_path="json_fd", channel_name="data", filedrop_type=FileDropType.JSON), | |
| ReadFileDropConfiguration(dir_path="json_fd", channel_name="list_data", filedrop_type=FileDropType.JSON), | |
| ReadFileDropConfiguration(dir_path="json_fd", channel_name="dict_data", filedrop_type=FileDropType.JSON), | |
| # PARQUET | |
| ReadFileDropConfiguration(dir_path="parquet_fd", channel_name="data", filedrop_type=FileDropType.PARQUET), | |
| ReadFileDropConfiguration(dir_path="parquet_fd", channel_name="list_data", filedrop_type=FileDropType.PARQUET), | |
| ReadFileDropConfiguration(dir_path="parquet_fd", channel_name="dict_data", filedrop_type=FileDropType.PARQUET), | |
| ], | |
| ) | |
| gateway = Gateway( | |
| modules=[ | |
| fd_module, | |
| PrintModule(), | |
| AddChannelsToGraphOutput(), | |
| ], | |
| channels=GWC(), | |
| channels_model=GWC, | |
| ) | |
| out = csp.run( | |
| gateway.graph, | |
| starttime=datetime.now(timezone.utc), | |
| realtime=True, | |
| ) | |
Xet Storage Details
- Size:
- 3.15 kB
- Xet hash:
- 7e5d8f34878573c3beeb5029decd921821d53d67a5d32e2819e0e68ebc88bbce
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.