File size: 1,188 Bytes
1cd137c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | import duckdb
from .base import RetailerAdapter
PRODUCTS_SOURCE = (
"hf://datasets/saraNour/compliments-brand/"
"raw_data/products.parquet"
)
NUTRITION_SOURCE = (
"hf://datasets/saraNour/compliments-brand/"
"raw_data/nutrition.parquet"
)
class ComplimentsAdapter(RetailerAdapter):
@property
def retailer(self) -> str:
return "sobeys"
def _read_parquet(self, source: str):
connection = duckdb.connect()
connection.execute(
"CREATE SECRET hf_token "
"(TYPE huggingface, PROVIDER credential_chain)"
)
reader = connection.sql(
f"SELECT * FROM '{source}'"
).to_arrow_reader()
return reader, connection
def products(self):
reader, connection = self._read_parquet(PRODUCTS_SOURCE)
try:
for batch in reader:
yield from batch.to_pylist()
finally:
connection.close()
def nutrition(self):
reader, connection = self._read_parquet(NUTRITION_SOURCE)
try:
for batch in reader:
yield from batch.to_pylist()
finally:
connection.close() |