saraNour's picture
Upload folder using huggingface_hub
1cd137c verified
Raw History Blame Contribute Delete
1.19 kB
import duckdb
from .base import RetailerAdapter
PRODUCTS_SOURCE = (
"hf://datasets/saraNour/compliments-brand/"
"raw_data/products.parquet"
)
NUTRITION_SOURCE = (
"hf://datasets/saraNour/compliments-brand/"
"raw_data/nutrition.parquet"
)
class ComplimentsAdapter(RetailerAdapter):
@property
def retailer(self) -> str:
return "sobeys"
def _read_parquet(self, source: str):
connection = duckdb.connect()
connection.execute(
"CREATE SECRET hf_token "
"(TYPE huggingface, PROVIDER credential_chain)"
)
reader = connection.sql(
f"SELECT * FROM '{source}'"
).to_arrow_reader()
return reader, connection
def products(self):
reader, connection = self._read_parquet(PRODUCTS_SOURCE)
try:
for batch in reader:
yield from batch.to_pylist()
finally:
connection.close()
def nutrition(self):
reader, connection = self._read_parquet(NUTRITION_SOURCE)
try:
for batch in reader:
yield from batch.to_pylist()
finally:
connection.close()