Spaces:
Paused
Paused
Download app.py from Seri19/Zstd-Super-Search: direct link, hf CLI and curl.
- Browser
- Download file 4.92 kB
-
https://huggingface.co/spaces/Seri19/Zstd-Super-Search/resolve/main/app.py
- Command line
-
hf download hf://spaces/Seri19/Zstd-Super-Search/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Seri19/Zstd-Super-Search/resolve/main/app.py
4.92 kB
| import streamlit as st | |
| import zstandard as zstd | |
| import os | |
| import io | |
| import time | |
| import csv | |
| import pandas as pd | |
| # Настройка страницы | |
| st.set_page_config(page_title="ZSTD Ultra Search", layout="wide") | |
| # ВАЖНО: Если грузишь файлы через вкладку "Files", они падают в корень (.). | |
| # Поэтому ищем файлы прямо здесь. | |
| DATA_DIR = "." | |
| def get_file_size(path): | |
| return os.path.getsize(path) / (1024 * 1024) | |
| def compress_file_path(input_path): | |
| """Сжимает файл, лежащий на диске""" | |
| save_path = input_path + ".zst" | |
| cctx = zstd.ZstdCompressor(level=3) | |
| with open(input_path, 'rb') as f_in, open(save_path, 'wb') as f_out: | |
| compressor = cctx.stream_writer(f_out) | |
| while True: | |
| chunk = f_in.read(65536) | |
| if not chunk: | |
| break | |
| compressor.write(chunk) | |
| compressor.flush(zstd.FLUSH_FRAME) | |
| return save_path | |
| def search_in_zstd(file_path, query, limit=100): | |
| results = [] | |
| headers = None | |
| # Проверяем, был ли исходник CSV (по имени zst файла: name.csv.zst) | |
| is_csv = ".csv" in file_path.lower() | |
| dctx = zstd.ZstdDecompressor() | |
| try: | |
| with open(file_path, 'rb') as f_in: | |
| with dctx.stream_reader(f_in) as reader: | |
| text_stream = io.TextIOWrapper(reader, encoding='utf-8', errors='replace') | |
| if is_csv: | |
| for line in text_stream: | |
| headers = next(csv.reader([line])) | |
| break | |
| count = 0 | |
| for line in text_stream: | |
| if query.lower() in line.lower(): | |
| if is_csv: | |
| try: | |
| row = next(csv.reader([line])) | |
| results.append(row) | |
| except: | |
| results.append([line.strip()]) | |
| else: | |
| results.append(line.strip()) | |
| count += 1 | |
| if count >= limit: | |
| break | |
| except Exception as e: | |
| return None, [f"Error: {e}"] | |
| return headers, results | |
| # --- АВТОМАТИЧЕСКИЙ ПОИСК НОВЫХ ФАЙЛОВ --- | |
| # Это блок для "Хакерского метода" | |
| st.sidebar.header("System Status") | |
| raw_files = [f for f in os.listdir(DATA_DIR) if f.endswith(('.csv', '.sql', '.txt', '.log'))] | |
| zst_files = [f for f in os.listdir(DATA_DIR) if f.endswith('.zst')] | |
| # Проверяем, есть ли "сырые" файлы без пары .zst | |
| for raw in raw_files: | |
| expected_zst = raw + ".zst" | |
| if expected_zst not in zst_files: | |
| st.toast(f"Found new file: {raw}. Compressing...", icon="⚙️") | |
| with st.spinner(f"Auto-compressing {raw}..."): | |
| compress_file_path(os.path.join(DATA_DIR, raw)) | |
| st.success(f"Processed {raw}!") | |
| time.sleep(1) | |
| st.rerun() # Перезагрузить страницу, чтобы обновить список | |
| # --- Интерфейс --- | |
| st.title("🚀 ZSTD Database Engine") | |
| # 1. Обычная загрузка (Drag & Drop) | |
| uploaded_file = st.sidebar.file_uploader("Upload via UI (Optional)") | |
| if uploaded_file is not None: | |
| # Сохраняем файл на диск | |
| save_path = os.path.join(DATA_DIR, uploaded_file.name) | |
| with open(save_path, "wb") as f: | |
| f.write(uploaded_file.getbuffer()) | |
| st.sidebar.success("Uploaded! Auto-compressor will pick it up.") | |
| st.rerun() | |
| # 2. Поиск | |
| st.header("Search Engine") | |
| # Обновляем список доступных баз | |
| available_dbs = [f for f in os.listdir(DATA_DIR) if f.endswith('.zst')] | |
| if available_dbs: | |
| selected_db = st.selectbox("Select Database", available_dbs) | |
| search_query = st.text_input("Search Query", "") | |
| if st.button("Search") and search_query: | |
| full_path = os.path.join(DATA_DIR, selected_db) | |
| with st.spinner("Scanning stream..."): | |
| headers, matches = search_in_zstd(full_path, search_query, limit=200) | |
| if matches: | |
| st.success(f"Found {len(matches)} results.") | |
| if headers and isinstance(matches[0], list): | |
| try: | |
| df = pd.DataFrame(matches, columns=headers) | |
| st.dataframe(df, use_container_width=True) | |
| except: | |
| st.table(matches) | |
| else: | |
| st.code("\n".join([str(m) for m in matches]), language="sql") | |
| else: | |
| st.warning("No matches found.") | |
| else: | |
| st.info("No databases found. Upload a .csv/.sql file via 'Files' tab or Sidebar.") |