import streamlit as st import zstandard as zstd import os import io import time import csv import pandas as pd # Настройка страницы st.set_page_config(page_title="ZSTD Ultra Search", layout="wide") # ВАЖНО: Если грузишь файлы через вкладку "Files", они падают в корень (.). # Поэтому ищем файлы прямо здесь. DATA_DIR = "." def get_file_size(path): return os.path.getsize(path) / (1024 * 1024) def compress_file_path(input_path): """Сжимает файл, лежащий на диске""" save_path = input_path + ".zst" cctx = zstd.ZstdCompressor(level=3) with open(input_path, 'rb') as f_in, open(save_path, 'wb') as f_out: compressor = cctx.stream_writer(f_out) while True: chunk = f_in.read(65536) if not chunk: break compressor.write(chunk) compressor.flush(zstd.FLUSH_FRAME) return save_path def search_in_zstd(file_path, query, limit=100): results = [] headers = None # Проверяем, был ли исходник CSV (по имени zst файла: name.csv.zst) is_csv = ".csv" in file_path.lower() dctx = zstd.ZstdDecompressor() try: with open(file_path, 'rb') as f_in: with dctx.stream_reader(f_in) as reader: text_stream = io.TextIOWrapper(reader, encoding='utf-8', errors='replace') if is_csv: for line in text_stream: headers = next(csv.reader([line])) break count = 0 for line in text_stream: if query.lower() in line.lower(): if is_csv: try: row = next(csv.reader([line])) results.append(row) except: results.append([line.strip()]) else: results.append(line.strip()) count += 1 if count >= limit: break except Exception as e: return None, [f"Error: {e}"] return headers, results # --- АВТОМАТИЧЕСКИЙ ПОИСК НОВЫХ ФАЙЛОВ --- # Это блок для "Хакерского метода" st.sidebar.header("System Status") raw_files = [f for f in os.listdir(DATA_DIR) if f.endswith(('.csv', '.sql', '.txt', '.log'))] zst_files = [f for f in os.listdir(DATA_DIR) if f.endswith('.zst')] # Проверяем, есть ли "сырые" файлы без пары .zst for raw in raw_files: expected_zst = raw + ".zst" if expected_zst not in zst_files: st.toast(f"Found new file: {raw}. Compressing...", icon="⚙️") with st.spinner(f"Auto-compressing {raw}..."): compress_file_path(os.path.join(DATA_DIR, raw)) st.success(f"Processed {raw}!") time.sleep(1) st.rerun() # Перезагрузить страницу, чтобы обновить список # --- Интерфейс --- st.title("🚀 ZSTD Database Engine") # 1. Обычная загрузка (Drag & Drop) uploaded_file = st.sidebar.file_uploader("Upload via UI (Optional)") if uploaded_file is not None: # Сохраняем файл на диск save_path = os.path.join(DATA_DIR, uploaded_file.name) with open(save_path, "wb") as f: f.write(uploaded_file.getbuffer()) st.sidebar.success("Uploaded! Auto-compressor will pick it up.") st.rerun() # 2. Поиск st.header("Search Engine") # Обновляем список доступных баз available_dbs = [f for f in os.listdir(DATA_DIR) if f.endswith('.zst')] if available_dbs: selected_db = st.selectbox("Select Database", available_dbs) search_query = st.text_input("Search Query", "") if st.button("Search") and search_query: full_path = os.path.join(DATA_DIR, selected_db) with st.spinner("Scanning stream..."): headers, matches = search_in_zstd(full_path, search_query, limit=200) if matches: st.success(f"Found {len(matches)} results.") if headers and isinstance(matches[0], list): try: df = pd.DataFrame(matches, columns=headers) st.dataframe(df, use_container_width=True) except: st.table(matches) else: st.code("\n".join([str(m) for m in matches]), language="sql") else: st.warning("No matches found.") else: st.info("No databases found. Upload a .csv/.sql file via 'Files' tab or Sidebar.")