Seri19's picture
Update app.py
92dcd3d verified
Raw History Blame Contribute Delete
4.92 kB
import streamlit as st
import zstandard as zstd
import os
import io
import time
import csv
import pandas as pd
# Настройка страницы
st.set_page_config(page_title="ZSTD Ultra Search", layout="wide")
# ВАЖНО: Если грузишь файлы через вкладку "Files", они падают в корень (.).
# Поэтому ищем файлы прямо здесь.
DATA_DIR = "."
def get_file_size(path):
return os.path.getsize(path) / (1024 * 1024)
def compress_file_path(input_path):
"""Сжимает файл, лежащий на диске"""
save_path = input_path + ".zst"
cctx = zstd.ZstdCompressor(level=3)
with open(input_path, 'rb') as f_in, open(save_path, 'wb') as f_out:
compressor = cctx.stream_writer(f_out)
while True:
chunk = f_in.read(65536)
if not chunk:
break
compressor.write(chunk)
compressor.flush(zstd.FLUSH_FRAME)
return save_path
def search_in_zstd(file_path, query, limit=100):
results = []
headers = None
# Проверяем, был ли исходник CSV (по имени zst файла: name.csv.zst)
is_csv = ".csv" in file_path.lower()
dctx = zstd.ZstdDecompressor()
try:
with open(file_path, 'rb') as f_in:
with dctx.stream_reader(f_in) as reader:
text_stream = io.TextIOWrapper(reader, encoding='utf-8', errors='replace')
if is_csv:
for line in text_stream:
headers = next(csv.reader([line]))
break
count = 0
for line in text_stream:
if query.lower() in line.lower():
if is_csv:
try:
row = next(csv.reader([line]))
results.append(row)
except:
results.append([line.strip()])
else:
results.append(line.strip())
count += 1
if count >= limit:
break
except Exception as e:
return None, [f"Error: {e}"]
return headers, results
# --- АВТОМАТИЧЕСКИЙ ПОИСК НОВЫХ ФАЙЛОВ ---
# Это блок для "Хакерского метода"
st.sidebar.header("System Status")
raw_files = [f for f in os.listdir(DATA_DIR) if f.endswith(('.csv', '.sql', '.txt', '.log'))]
zst_files = [f for f in os.listdir(DATA_DIR) if f.endswith('.zst')]
# Проверяем, есть ли "сырые" файлы без пары .zst
for raw in raw_files:
expected_zst = raw + ".zst"
if expected_zst not in zst_files:
st.toast(f"Found new file: {raw}. Compressing...", icon="⚙️")
with st.spinner(f"Auto-compressing {raw}..."):
compress_file_path(os.path.join(DATA_DIR, raw))
st.success(f"Processed {raw}!")
time.sleep(1)
st.rerun() # Перезагрузить страницу, чтобы обновить список
# --- Интерфейс ---
st.title("🚀 ZSTD Database Engine")
# 1. Обычная загрузка (Drag & Drop)
uploaded_file = st.sidebar.file_uploader("Upload via UI (Optional)")
if uploaded_file is not None:
# Сохраняем файл на диск
save_path = os.path.join(DATA_DIR, uploaded_file.name)
with open(save_path, "wb") as f:
f.write(uploaded_file.getbuffer())
st.sidebar.success("Uploaded! Auto-compressor will pick it up.")
st.rerun()
# 2. Поиск
st.header("Search Engine")
# Обновляем список доступных баз
available_dbs = [f for f in os.listdir(DATA_DIR) if f.endswith('.zst')]
if available_dbs:
selected_db = st.selectbox("Select Database", available_dbs)
search_query = st.text_input("Search Query", "")
if st.button("Search") and search_query:
full_path = os.path.join(DATA_DIR, selected_db)
with st.spinner("Scanning stream..."):
headers, matches = search_in_zstd(full_path, search_query, limit=200)
if matches:
st.success(f"Found {len(matches)} results.")
if headers and isinstance(matches[0], list):
try:
df = pd.DataFrame(matches, columns=headers)
st.dataframe(df, use_container_width=True)
except:
st.table(matches)
else:
st.code("\n".join([str(m) for m in matches]), language="sql")
else:
st.warning("No matches found.")
else:
st.info("No databases found. Upload a .csv/.sql file via 'Files' tab or Sidebar.")