Neon-tech commited on
Commit
c66476d
·
verified ·
1 Parent(s): 7a4c541

Delete app.p

Browse files
Files changed (1) hide show
  1. app.p +0 -166
app.p DELETED
@@ -1,166 +0,0 @@
1
- import os
2
- import io
3
- import json
4
- import time
5
- import socket
6
- import asyncio
7
- import threading
8
- import zipfile
9
- from pathlib import Path
10
-
11
- import pandas as pd
12
- import pyarrow as pa
13
- import pyarrow.parquet as pq
14
- from pyrogram import Client
15
-
16
- # ── Config ───────────────────────────────────────────────────────────────────
17
- SESSION = "session"
18
- SESSION_STRING = os.environ.get("sesson_string")
19
- CHANNEL = os.environ.get("channel")
20
- API_ID = os.environ.get("api_id")
21
- API_HASH = os.environ.get("api_hash")
22
- OUT_DIR = "/data/image-shards"
23
- STATE_FILE = "/data/images_state.json"
24
- IMAGES_PER_SHARD = 1000
25
- DOWNLOAD_DELAY = 10 # seconds between zip downloads
26
-
27
- os.makedirs(OUT_DIR, exist_ok=True)
28
-
29
- # ── Port 7860 ─────────────────────────────────────────────────────────────────
30
- def serve():
31
- s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
32
- s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
33
- s.bind(("0.0.0.0", 7860))
34
- s.listen(5)
35
- print("✓ Listening on port 7860")
36
- while True:
37
- conn, _ = s.accept()
38
- conn.send(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK")
39
- conn.close()
40
-
41
- # ── State ─────────────────────────────────────────────────────────────────────
42
- def load_state():
43
- if os.path.exists(STATE_FILE):
44
- with open(STATE_FILE) as f:
45
- state = json.load(f)
46
- print(f"Resuming — {len(state['done_msgs'])} zips done | {state['total_images']} images | {state['total_shards']} shards")
47
- else:
48
- state = {
49
- "done_msgs": [], # message IDs already processed
50
- "total_images": 0,
51
- "total_shards": 0,
52
- }
53
- print("Starting fresh")
54
- return state
55
-
56
- def save_state(state):
57
- try:
58
- with open(STATE_FILE, "w") as f:
59
- json.dump(state, f, indent=2)
60
- except OSError as e:
61
- print(f" ⚠ State save failed: {e}")
62
-
63
- # ── Write one parquet shard ───────────────────────────────────────────────────
64
- def write_shard(rows, shard_idx):
65
- shard_path = Path(OUT_DIR) / f"shard_{shard_idx:06d}.parquet"
66
- df = pd.DataFrame(rows, columns=["image_id", "image", "description", "tags", "source"])
67
- table = pa.Table.from_pandas(df)
68
- pq.write_table(table, shard_path, compression="snappy")
69
- print(f" ✓ shard_{shard_idx:06d}.parquet | {len(rows)} images")
70
-
71
- # ── Main async loop ───────────────────────────────────────────────────────────
72
- async def process(state):
73
- buffer = [] # accumulate rows here
74
- shard_idx = state["total_shards"]
75
-
76
- async with Client(SESSION, session_string=SESSION_STRING, api_id=API_ID, api_hash=API_HASH) as app:
77
- # iterate all messages in channel
78
- async for msg in app.get_chat_history(CHANNEL):
79
- # only process zip file messages
80
- if not msg.document:
81
- continue
82
- if not msg.document.file_name or not msg.document.file_name.endswith(".zip"):
83
- continue
84
- if msg.id in state["done_msgs"]:
85
- print(f" SKIP msg {msg.id}")
86
- continue
87
-
88
- print(f"Downloading zip msg {msg.id} — {msg.document.file_name} ({msg.document.file_size:,} bytes)")
89
-
90
- # download zip into memory
91
- zip_bytes = io.BytesIO()
92
- await app.download_media(msg, in_memory=True, progress=lambda c, t: None)
93
-
94
- # re-download properly into BytesIO
95
- zip_bytes = io.BytesIO()
96
- async for chunk in app.stream_media(msg):
97
- zip_bytes.write(chunk)
98
- zip_bytes.seek(0)
99
-
100
- # extract image + meta pairs
101
- try:
102
- with zipfile.ZipFile(zip_bytes) as zf:
103
- names = zf.namelist()
104
- img_names = [n for n in names if not n.endswith(".meta.json") and not n.endswith("/")]
105
-
106
- for img_name in img_names:
107
- meta_name = img_name + ".meta.json"
108
- image_id = Path(img_name).stem
109
-
110
- try:
111
- img_bytes = zf.read(img_name)
112
- except Exception:
113
- continue
114
-
115
- description = ""
116
- tags = []
117
- if meta_name in names:
118
- try:
119
- meta = json.loads(zf.read(meta_name))
120
- description = meta.get("description", "")
121
- tags = meta.get("tags", [])
122
- except Exception:
123
- pass
124
-
125
- buffer.append({
126
- "image_id": image_id,
127
- "image": img_bytes,
128
- "description": description,
129
- "tags": tags,
130
- # "source": "unsplash",
131
- })
132
-
133
- state["total_images"] += 1
134
-
135
- if len(buffer) >= IMAGES_PER_SHARD:
136
- write_shard(buffer, shard_idx)
137
- shard_idx += 1
138
- state["total_shards"] = shard_idx
139
- buffer = []
140
- save_state(state)
141
-
142
- except Exception as e:
143
- print(f" ✗ zip error msg {msg.id}: {e}")
144
- continue
145
-
146
- state["done_msgs"].append(msg.id)
147
- save_state(state)
148
- print(f" ✓ msg {msg.id} done | {state['total_images']} images total")
149
- await asyncio.sleep(DOWNLOAD_DELAY)
150
-
151
- # write remaining buffer
152
- if buffer:
153
- write_shard(buffer, shard_idx)
154
- state["total_shards"] = shard_idx + 1
155
- save_state(state)
156
-
157
- print(f"\n✓ All done! {state['total_images']} images | {state['total_shards']} shards in /data/image-shards")
158
-
159
- # ── Entry point ───────────────────────────────────────────────────────────────
160
- def main():
161
- threading.Thread(target=serve, daemon=True).start()
162
- state = load_state()
163
- asyncio.run(process(state))
164
-
165
- if __name__ == "__main__":
166
- main()