urstark commited on
Commit
52ca8dd
·
0 Parent(s):

feat: implement dual-header authentication and integrated vaulting pipeline

Browse files
Files changed (6) hide show
  1. .gitattributes +35 -0
  2. .gitignore +8 -0
  3. Dockerfile +28 -0
  4. README.md +37 -0
  5. app.py +179 -0
  6. requirements.txt +8 -0
.gitattributes ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ __pycache__/
2
+ *.pyc
3
+ *.pyo
4
+ *.pyd
5
+ .env
6
+ venv/
7
+ .venv/
8
+ *.log
Dockerfile ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Use a slim Python image for efficiency
2
+ FROM python:3.10-slim
3
+
4
+ # Install system dependencies needed for tgcrypto and network tools
5
+ RUN apt-get update && apt-get install -y \
6
+ gcc \
7
+ python3-dev \
8
+ && rm -rf /var/lib/apt/lists/*
9
+
10
+ # Create a non-root user as required by Hugging Face Spaces
11
+ RUN useradd -m -u 1000 user
12
+ USER user
13
+ ENV PATH="/home/user/.local/bin:$PATH"
14
+
15
+ WORKDIR /app
16
+
17
+ # Copy requirements first to leverage Docker cache
18
+ COPY --chown=user requirements.txt requirements.txt
19
+ RUN pip install --no-cache-dir --upgrade -r requirements.txt
20
+
21
+ # Copy the application code
22
+ COPY --chown=user . /app
23
+
24
+ # Hugging Face Spaces listen on port 7860 by default
25
+ EXPOSE 7860
26
+
27
+ # Start the FastAPI app using uvicorn
28
+ CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
README.md ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: Sonic Vault Heavy Lifter
3
+ emoji: 🚀
4
+ colorFrom: blue
5
+ colorTo: indigo
6
+ sdk: docker
7
+ pinned: false
8
+ ---
9
+
10
+ # Sonic Vault Heavy Lifter
11
+
12
+ This is the high-concurrency media vaulting engine for the Sonic API. It handles large media downloads and uploads to Telegram using MTProto for maximum performance and to avoid Cloudflare Worker timeout limitations.
13
+
14
+ ## Role
15
+ - Receives vaulting requests from the Sonic API Worker.
16
+ - Downloads media using high-speed external streamers.
17
+ - Uploads to Telegram using Pyrogram (MTProto).
18
+ - Synchronizes metadata back to the API via a secure callback.
19
+
20
+ ## Technical Details
21
+ - **Framework**: FastAPI (Asynchronous)
22
+ - **Engine**: Pyrogram (MTProto)
23
+ - **Deployment**: Docker (Standard HF Port 7860)
24
+
25
+ ## Required Secrets
26
+ To run this Space, you must configure the following secrets in the Space Settings:
27
+
28
+ | Secret | Description |
29
+ | :--- | :--- |
30
+ | `SECRET_TOKEN` | Shared secret for authentication with the main API. |
31
+ | `TELEGRAM_API_ID` | Your API ID from my.telegram.org. |
32
+ | `TELEGRAM_API_HASH` | Your API Hash from my.telegram.org. |
33
+ | `TELEGRAM_BOT_TOKEN` | Your Telegram Bot token. |
34
+ | `TELEGRAM_CHAT_ID` | The numeric ID of the storage chat. |
35
+
36
+ ## Local Testing
37
+ You can test the logic locally using `run_local_test.py`. Ensure your `.env` is configured correctly.
app.py ADDED
@@ -0,0 +1,179 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import FastAPI, HTTPException, Header, BackgroundTasks
2
+ from pydantic import BaseModel
3
+ import os
4
+ import asyncio
5
+ import aiohttp
6
+ import aiofiles
7
+ from pyrogram import Client
8
+ import uuid
9
+ from dotenv import load_dotenv
10
+
11
+ # Load .env from parent directory if it exists
12
+ load_dotenv("../.env")
13
+
14
+ app = FastAPI()
15
+
16
+ # Environment Variables
17
+ API_ID = int(os.getenv("TELEGRAM_API_ID") or os.getenv("API_ID") or "0")
18
+ API_HASH = os.getenv("TELEGRAM_API_HASH") or os.getenv("API_HASH") or ""
19
+ BOT_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN", "")
20
+ CHAT_ID = int(os.getenv("TELEGRAM_CHAT_ID", "0"))
21
+ SECRET_TOKEN = os.getenv("SECRET_TOKEN", "")
22
+ EXTERNAL_API_URL = os.getenv("EXTERNAL_API_URL", "https://shrutibots.site")
23
+
24
+ # Initialize Pyrogram Client
25
+ bot = Client(
26
+ "sonic_vault_bot",
27
+ api_id=API_ID,
28
+ api_hash=API_HASH,
29
+ bot_token=BOT_TOKEN,
30
+ in_memory=True
31
+ )
32
+
33
+ class VaultRequest(BaseModel):
34
+ videoId: str
35
+ type: str # 'audio' or 'video'
36
+ title: str
37
+ downloadUrl: str = None # Optional now, we can fetch it ourselves
38
+ callbackUrl: str = None
39
+
40
+ @app.on_event("startup")
41
+ async def startup():
42
+ await bot.start()
43
+
44
+ @app.on_event("shutdown")
45
+ async def shutdown():
46
+ await bot.stop()
47
+
48
+ async def fetch_download_url(video_id: str, media_type: str):
49
+ try:
50
+ async with aiohttp.ClientSession() as session:
51
+ # 1. Get Token
52
+ async with session.get(f"{EXTERNAL_API_URL}/download?url={video_id}&type={media_type}") as resp:
53
+ if resp.status == 200:
54
+ data = await resp.json()
55
+ token = data.get("download_token")
56
+ if token:
57
+ return f"{EXTERNAL_API_URL}/stream/{video_id}?type={media_type}&token={token}"
58
+ except Exception as e:
59
+ print(f"[MTProto] Error fetching download URL: {str(e)}")
60
+ return None
61
+
62
+ async def process_vault(request: VaultRequest):
63
+ file_path = f"/tmp/{request.videoId}_{uuid.uuid4().hex[:8]}.{'mp4' if request.type == 'video' else 'mp3'}"
64
+
65
+ try:
66
+ # 0. Resolve Download URL if not provided
67
+ download_url = request.downloadUrl
68
+ if not download_url:
69
+ print(f"[MTProto] Resolving download URL for {request.videoId}...")
70
+ download_url = await fetch_download_url(request.videoId, request.type)
71
+
72
+ if not download_url:
73
+ raise Exception("Could not resolve download URL")
74
+
75
+ # 1. Download
76
+ print(f"[MTProto] Downloading from {download_url}...")
77
+ async with aiohttp.ClientSession() as session:
78
+ async with session.get(download_url) as resp:
79
+ if resp.status != 200:
80
+ raise Exception(f"Download failed with status {resp.status}")
81
+
82
+ async with aiofiles.open(file_path, mode='wb') as f:
83
+ async for chunk in resp.content.iter_chunked(1024 * 1024): # 1MB chunks
84
+ await f.write(chunk)
85
+
86
+ file_size = os.path.getsize(file_path)
87
+ print(f"[MTProto] Downloaded {file_size} bytes to {file_path}")
88
+
89
+ # 2. Upload
90
+ print(f"[MTProto] Starting upload to chat {CHAT_ID} (Type: {request.type})")
91
+ caption = f"{request.title}\nID: {request.videoId}"
92
+
93
+ if request.type == 'video':
94
+ msg = await bot.send_video(
95
+ chat_id=CHAT_ID,
96
+ video=file_path,
97
+ caption=caption,
98
+ file_name=f"{request.title}.mp4"
99
+ )
100
+ file_id = msg.video.file_id
101
+ else:
102
+ msg = await bot.send_audio(
103
+ chat_id=CHAT_ID,
104
+ audio=file_path,
105
+ caption=caption,
106
+ title=request.title,
107
+ file_name=f"{request.title}.mp3"
108
+ )
109
+ file_id = msg.audio.file_id
110
+
111
+ print(f"[MTProto] Upload success! Message ID: {msg.id}")
112
+
113
+ # 3. Callback with Retry Logic
114
+ if request.callbackUrl:
115
+ payload = {
116
+ "status": "success",
117
+ "telegram_file_id": file_id,
118
+ "message_id": msg.id,
119
+ "videoId": request.videoId,
120
+ "type": request.type,
121
+ "title": request.title
122
+ }
123
+
124
+ for attempt in range(3):
125
+ try:
126
+ async with aiohttp.ClientSession() as session:
127
+ async with session.post(
128
+ request.callbackUrl,
129
+ json=payload,
130
+ headers={"Authorization": f"Bearer {SECRET_TOKEN}"},
131
+ timeout=10
132
+ ) as cb_resp:
133
+ if cb_resp.status == 200:
134
+ print(f"[MTProto] Callback success on attempt {attempt+1}")
135
+ break
136
+ else:
137
+ print(f"[MTProto] Callback failed with status {cb_resp.status} on attempt {attempt+1}")
138
+ except Exception as cb_err:
139
+ print(f"[MTProto] Callback error on attempt {attempt+1}: {str(cb_err)}")
140
+
141
+ if attempt < 2:
142
+ await asyncio.sleep(2 ** attempt * 5) # 5s, 10s backoff
143
+
144
+ except Exception as e:
145
+ print(f"Error processing vault: {str(e)}")
146
+ if request.callbackUrl:
147
+ # Error Callback (also with 1 retry)
148
+ try:
149
+ async with aiohttp.ClientSession() as session:
150
+ await session.post(
151
+ request.callbackUrl,
152
+ json={"status": "error", "error": str(e), "videoId": request.videoId},
153
+ headers={"Authorization": f"Bearer {SECRET_TOKEN}"},
154
+ timeout=10
155
+ )
156
+ except Exception as cb_err:
157
+ print(f"[MTProto] Error callback failed: {str(cb_err)}")
158
+ finally:
159
+ if os.path.exists(file_path):
160
+ os.remove(file_path)
161
+
162
+ @app.post("/vault")
163
+ async def vault_media(request: VaultRequest, background_tasks: BackgroundTasks, x_secret_token: str = Header(None), authorization: str = Header(None)):
164
+ # Prefer X-Secret-Token, fallback to Authorization
165
+ token = x_secret_token
166
+ if not token and authorization and authorization.startswith("Bearer "):
167
+ parts = authorization.split(" ")
168
+ token = parts[1] if len(parts) > 1 else None
169
+
170
+ if token != SECRET_TOKEN:
171
+ raise HTTPException(status_code=401, detail="Unauthorized")
172
+
173
+ background_tasks.add_task(process_vault, request)
174
+ return {"status": "accepted", "message": "Vaulting started in background"}
175
+
176
+
177
+ @app.get("/health")
178
+ async def health():
179
+ return {"status": "ok"}
requirements.txt ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ fastapi
2
+ uvicorn[standard]
3
+ python-dotenv
4
+ pydantic
5
+ aiohttp
6
+ aiofiles
7
+ pyrogram
8
+ tgcrypto