tgtobl2 / main.py
o7c1's picture
Update main.py
13149ac verified
Raw History Blame Contribute Delete
24.1 kB
import os
import asyncio
import tempfile
import zipfile
import httpx
import shutil
import re
from telethon import TelegramClient, events
from telethon.sessions import StringSession
# Import the specific Telethon error
from telethon.errors import AuthKeyDuplicatedError
from fastapi import FastAPI
import uvicorn
from contextlib import asynccontextmanager
# --- 1. CONFIGURATION ---
def get_env(name):
return os.getenv(name)
TG_API_ID = int(get_env("TG_API_ID") or 0)
TG_API_HASH = get_env("TG_API_HASH")
TG_SESSION_STRING = get_env("TG_SESSION_STRING")
BALE_BOT_TOKEN = get_env("BALE_BOT_TOKEN")
BALE_CHAT_ID = get_env("BALE_CHAT_ID") # kept for backward compatibility
bale_ids_raw = get_env("BALE_CHAT_IDS") # comma-separated for multiple chats
if bale_ids_raw:
BALE_CHAT_IDS = [x.strip() for x in bale_ids_raw.split(",") if x.strip()]
else:
BALE_CHAT_IDS = [BALE_CHAT_ID] if BALE_CHAT_ID else []
BALE_LOG_CHAT_ID = get_env("BALE_LOG_CHAT_ID") or (BALE_CHAT_IDS[0] if BALE_CHAT_IDS else None)
env_channels = [c.strip().lower() for c in (get_env("TARGET_CHANNELS") or "").split(",") if c.strip()]
TARGET_CHANNELS = list(set(env_channels + ["iu39f", "TirexNet", "xsfilterrnet"]))
CHUNK_SIZE = 19 * 1024 * 1024 # 19 MB
MAX_TEXT_LEN = 4000 # Bale maximum message length
LARGE_FILE_SEMAPHORE = asyncio.Semaphore(2)
BALE_API_SEMAPHORE = asyncio.Semaphore(3)
# Optional filter expression
FILTER_EXPR = get_env("FILTER_EXPR") or ""
REMOVE_WORDS = [w.strip().lower() for w in (get_env("REMOVE_WORDS") or "").split(",") if w.strip()]
# --- 2. FILTER EXPRESSION PARSER & EVALUATOR ---
class FilterExpr:
"""Compile a filter expression string into an AST and evaluate against text."""
def __init__(self, expr_str: str):
self.ast = None
if not expr_str.strip():
return
try:
tokens = self._tokenize(expr_str)
tokens = self._insert_and_operators(tokens)
self.ast = self._parse(tokens)
except Exception as e:
print(f"⚠️ Invalid FILTER_EXPR: {e} – ignoring filter")
self.ast = None
def _tokenize(self, s):
token_re = re.compile(r'\(|\)|!|\||"[^"]*"|[^\s()!|"]+')
raw_tokens = token_re.findall(s)
typed = []
for t in raw_tokens:
if t == '(':
typed.append(('LPAREN', t))
elif t == ')':
typed.append(('RPAREN', t))
elif t == '!':
typed.append(('NOT', t))
elif t == '|':
typed.append(('OR', t))
elif t.startswith('"') and t.endswith('"'):
typed.append(('WORD', t[1:-1].lower()))
else:
typed.append(('WORD', t.lower()))
return typed
def _insert_and_operators(self, tokens):
if not tokens:
return []
terminators = {'WORD', 'RPAREN'}
starters = {'WORD', 'LPAREN', 'NOT'}
result = [tokens[0]]
for i in range(1, len(tokens)):
prev = tokens[i-1]
curr = tokens[i]
if prev[0] in terminators and curr[0] in starters:
result.append(('AND', '&'))
result.append(curr)
return result
def _parse(self, tokens):
self._pos = 0
self._tokens = tokens
ast = self._expr()
if self._pos != len(self._tokens):
raise ValueError(f"Unexpected token: {self._current()}")
return ast
def _current(self):
return self._tokens[self._pos] if self._pos < len(self._tokens) else None
def _advance(self):
self._pos += 1
def _expr(self): # OR level
left = self._term()
while self._current() and self._current()[0] == 'OR':
self._advance()
right = self._term()
left = ('OR', left, right)
return left
def _term(self): # AND level
left = self._factor()
while self._current() and self._current()[0] == 'AND':
self._advance()
right = self._factor()
left = ('AND', left, right)
return left
def _factor(self): # NOT level
if self._current() and self._current()[0] == 'NOT':
self._advance()
child = self._factor()
return ('NOT', child)
return self._atom()
def _atom(self):
tok = self._current()
if not tok:
raise ValueError("Unexpected end of expression")
if tok[0] == 'LPAREN':
self._advance()
node = self._expr()
if not self._current() or self._current()[0] != 'RPAREN':
raise ValueError("Missing )")
self._advance()
return node
elif tok[0] == 'WORD':
self._advance()
return ('WORD', tok[1])
else:
raise ValueError(f"Unexpected token: {tok}")
def evaluate(self, text: str) -> bool:
"""Return True if the filter expression matches the text (message passes)."""
if self.ast is None:
return True
text_lower = text.lower()
return self._eval_node(self.ast, text_lower)
def _eval_node(self, node, text):
node_type = node[0]
if node_type == 'WORD':
return node[1] in text
elif node_type == 'AND':
return self._eval_node(node[1], text) and self._eval_node(node[2], text)
elif node_type == 'OR':
return self._eval_node(node[1], text) or self._eval_node(node[2], text)
elif node_type == 'NOT':
return not self._eval_node(node[1], text)
else:
return False
msg_filter = FilterExpr(FILTER_EXPR)
def remove_words(text: str, words: list) -> str:
"""Remove all occurrences of each word (case‑insensitive) from text."""
if not words or not text:
return text
import re
for w in words:
text = re.sub(re.escape(w), '', text, flags=re.IGNORECASE)
return text
# --- 3. TELEGRAM CLIENT ---
# Note: AuthKeyDuplicatedError is imported above in the global imports block
tg_client = TelegramClient(StringSession(TG_SESSION_STRING), TG_API_ID, TG_API_HASH)
# --- 4. BALE HELPERS (multi‑chat, with retries & splitting) ---
async def send_text_to_bale(text, parse_mode=None, retries=3, log_error=True):
if not BALE_BOT_TOKEN or not BALE_CHAT_IDS:
return False
overall = True
for chat_id in BALE_CHAT_IDS:
url = f"https://tapi.bale.ai/bot{BALE_BOT_TOKEN}/sendMessage"
payload = {"chat_id": chat_id, "text": text}
if parse_mode:
payload["parse_mode"] = parse_mode
success = False
for attempt in range(1, retries + 1):
try:
async with BALE_API_SEMAPHORE:
async with httpx.AsyncClient() as client:
resp = await client.post(url, json=payload, timeout=10)
if resp.status_code == 200:
success = True
break
else:
if attempt < retries:
await asyncio.sleep(2 ** (attempt - 1))
continue
if log_error:
await send_log_to_bale(
f"send_text_to_bale failed after {retries} attempts (chat {chat_id}): "
f"{resp.status_code} {resp.text[:200]}"
)
except Exception as e:
if attempt < retries:
await asyncio.sleep(2 ** (attempt - 1))
continue
if log_error:
await send_log_to_bale(
f"send_text_to_bale exception after {retries} attempts (chat {chat_id}): {e}"
)
if not success:
overall = False
return overall
async def send_media_to_bale(file_path, caption, media_type, parse_mode=None, retries=3):
if not BALE_BOT_TOKEN or not BALE_CHAT_IDS:
return False
endpoint_map = {
"photo": "sendPhoto",
"video": "sendVideo",
"document": "sendDocument",
"audio": "sendAudio",
"voice": "sendVoice",
"sticker": "sendSticker",
"video_note": "sendVideoNote",
}
endpoint = endpoint_map.get(media_type, "sendDocument")
field_name = media_type if media_type in endpoint_map else "document"
overall = True
for chat_id in BALE_CHAT_IDS:
url = f"https://tapi.bale.ai/bot{BALE_BOT_TOKEN}/{endpoint}"
data = {"chat_id": chat_id}
if caption:
data["caption"] = caption
if parse_mode:
data["parse_mode"] = parse_mode
success = False
for attempt in range(1, retries + 1):
try:
with open(file_path, "rb") as f:
files = {field_name: f}
async with BALE_API_SEMAPHORE:
async with httpx.AsyncClient() as client:
resp = await client.post(url, data=data, files=files, timeout=30)
if resp.status_code == 200:
success = True
break
else:
if attempt < retries:
await asyncio.sleep(2 ** (attempt - 1))
continue
await send_log_to_bale(
f"send_media_to_bale failed after {retries} attempts "
f"({media_type}) chat {chat_id}: {resp.status_code} {resp.text[:300]}\n"
f"File: {os.path.basename(file_path)}"
)
except Exception as e:
if attempt < retries:
await asyncio.sleep(2 ** (attempt - 1))
continue
await send_log_to_bale(
f"send_media_to_bale exception after {retries} attempts "
f"({media_type}) chat {chat_id}: {e}\nFile: {os.path.basename(file_path)}"
)
if not success:
overall = False
return overall
async def send_log_to_bale(message):
if not BALE_BOT_TOKEN or not BALE_LOG_CHAT_ID:
return
url = f"https://tapi.bale.ai/bot{BALE_BOT_TOKEN}/sendMessage"
payload = {"chat_id": BALE_LOG_CHAT_ID, "text": f"🔧 Bot Log:\n{message}"}
try:
async with httpx.AsyncClient() as client:
await client.post(url, json=payload, timeout=5)
except Exception:
pass
def split_long_text(text, max_len=MAX_TEXT_LEN):
"""Split a long text into chunks of max_len characters."""
if len(text) <= max_len:
return [text]
chunks = []
while text:
if len(text) <= max_len:
chunks.append(text)
break
split_at = text.rfind('\n', 0, max_len)
if split_at == -1:
split_at = text.rfind(' ', 0, max_len)
if split_at == -1:
split_at = max_len
chunk = text[:split_at].rstrip()
text = text[split_at:].lstrip()
if not chunk:
chunk = text[:max_len]
text = text[max_len:]
chunks.append(chunk)
return chunks
async def send_text_or_caption(text, parse_mode="HTML", is_caption=False, max_len=MAX_TEXT_LEN):
"""Send/caption-split long text. Returns (first_part, success) for captions, success (bool) for plain text."""
if not text:
return ("" if is_caption else True)
if len(text) > max_len and parse_mode:
parse_mode = None # avoid broken HTML when splitting
chunks = split_long_text(text, max_len)
total = len(chunks)
if is_caption:
first_chunk = chunks[0]
if total == 1:
return first_chunk, True
first_chunk = first_chunk + f"\n… (1/{total})"
success = True
for i in range(1, total):
chunk_text = f"{chunks[i]}\n… ({i+1}/{total})"
ok = await send_text_to_bale(chunk_text, parse_mode=None, log_error=False)
if not ok:
await send_log_to_bale(f"Failed to send caption part {i+1}/{total}")
success = False
await asyncio.sleep(0.5)
return first_chunk, success
# Plain text (no media)
success = True
for i, chunk in enumerate(chunks, 1):
if total > 1:
chunk = f"{chunk}\n… ({i}/{total})"
ok = await send_text_to_bale(chunk, parse_mode=parse_mode, log_error=False)
if not ok:
await send_log_to_bale(f"Failed to send text part {i}/{total}")
success = False
if i < total:
await asyncio.sleep(0.5)
return success
# --- 5. TEMP DIRECTORY HELPERS ---
def create_temp_dir():
return tempfile.mkdtemp(prefix="tg2bale_")
def cleanup_temp_dir(temp_dir):
try:
shutil.rmtree(temp_dir, ignore_errors=True)
except Exception:
pass
# --- 6. FILE OPERATIONS (threaded) ---
async def download_media(message, temp_dir):
if not message.media:
return None
try:
suffix = ""
if message.photo:
suffix = ".jpg"
elif message.video:
suffix = ".mp4"
elif getattr(message, 'document', None):
attrs = message.document.attributes
for attr in attrs:
if hasattr(attr, 'file_name'):
if '.' in attr.file_name:
suffix = f".{attr.file_name.split('.')[-1]}"
break
if not suffix:
mime = message.document.mime_type
if mime and '/' in mime:
suffix = f".{mime.split('/')[-1]}"
elif message.audio:
suffix = ".mp3"
elif message.voice:
suffix = ".ogg"
elif message.sticker:
suffix = ".webp"
elif message.video_note:
suffix = ".mp4"
file_path = os.path.join(temp_dir, f"media{suffix}")
await message.download_media(file=file_path)
return file_path
except Exception as e:
await send_log_to_bale(f"Media download error: {e}")
return None
def zip_file_sync(file_path, zip_path):
with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zf:
zf.write(file_path, arcname=os.path.basename(file_path))
return zip_path
async def zip_file(file_path, zip_path):
return await asyncio.to_thread(zip_file_sync, file_path, zip_path)
def split_file_sync(file_path, chunk_size=CHUNK_SIZE):
chunk_paths = []
base = file_path + ".part"
with open(file_path, "rb") as f:
i = 0
while True:
chunk_data = f.read(chunk_size)
if not chunk_data:
break
chunk_path = f"{base}{i:03d}.zip"
with open(chunk_path, "wb") as chunk_file:
chunk_file.write(chunk_data)
chunk_paths.append(chunk_path)
i += 1
return chunk_paths
async def split_file(file_path, chunk_size=CHUNK_SIZE):
return await asyncio.to_thread(split_file_sync, file_path, chunk_size)
# --- 7. MEDIA DETECTION / SIZE ---
def detect_media_type(event):
if event.photo:
return "photo"
if event.video:
return "video"
if event.audio:
return "audio"
if event.voice:
return "voice"
if event.sticker:
return "sticker"
if event.video_note:
return "video_note"
if event.document:
return "document"
return None
def get_media_size(event):
try:
if event.photo:
return event.photo.sizes[-1].size
if event.video:
return event.video.size
if event.audio:
return event.audio.size
if event.voice:
return event.voice.size
if event.video_note:
return event.video_note.size
if event.sticker:
return event.sticker.size if hasattr(event.sticker, 'size') else None
if event.document:
return event.document.size
except Exception:
pass
return None
def get_message_link(event, chat):
username = getattr(chat, 'username', None)
if username:
return f"https://t.me/{username}/{event.message.id}"
if getattr(chat, 'id', None):
return f"https://t.me/c/{abs(chat.id)}/{event.message.id}"
return ""
def build_caption(event, chat, cleaned_text=None):
msg_text = cleaned_text if cleaned_text is not None else (event.raw_text or "")
msg_link = get_message_link(event, chat)
preview_url = ""
if event.message.web_preview and event.message.web_preview.url:
preview_url = event.message.web_preview.url
parts = []
if msg_text:
parts.append(msg_text)
if preview_url:
parts.append(f'<a href="{preview_url}">{preview_url}</a>')
if msg_link:
parts.append(f'<a href="{msg_link}">📎 Open in Telegram</a>')
return "\n\n".join(parts)
async def handle_large_document(event, media_path, final_caption, temp_dir):
file_size = os.path.getsize(media_path) if os.path.exists(media_path) else 0
await send_log_to_bale(
f"Processing large file: {os.path.basename(media_path)}, size={file_size/(1024*1024):.1f} MB"
)
zip_path = os.path.join(temp_dir, "media.zip")
await zip_file(media_path, zip_path)
zip_size = os.path.getsize(zip_path)
if zip_size > CHUNK_SIZE:
chunks = await split_file(zip_path)
total = len(chunks)
await send_log_to_bale(f"ZIP {zip_size/(1024*1024):.1f} MB split into {total} chunks")
for idx, chunk_path in enumerate(chunks, start=1):
chunk_size_mb = os.path.getsize(chunk_path) / (1024 * 1024)
part_caption = f"{final_caption}\n📦 Part {idx}/{total}"
first_part, _ = await send_text_or_caption(part_caption, parse_mode="HTML", is_caption=True)
success = await send_media_to_bale(chunk_path, first_part, "document", parse_mode="HTML")
if not success:
success = await send_media_to_bale(chunk_path, first_part, "document", parse_mode=None)
if not success:
await send_log_to_bale(f"❌ Chunk {idx}/{total} FAILED permanently")
else:
await send_log_to_bale(f"✅ Chunk {idx}/{total} sent ({chunk_size_mb:.1f} MB)")
await asyncio.sleep(1.5)
else:
await send_log_to_bale(f"ZIP size {zip_size/(1024*1024):.1f} MB, sending whole")
first_part, _ = await send_text_or_caption(final_caption, parse_mode="HTML", is_caption=True)
await send_media_to_bale(zip_path, first_part, "document", parse_mode="HTML")
# --- 8. TELEGRAM EVENT HANDLER (main logic) ---
@tg_client.on(events.NewMessage)
async def handler(event):
if not event.is_channel:
return
chat = None
username = None
try:
chat = await event.get_chat()
username = getattr(chat, 'username', None)
if not username or username.lower() not in TARGET_CHANNELS:
return
# --- Clean up the text before forwarding ---
cleaned_text = remove_words(event.raw_text or "", REMOVE_WORDS)
header = f"📢 @{username}\n\n" if username else f"📢 {chat.title}\n\n"
caption_body = build_caption(event, chat, cleaned_text=cleaned_text)
final_caption = header + caption_body
# ----- Filter expression check ----- (still uses original raw_text)
if not msg_filter.evaluate(event.raw_text or ""):
return
# -----------------------------------
media_type = detect_media_type(event)
if not media_type:
await send_text_or_caption(final_caption, parse_mode="HTML")
return
media_size = get_media_size(event)
if media_size and media_size > CHUNK_SIZE:
async with LARGE_FILE_SEMAPHORE:
temp_dir = create_temp_dir()
try:
await send_log_to_bale(f"Large {media_type} from {username or chat.title}: {media_size/(1024*1024):.1f} MB")
media_path = await download_media(event, temp_dir)
if media_path:
await handle_large_document(event, media_path, final_caption, temp_dir)
finally:
cleanup_temp_dir(temp_dir)
return
# Small media
temp_dir = create_temp_dir()
try:
media_path = await download_media(event, temp_dir)
if media_path:
first_caption, _ = await send_text_or_caption(final_caption, parse_mode="HTML", is_caption=True)
success = await send_media_to_bale(media_path, first_caption, media_type, parse_mode="HTML")
if not success:
await send_log_to_bale(f"Direct send failed for {media_type}, trying as document...")
zip_path = os.path.join(temp_dir, "fallback.zip")
await zip_file(media_path, zip_path)
success = await send_media_to_bale(zip_path, first_caption, "document", parse_mode=None)
if not success:
await send_log_to_bale(f"Fallback also failed for {media_type}")
finally:
cleanup_temp_dir(temp_dir)
except Exception as e:
name = username if username else getattr(chat, 'title', '?') if chat else '?'
await send_log_to_bale(f"Handler Error (@{name}): {e}")
# --- 9. SERVER LIFESPAN & HEALTH STATUS ---
TELEGRAM_STATUS = "Stopped"
TELEGRAM_ERROR = None
@asynccontextmanager
async def lifespan(app: FastAPI):
global TELEGRAM_STATUS, TELEGRAM_ERROR
TELEGRAM_STATUS = "Connecting..."
await send_log_to_bale("Bot starting...")
print("Connecting to Telegram...")
try:
await tg_client.connect()
if not await tg_client.is_user_authorized():
TELEGRAM_STATUS = "Unauthorized"
msg = "❌ TELEGRAM NOT AUTHORIZED!"
print(msg)
await send_log_to_bale(msg)
else:
TELEGRAM_STATUS = "Connected"
msg = "✅ Telegram Connected!"
print(msg)
await send_log_to_bale(msg)
asyncio.create_task(tg_client.run_until_disconnected())
except AuthKeyDuplicatedError:
TELEGRAM_STATUS = "Session Revoked (AuthKeyDuplicatedError)"
TELEGRAM_ERROR = (
"This session key was used under two different IPs simultaneously "
"(e.g., local tests and Hugging Face active at the same time, "
"or Hugging Face running a zero-downtime rolling update with overlap). "
"Telegram has permanently deactivated this session. You must generate a new session string."
)
print(f"❌ {TELEGRAM_STATUS}: {TELEGRAM_ERROR}")
await send_log_to_bale(f"❌ Bot startup failed: {TELEGRAM_STATUS}")
except Exception as e:
TELEGRAM_STATUS = f"Error: {type(e).__name__}"
TELEGRAM_ERROR = str(e)
print(f"❌ Unexpected connection error: {e}")
await send_log_to_bale(f"❌ Bot startup failed: {e}")
yield
TELEGRAM_STATUS = "Disconnected"
try:
await tg_client.disconnect()
except Exception:
pass
await send_log_to_bale("Bot disconnected")
app = FastAPI(lifespan=lifespan)
@app.get("/")
async def health():
res = {
"status": "running",
"telegram_connection": TELEGRAM_STATUS,
"monitoring": TARGET_CHANNELS
}
if TELEGRAM_ERROR:
res["error_details"] = TELEGRAM_ERROR
return res
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=7860)