""" Manual send — the Phase 1 slice of docs/blueprint/blueprint.md "Sending pipeline". Every send is tracked per-recipient (SRS FR-18), never only per-broadcast — this is also what billing (per-recipient, ADR-0005) and rate limiting (per-send, not per-batch) both read from. Scheduled sends and bounce/DSN detection (SRS FR-19/20) are Phase 2 — this module only covers a synchronous, immediate, manual send (a "Campaign" in the frontend's terms — see frontend/src/pages/Campaigns.tsx — not yet an automation), which is enough to prove per-recipient status tracking end-to-end. Which mailer actually does the sending (mock vs. real SMTP) is decided by backend/mailer_client.py's REAL_SEND_ENABLED flag, not by anything in this file. Recipients come from backend/databases.py (a database's rows), not a flat contact list — each row's own column values are substituted into {Placeholder} tokens in the subject/body before that recipient's copy is sent, via substitute_placeholders. """ from __future__ import annotations import base64 import threading import time import uuid from datetime import datetime, timezone from backend import brands, certificates, databases, delivery, email_parts, firestore_client as db, mailboxes, mailer_client, messages, orgs, rate_limiter, settings, templating, unsubscribe from backend.mock_mailer import AuthError, ConnectionFailure def _now() -> str: return datetime.now(timezone.utc).isoformat() def _send_path(org_id: str, send_id: str) -> str: return f"orgs/{org_id}/sends/{send_id}" def substitute_placeholders(template: str, row: dict, columns: list[dict] | None = None, html: bool = False) -> str: """Renders one recipient's copy of `template` (see backend/templating.py): {ColumnName} is replaced by that row's value (blank if the row has none — a gap for one row, not an error), {Name|fallback} covers blanks, list columns become bullets or "A, B and C", and {if ...}{else}{/if} blocks pick content by the row's values. With html=True, row values are escaped so they cannot become markup.""" return templating.render(template, row, columns, html=html) def _template_problems(subject: str, html_body: str, columns: list[dict]) -> list[str]: # Checked separately: a block opened in the subject must not be "closed" by the body. return list(dict.fromkeys( templating.validate_template(subject, columns) + templating.validate_template(html_body, columns) )) def preview_render( org_id: str, database_id: str, subject: str, html_body: str, segment: list[dict] | None = None, index: int = 0, email_column: str | None = None, ) -> dict: """What recipient number `index` (of those the segment matches) would receive, using the same renderer as the real send, plus any template problems that would block a send.""" database = databases.get_database(org_id, database_id) columns = database["columns"] problems = _template_problems(subject, html_body, columns) rows = databases.list_rows(org_id, database_id) if segment: rows = databases.apply_segment(rows, columns, segment) email_column = _email_column(database, email_column) if email_column: rows = [r for r in rows if r.get(email_column)] if problems or not rows: return {"subject": subject, "html_body": html_body, "problems": problems, "recipient": None} row = rows[index % len(rows)] return { "subject": substitute_placeholders(subject, row, columns), "html_body": substitute_placeholders(html_body, row, columns, html=True), "problems": [], "recipient": row.get(email_column) if email_column else None, } def _recipient_rows( org_id: str, database_id: str, database: dict, segment: list[dict] | None, email_column: str | None = None, ) -> list[dict]: rows = databases.list_rows(org_id, database_id) if segment: rows = databases.apply_segment(rows, database["columns"], segment) email_column = _email_column(database, email_column) return [r for r in rows if email_column and r.get(email_column)] def preview_certificate( org_id: str, database_id: str, certificate: dict, segment: list[dict] | None = None, index: int = 0, email_column: str | None = None, ) -> dict: """The certificate recipient number `index` would get, as a scaled-down PNG, so the composer shows the real generated file rather than an approximation.""" spec = certificates.parse_spec(certificate) database = databases.get_database(org_id, database_id) email_column = _email_column(database, email_column) rows = _recipient_rows(org_id, database_id, database, segment, email_column) if rows: row = rows[index % len(rows)] text = certificates.text_for(spec, row, database["columns"]) recipient = row.get(email_column) else: text, recipient = spec.text, None return {"recipient": recipient, "text": text, "image_b64": base64.b64encode(certificates.preview_png(spec, text)).decode()} def _email_column(database: dict, chosen: str | None = None) -> str | None: """The column whose addresses are sent to: `chosen` when given (a list can hold more than one email column, such as a guest's address and the staff member who checked them in), otherwise the first email column.""" emails = [col["name"] for col in database["columns"] if col["type"] == "email"] if chosen: if chosen not in emails: raise ValueError(f"{chosen!r} is not an email column in this list") return chosen return emails[0] if emails else None ACTIVE = ("queued", "sending") _locks: dict[tuple[str, str], threading.Lock] = {} def _parse_when(value: str) -> datetime: try: when = datetime.fromisoformat(value.replace("Z", "+00:00")) except (ValueError, AttributeError) as exc: raise ValueError("That date and time could not be read") from exc return when if when.tzinfo else when.replace(tzinfo=timezone.utc) def clean_window(window: dict | None, default_timezone: str = "UTC") -> dict | None: """A sending window: only send between `start` and `end` (HH:MM) on the given weekdays (0 is Monday), in the given time zone. None means no restriction.""" if not window: return None from zoneinfo import ZoneInfo try: start, end = str(window["start"]), str(window["end"]) for value in (start, end): datetime.strptime(value, "%H:%M") days = sorted({int(d) for d in window.get("days", range(7))}) zone = str(window.get("timezone") or default_timezone or "UTC") ZoneInfo(zone) except Exception as exc: raise ValueError("The sending window is not valid") from exc if start >= end: raise ValueError("The sending window must end after it starts") if not days or any(d < 0 or d > 6 for d in days): raise ValueError("Choose at least one day for the sending window") return {"start": start, "end": end, "days": days, "timezone": zone} def _in_window(window: dict | None, now: datetime) -> bool: if not window: return True from zoneinfo import ZoneInfo local = now.astimezone(ZoneInfo(window["timezone"])) return local.weekday() in window["days"] and window["start"] <= local.strftime("%H:%M") < window["end"] def _payload_path(org_id: str, send_id: str) -> str: return f"orgs/{org_id}/send_payloads/{send_id}" def _active_path(org_id: str, send_id: str) -> str: return f"active_sends/{org_id}__{send_id}" def _set_active(org_id: str, send_id: str, active: bool) -> None: if active: db.set_doc(_active_path(org_id, send_id), {"org_id": org_id, "send_id": send_id}) else: db.delete_doc(_active_path(org_id, send_id)) def send_campaign( org_id: str, actor_uid: str, subject: str, html_body: str, mailbox_connection_id: str, database_id: str, segment: list[dict] | None = None, attachments: list[dict] | None = None, certificate: dict | None = None, email_column: str | None = None, mode: str | None = None, design: dict | None = None, scheduled_at: str | None = None, window: dict | None = None, ) -> str: """Queues the campaign and sends what can go now: `queue_campaign` followed by `process_send`. A request handler calls queue_campaign and hands process_send to a background task so the page does not wait; this does both in one go.""" send_id = queue_campaign( org_id, actor_uid, subject, html_body, mailbox_connection_id, database_id, segment, attachments, certificate, email_column, mode, design, scheduled_at, window, ) process_send(org_id, send_id) return send_id def queue_campaign( org_id: str, actor_uid: str, subject: str, html_body: str, mailbox_connection_id: str, database_id: str, segment: list[dict] | None = None, attachments: list[dict] | None = None, certificate: dict | None = None, email_column: str | None = None, mode: str | None = None, design: dict | None = None, scheduled_at: str | None = None, window: dict | None = None, only_row_ids: list[str] | None = None, automated: bool = False, ) -> str: """Saves a campaign to send to every row in `database_id` that has a value in its email-typed column, substituting that row's own values into {Placeholder} tokens first. `segment`, if given, is a list of {field, op, value} conditions (ANDed, see databases.apply_segment) that narrows the recipient list before the email-column check — the same predicate the Campaigns page's live "N recipients" preview runs, so what you previewed is exactly who gets sent to. Raises ValueError if the mailbox isn't verified or the database has no column classified as "email" (set one via databases.update_column_type first). `attachments` ([{filename, content_type, data_b64}]) go to every recipient as they are. `certificate` (see backend/certificates.parse_spec) adds one generated file per recipient, with that recipient's own values drawn onto the image. Images uploaded into the message travel as inline parts (backend/email_parts.extract_inline_images). `scheduled_at` (an ISO date-time) holds the send until then; `window` limits sending to certain hours and days (see clean_window). The send is carried out by process_send, which keeps to the mailbox's daily limit and the organisation's hourly cap by sending what fits and continuing when room opens up. `only_row_ids` narrows the send to those rows (a welcome email goes to the one new person). `automated` is for mail the system sends on a member's earlier say-so, such as a form's welcome email: no one is signed in then, so the permission check is skipped.""" if not automated: orgs.require_permission(org_id, actor_uid, "send") connection = mailboxes.get_mailbox(org_id, mailbox_connection_id) if connection["status"] != "verified": raise ValueError( f"Mailbox connection {mailbox_connection_id!r} is {connection['status']!r}, " f"not verified — it can't be used to send" ) database = databases.get_database(org_id, database_id) email_column = _email_column(database, email_column) if email_column is None: raise ValueError( f"Database {database_id!r} has no column classified as an email address — " f"set one via Manage columns before sending" ) problems = _template_problems(subject, html_body, database["columns"]) if problems: raise ValueError("The message cannot be sent: " + "; ".join(problems)) # Checked now so a bad attachment or image fails the request, not the background send. email_parts.decode_attachments(attachments) spec = certificates.parse_spec(certificate) if certificate else None email_parts.extract_inline_images(html_body) if scheduled_at: scheduled_at = _parse_when(scheduled_at).isoformat() window = clean_window(window, (orgs.get_org(org_id).get("timezone") or "UTC")) rows = _recipient_rows(org_id, database_id, database, segment, email_column) if only_row_ids is not None: rows = [r for r in rows if r["row_id"] in only_row_ids] brand = brands.brand_for_mailbox(org_id, mailbox_connection_id) or {} # The HTML as written (inline images still in place), so the details page shows it whole. message_id = messages.create_message( org_id, actor_uid, subject, html_body if len(html_body) <= 900_000 else email_parts.extract_inline_images(html_body)[0], mode, design, automated=automated, ) return _enqueue( org_id, actor_uid, message_id=message_id, subject=subject, html_body=html_body, mailbox_connection_id=mailbox_connection_id, from_email=connection.get("email"), brand_id=brand.get("brand_id"), database_id=database_id, email_column=email_column, segment=segment or [], row_ids=[r["row_id"] for r in rows], attachments=attachments, certificate=certificate if spec else None, attachment_names=[a.get("filename") for a in attachments or []], scheduled_at=scheduled_at, window=window, ) def _enqueue(org_id: str, actor_uid: str, *, row_ids: list[str], html_body: str, attachments: list[dict] | None, certificate: dict | None, **fields) -> str: send_id = uuid.uuid4().hex created = _now() db.set_doc(_payload_path(org_id, send_id), {"html_body": html_body, "attachments": attachments or [], "certificate": certificate}) db.set_doc( _send_path(org_id, send_id), { **fields, "certificate": certificate is not None, "created_by": actor_uid, "created_at": created, "sent_at": fields.get("scheduled_at") or created, "status": "queued", "note": "", "pending": row_ids, "total": len(row_ids), "per_recipient": {}, }, ) _set_active(org_id, send_id, True) return send_id def process_send(org_id: str, send_id: str, now: datetime | None = None, budget_seconds: float | None = None) -> dict: """Sends what can go right now for one queued send, then returns its record. Safe to call at any time and from several places: it does nothing unless the send is due, inside its window, within the mailbox's daily limit and the organisation's hourly cap, and only one caller works on a send at once. The rest stays queued for the next call.""" lock = _locks.setdefault((org_id, send_id), threading.Lock()) if not lock.acquire(blocking=False): return get_send(org_id, send_id) try: return _process(org_id, send_id, now or datetime.now(timezone.utc), budget_seconds) finally: lock.release() def _process(org_id: str, send_id: str, now: datetime, budget_seconds: float | None) -> dict: send = get_send(org_id, send_id) if send.get("status") not in ACTIVE: return send if send.get("scheduled_at") and _parse_when(send["scheduled_at"]) > now: return send stamp = now.isoformat() pending: list[str] = list(send.get("pending", [])) if not pending: return _finish(org_id, send_id, send) if not _in_window(send.get("window"), now): return _note(org_id, send_id, send, "Waiting for the sending window to open.") try: connection = mailboxes.get_mailbox(org_id, send["mailbox_connection_id"]) database = databases.get_database(org_id, send["database_id"]) except orgs.NotFoundError: return _stop(org_id, send_id, send, "paused", "The mailbox or the list for this send was removed. Resume after restoring it, or cancel.") if connection["status"] != "verified": return _stop(org_id, send_id, send, "paused", "The mailbox is not working. Fix it in Settings, then resume.") box = mailboxes.mailbox_settings(connection) room = len(pending) reason = "" if box["daily_limit"]: left = max(box["daily_limit"] - sent_today(org_id, send["mailbox_connection_id"], now), 0) if left < room: room, reason = left, f"The mailbox's daily limit of {box['daily_limit']} is reached. Sending continues tomorrow." hourly = rate_limiter.remaining_capacity(org_id) if hourly < room: room, reason = hourly, "Waiting for the hourly sending limit. Sending continues as room opens up." if room <= 0: return _note(org_id, send_id, send, reason) rate_limiter.check_and_consume(org_id, room) payload = db.get_doc(_payload_path(org_id, send_id)) or {} shared_files = email_parts.decode_attachments(payload.get("attachments")) spec = certificates.parse_spec(payload["certificate"]) if payload.get("certificate") else None html_body, inline_images = email_parts.extract_inline_images(payload.get("html_body", "")) rows = {r["row_id"]: r for r in databases.list_rows(org_id, send["database_id"])} columns = database["columns"] email_column = send.get("email_column") try: email_column = _email_column(database, email_column) except ValueError: # renamed or retyped since the send was queued email_column = _email_column(database) blocked = unsubscribe.suppressed(org_id) brand = brands.brand_for_mailbox(org_id, send["mailbox_connection_id"]) or {} prefs = settings.email_settings(org_id) credentials = mailboxes.get_decrypted_credentials(org_id, send["mailbox_connection_id"]) mailer = mailer_client.get_mailer(connection["kind"], credentials) if hasattr(mailer, "configure"): mailer.configure( box["from_name"] or brand.get("from_name") or prefs["from_name"], box["reply_to"] or brand.get("reply_to") or prefs["reply_to"], prefs["save_sent_copy"] if box["save_sent_copy"] is None else box["save_sent_copy"], ) per_recipient = dict(send.get("per_recipient", {})) started = time.monotonic() sent_now = 0 status = "sending" save_every = 10 def save(final: bool = False) -> None: # The status is only written at the end, so a pause pressed meanwhile is not overwritten. update = {"pending": pending, "per_recipient": per_recipient, "note": reason} if final: update["status"] = status db.update_doc(_send_path(org_id, send_id), update) while pending and sent_now < room: row_id = pending[0] row = rows.get(row_id) address = row.get(email_column) if row and email_column else None if row is None or not address: per_recipient[row_id] = {"status": "skipped", "error": "No longer in the list", "at": stamp} elif address.strip().lower() in blocked: per_recipient[row_id] = {"status": "skipped", "error": "Unsubscribed", "at": stamp} else: if sent_now and prefs["pause_seconds"]: time.sleep(prefs["pause_seconds"]) files = list(shared_files) try: if spec is not None: files.append(certificates.attachment_for(spec, row, columns)) reference = mailer.send( to_addr=address, subject=substitute_placeholders(send["subject"], row, columns), body=substitute_placeholders(html_body, row, columns, html=True) + unsubscribe.footer_html(org_id, address, send.get("from_email") or ""), attachments=files, inline_images=inline_images, headers=unsubscribe.headers(org_id, address, send.get("from_email") or ""), ) # "sent" means the mailbox's own server accepted the message. Whether it then # reached the inbox is only known from bounces (Phase 2); the Message-ID lets the # hosting provider trace a message in their logs. per_recipient[row_id] = {"status": "sent", "error": None, "at": stamp} if isinstance(reference, str): per_recipient[row_id]["reference"] = reference except (AuthError, ConnectionFailure, OSError) as exc: per_recipient[row_id] = {"status": "failed", "error": str(exc), "at": stamp} sent_now += 1 pending.pop(0) if len(per_recipient) % save_every == 0: save() # A pause or cancel pressed while this runs takes effect within a few emails. if (db.get_doc(_send_path(org_id, send_id)) or {}).get("status") not in ACTIVE: break if budget_seconds is not None and time.monotonic() - started > budget_seconds: reason = "Sending continues on the next run." break if hasattr(mailer, "close"): mailer.close() current = (db.get_doc(_send_path(org_id, send_id)) or {}).get("status") if current not in ACTIVE: # paused or cancelled while sending: keep that, save the progress status = current elif not pending: status = "done" reason = "" elif not reason and sent_now >= room: reason = "Sending continues shortly." save(final=True) if sent_now: delivery.watch(org_id, send_id) # look for delays, bounces and replies over the next days if status not in ACTIVE: _set_active(org_id, send_id, False) return get_send(org_id, send_id) def _note(org_id: str, send_id: str, send: dict, note: str) -> dict: if send.get("note") != note: db.update_doc(_send_path(org_id, send_id), {"note": note}) return send | {"note": note} def _stop(org_id: str, send_id: str, send: dict, status: str, note: str) -> dict: db.update_doc(_send_path(org_id, send_id), {"status": status, "note": note}) _set_active(org_id, send_id, False) return send | {"status": status, "note": note} def _finish(org_id: str, send_id: str, send: dict) -> dict: db.update_doc(_send_path(org_id, send_id), {"status": "done", "note": ""}) _set_active(org_id, send_id, False) return send | {"status": "done", "note": ""} def run_due(now: datetime | None = None, budget_seconds: float = 240) -> dict: """Carries out every active send that is due. A timer calls this every minute or so (Cloud Scheduler in production, a small thread when running locally); each run does a bounded amount of work and exits.""" started = time.monotonic() checked = 0 for doc in list(db.query_prefix("active_sends/").values()): left = budget_seconds - (time.monotonic() - started) if left <= 0: break try: process_send(doc["org_id"], doc["send_id"], now, left) except orgs.NotFoundError: _set_active(doc["org_id"], doc["send_id"], False) checked += 1 return {"checked": checked} def control_send(org_id: str, actor_uid: str, send_id: str, action: str) -> dict: """Pause, resume or cancel a send that has not finished.""" orgs.require_permission(org_id, actor_uid, "send") send = get_send(org_id, send_id) status = send.get("status", "done") if action == "pause" and status in ACTIVE: update = {"status": "paused", "note": "Paused."} elif action == "resume" and status == "paused": update = {"status": "queued", "note": ""} elif action == "cancel" and status in ACTIVE + ("paused",): update = {"status": "cancelled", "note": "Cancelled."} else: raise ValueError(f"This send is {status}, so it cannot be {action}d" if action != "cancel" else f"This send is {status}, so it cannot be cancelled") db.update_doc(_send_path(org_id, send_id), update) _set_active(org_id, send_id, update["status"] == "queued") return get_send(org_id, send_id) def sent_today(org_id: str, mailbox_connection_id: str, now: datetime | None = None) -> int: """Emails this mailbox's server accepted since midnight UTC.""" today = (now or datetime.now(timezone.utc)).astimezone(timezone.utc).date().isoformat() total = 0 for send in db.query_prefix(f"orgs/{org_id}/sends/").values(): if send.get("mailbox_connection_id") != mailbox_connection_id: continue for r in send["per_recipient"].values(): if r["status"] == "sent" and str(r.get("at") or send.get("sent_at", "")).startswith(today): total += 1 return total def get_send(org_id: str, send_id: str) -> dict: doc = db.get_doc(_send_path(org_id, send_id)) if doc is None: raise orgs.NotFoundError(f"No send {send_id!r} in org {org_id!r}") return doc def list_sends(org_id: str, actor_uid: str, limit: int = 50) -> list[dict]: """Recent sends, newest first, with counts instead of the per-recipient map.""" orgs.require_permission(org_id, actor_uid) prefix = f"orgs/{org_id}/sends/" out = [] for path, doc in db.query_prefix(prefix).items(): statuses = [r.get("status") for r in (doc.get("per_recipient") or {}).values()] subject = doc.get("subject") if subject is None and doc.get("message_id"): try: subject = messages.get_message(org_id, doc["message_id"])["subject"] except orgs.NotFoundError: subject = "" out.append({ "send_id": path[len(prefix):], "subject": subject or "", "database_id": doc.get("database_id"), "mailbox_connection_id": doc.get("mailbox_connection_id"), "sent_at": doc.get("sent_at"), "sent": statuses.count("sent"), "failed": statuses.count("failed"), "skipped": statuses.count("skipped"), "pending": len(doc.get("pending") or []), "status": doc.get("status", "done"), "scheduled_at": doc.get("scheduled_at"), "note": doc.get("note", ""), "attachment_names": doc.get("attachment_names", []), "certificate": bool(doc.get("certificate")), }) out.sort(key=lambda s: s["sent_at"] or "", reverse=True) return out[:limit] def get_send_details(org_id: str, actor_uid: str, send_id: str) -> dict: """Everything about one send for its details page: who it went to (addresses looked up from the list now, since the send record keeps row ids only), how each went, where it came from, and the email itself so it can be read again or saved as a preset.""" orgs.require_permission(org_id, actor_uid) send = get_send(org_id, send_id) try: message = messages.get_message(org_id, send["message_id"]) except (orgs.NotFoundError, KeyError): message = {} database = None try: database = databases.get_database(org_id, send["database_id"]) except orgs.NotFoundError: pass addresses: dict[str, str] = {} column = send.get("email_column") if database is not None: try: column = _email_column(database, column) except ValueError: # the column was renamed or retyped since column = _email_column(database) if column: addresses = {r["row_id"]: str(r.get(column) or "") for r in databases.list_rows(org_id, send["database_id"])} try: mailbox_email = mailboxes.get_mailbox(org_id, send["mailbox_connection_id"]).get("email") except (orgs.NotFoundError, KeyError): mailbox_email = send.get("from_email") try: tracking = bool(mailboxes.get_mailbox(org_id, send["mailbox_connection_id"]).get("has_imap")) except (orgs.NotFoundError, KeyError): tracking = False recipients = [ { "row_id": row_id, "address": addresses.get(row_id) or None, "status": r.get("status"), "error": r.get("error"), "reference": r.get("reference"), "delivery": delivery.shown_state(r, tracking=tracking), } for row_id, r in (send.get("per_recipient") or {}).items() ] waiting = "queued" if send.get("status") in ACTIVE + ("paused",) else "not sent" recipients += [ {"row_id": row_id, "address": addresses.get(row_id) or None, "status": waiting, "error": None, "reference": None, "delivery": delivery.shown_state({"status": waiting})} for row_id in send.get("pending") or [] ] recipients.sort(key=lambda r: (r["status"] != "failed", (r["address"] or "~").lower())) return { "send_id": send_id, "subject": send.get("subject") or message.get("subject") or "", "sent_at": send.get("sent_at"), "status": send.get("status", "done"), "note": send.get("note", ""), "scheduled_at": send.get("scheduled_at"), "window": send.get("window"), "from_email": mailbox_email, "database_id": send.get("database_id"), "database_name": database.get("name") if database else None, "email_column": column, "segment": send.get("segment") or [], "attachment_names": send.get("attachment_names", []), "certificate": bool(send.get("certificate")), "recipients": recipients, "html_body": message.get("html_body", ""), "mode": message.get("mode") or "html", "design": message.get("design"), }