document-parser / .github /workflows /orchestrator-call.yml
segovia
Move the scanning to a job
db9f03c
Raw History Blame Contribute Delete
4.53 kB
name: Call Orchestrator Endpoint
on:
workflow_dispatch:
inputs:
model:
description: OpenAI model name
required: false
default: gpt-4o
include_subfolders:
description: Include subfolders in Drive scan
required: false
default: "true"
send_to_sheet:
description: Send rows to Google Sheets
required: false
default: "false"
schedule:
- cron: '*/5 * * * *'
jobs:
call-orchestrator:
runs-on: ubuntu-latest
timeout-minutes: 1
env:
ORCHESTRATOR_BASE_URL: ${{ secrets.ORCHESTRATOR_BASE_URL }}
ORCHESTRATOR_API_KEY: ${{ secrets.ORCHESTRATOR_API_KEY }}
MODEL_INPUT: ${{ github.event.inputs.model || 'gpt-4o' }}
INCLUDE_SUBFOLDERS_INPUT: ${{ github.event.inputs.include_subfolders || 'true' }}
SEND_TO_SHEET_INPUT: ${{ github.event.inputs.send_to_sheet || 'false' }}
steps:
- name: Validate required secrets
run: |
set -euo pipefail
if [ -z "${ORCHESTRATOR_BASE_URL}" ]; then
echo "Missing secret: ORCHESTRATOR_BASE_URL"
exit 1
fi
if [ -z "${ORCHESTRATOR_API_KEY}" ]; then
echo "Missing secret: ORCHESTRATOR_API_KEY"
exit 1
fi
- name: Start orchestrator job (POST)
id: start
shell: bash
run: |
set -euo pipefail
python - <<'PY'
import json
import os
import urllib.request
base_url = os.environ["ORCHESTRATOR_BASE_URL"].rstrip("/")
payload = {
"data": [
os.environ["MODEL_INPUT"],
os.environ["INCLUDE_SUBFOLDERS_INPUT"].lower() == "true",
os.environ["SEND_TO_SHEET_INPUT"].lower() == "true",
os.environ["ORCHESTRATOR_API_KEY"],
]
}
req = urllib.request.Request(
url=f"{base_url}/gradio_api/call/orchestrate_drive_documents",
data=json.dumps(payload).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(req, timeout=60) as resp:
body = resp.read().decode("utf-8")
parsed = json.loads(body)
event_id = parsed.get("event_id")
if not event_id:
raise SystemExit(f"POST succeeded but event_id missing. Response: {body}")
with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as fh:
fh.write(f"event_id={event_id}\n")
print("Orchestrator job started.")
PY
- name: Read orchestrator stream result
shell: bash
run: |
set -euo pipefail
STREAM_URL="${ORCHESTRATOR_BASE_URL%/}/gradio_api/call/orchestrate_drive_documents/${{ steps.start.outputs.event_id }}"
curl -sS --max-time 900 -N "$STREAM_URL" > stream.txt
echo "Stream captured in stream.txt"
id: stream
- name: Validate orchestrator response
shell: bash
run: |
set -euo pipefail
python - <<'PY'
import json
with open("stream.txt", "r", encoding="utf-8") as fh:
lines = [line.rstrip("\n") for line in fh]
data_lines = [line[6:] for line in lines if line.startswith("data: ")]
non_null = [line for line in data_lines if line and line != "null"]
if not non_null:
raise SystemExit("No non-null data payload found in stream.")
payload = json.loads(non_null[-1])
if not isinstance(payload, list) or not payload:
raise SystemExit(f"Unexpected payload shape: {payload}")
result = payload[0]
ok = bool(result.get("ok")) if isinstance(result, dict) else False
if not ok:
raise SystemExit(f"Orchestrator failed: {json.dumps(result, ensure_ascii=False)}")
summary = {
"ok": result.get("ok"),
"scanned": result.get("scanned"),
"to_process": result.get("to_process"),
"skipped": result.get("skipped"),
"parsed": result.get("parsed"),
"modified": result.get("modified"),
"sent": result.get("sent"),
"errors_count": len(result.get("errors", [])) if isinstance(result.get("errors"), list) else None,
}
print(json.dumps(summary, ensure_ascii=False))
PY