Spaces:
Sleeping
Sleeping
Donne120
Claude Opus 5
Fix generation outage: retired Groq model and anthropic SDK signature
107a904 Download bedrock_fallback.py from StudentCompanion/alu-chatbot: direct link, hf CLI and curl.
- Browser
- Download file 11.5 kB
-
https://huggingface.co/spaces/StudentCompanion/alu-chatbot/resolve/main/bedrock_fallback.py
- Command line
-
hf download hf://spaces/StudentCompanion/alu-chatbot/bedrock_fallback.py
-
curl -L -o bedrock_fallback.py https://huggingface.co/spaces/StudentCompanion/alu-chatbot/resolve/main/bedrock_fallback.py
11.5 kB
| """ | |
| Amazon Bedrock fallback β Claude, served from inside AWS. | |
| Why this exists alongside claude_engine (which calls Anthropic directly): | |
| - No API key to manage. On AWS the task's IAM role supplies credentials, so | |
| there is no secret to rotate, leak, or let expire. Every outage this | |
| project has had at the generation step was an expired key; this removes | |
| that failure mode entirely for the AWS path. | |
| - Requests never leave AWS. Useful for the data-residency question | |
| institutions ask. | |
| - Billed to the AWS account, so it draws on AWS credit rather than a | |
| separate Anthropic invoice. | |
| Position in the chain: this sits AFTER the free providers on purpose. Groq's | |
| free tier costs nothing, so we spend credit only when the free paths are | |
| unavailable. Set BEDROCK_PRIORITY=high to move it ahead of them once the | |
| credit is there to spend or the free tiers stop being reliable. | |
| Configuration: | |
| BEDROCK_MODEL_ID -> optional, defaults to the Claude Haiku inference | |
| profile. Cross-region profiles ("us.anthropic...") | |
| are the ones that work in most accounts; a bare | |
| "anthropic.claude-..." id fails with a validation | |
| error in regions that require a profile. | |
| BEDROCK_REGION -> optional, falls back to AWS_REGION then us-east-1. | |
| BEDROCK_PRIORITY -> "high" to try Bedrock before the free fallbacks. | |
| Credentials come from the standard boto3 chain: task role on ECS/App Runner, | |
| or AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY anywhere else (which is how this | |
| can be tested from the Hugging Face Space before migrating). | |
| IMPORTANT: Bedrock model access is opt-in per model per region. Until it is | |
| granted in the Bedrock console, calls fail with AccessDeniedException even | |
| though the credentials are valid. `probe()` reports that distinctly rather | |
| than as a generic failure, because the two need very different fixes. | |
| """ | |
| import os | |
| import json | |
| import logging | |
| logger = logging.getLogger(__name__) | |
| # Cross-region inference profile. Cheap and fast, which is what a fallback | |
| # should be β the expensive model is not the one you want on the path that | |
| # only runs when something else already broke. | |
| DEFAULT_BEDROCK_MODEL = "us.anthropic.claude-haiku-4-5-20251001-v1:0" | |
| MAX_TOKENS = 1024 | |
| TEMPERATURE = 0.4 | |
| HISTORY_TURNS = 8 | |
| class BedrockFallback: | |
| """Grounded answer generation via Bedrock. | |
| Same interface as NvidiaFallback and the Groq fallback β | |
| `generate_response(query, alu_context, history)` returning a string or | |
| None β so app.py can slot it into the chain without special-casing. | |
| """ | |
| SYSTEM_PROMPT = """You are the ALU Student Companion, an AI assistant for African Leadership University (ALU). | |
| # Most important rule | |
| Answer ONLY from the context provided below. If the context does not contain a fact, do not invent it. Specifically NEVER invent course codes, credit hours, module names, tuition amounts, fees, scholarship values, faculty names/titles, deadlines, term dates, or statistics. When the context doesn't have what's needed, say so plainly and point to the official source β students would rather hear "I don't have that, here's where to find it" than a confident wrong answer. | |
| # Style | |
| - Lead with a one-sentence direct answer, then detail. | |
| - Use markdown sparingly; bullet lists only for genuine lists. | |
| - No emoji. No "Is there anything else..." filler. | |
| - When a Source in the context has a url=..., cite it as a markdown link the first time you use that fact. | |
| - Do not show your reasoning or thinking steps; give only the final answer. | |
| # Facts you may always state without context | |
| - ALU was founded by Fred Swaniker in 2015, part of the African Leadership Group. | |
| - Campuses in Kigali, Rwanda and Pamplemousses, Mauritius. | |
| - Main site https://www.alueducation.com ; support https://support.alueducation.com | |
| - Student emails end @alustudent.com ; staff/public @alueducation.com | |
| Everything else β programmes, fees, dates, people β must come from the context. If it is not there, say you don't know and point to support.alueducation.com.""" | |
| def __init__(self): | |
| self.model = os.getenv("BEDROCK_MODEL_ID", DEFAULT_BEDROCK_MODEL) | |
| self.region = ( | |
| os.getenv("BEDROCK_REGION") | |
| or os.getenv("AWS_REGION") | |
| or "us-east-1" | |
| ) | |
| self.priority = os.getenv("BEDROCK_PRIORITY", "low").strip().lower() | |
| self.client = None | |
| self.enabled = False | |
| # Why it's off, in words, so /api/services/status can say so rather | |
| # than just reporting False. | |
| self.disabled_reason = None | |
| try: | |
| import boto3 | |
| from botocore.exceptions import NoCredentialsError # noqa: F401 | |
| session = boto3.session.Session() | |
| # No credentials at all is the common case off-AWS. Detect it here | |
| # rather than on the first student question. | |
| if session.get_credentials() is None: | |
| self.disabled_reason = "no AWS credentials available" | |
| logger.info("[INFO] Bedrock fallback disabled (no AWS credentials)") | |
| return | |
| self.client = session.client("bedrock-runtime", region_name=self.region) | |
| self.enabled = True | |
| logger.info( | |
| "[OK] Bedrock fallback enabled (model=%s region=%s priority=%s)", | |
| self.model, self.region, self.priority, | |
| ) | |
| except ImportError: | |
| self.disabled_reason = "boto3 not installed" | |
| logger.warning("[!] boto3 not installed β Bedrock fallback unavailable") | |
| except Exception as e: | |
| self.disabled_reason = f"client init failed: {e}" | |
| logger.warning("[!] Bedrock client init failed: %s", e) | |
| def prefer_before_free_tiers(self) -> bool: | |
| """True when this should be tried ahead of NVIDIA/Groq.""" | |
| return self.enabled and self.priority == "high" | |
| def _build_body(self, query: str, alu_context: str, history) -> dict: | |
| messages = [] | |
| for entry in (history or [])[-HISTORY_TURNS:]: | |
| role = entry.get("role") | |
| content = entry.get("content", "") | |
| if role in ("user", "assistant") and content: | |
| messages.append({"role": role, "content": [{"type": "text", "text": content}]}) | |
| if alu_context: | |
| user_message = ( | |
| "Use the following context from the ALU knowledge base to " | |
| "answer the question. If the context does not contain the " | |
| "answer, say so honestly and point to the official source.\n\n" | |
| f"=== CONTEXT ===\n{alu_context}\n=== END CONTEXT ===\n\n" | |
| f"Question: {query}" | |
| ) | |
| else: | |
| user_message = query | |
| messages.append({"role": "user", "content": [{"type": "text", "text": user_message}]}) | |
| # Bedrock requires the first message to be from the user. A history | |
| # that begins with an assistant turn would be rejected, so drop any | |
| # leading assistant turns rather than let the call fail. | |
| while messages and messages[0]["role"] != "user": | |
| messages.pop(0) | |
| return { | |
| "anthropic_version": "bedrock-2023-05-31", | |
| "max_tokens": MAX_TOKENS, | |
| "temperature": TEMPERATURE, | |
| "system": self.SYSTEM_PROMPT, | |
| "messages": messages, | |
| } | |
| def generate_response(self, query: str, alu_context: str = "", history=None): | |
| """Return an answer, or None so the caller falls through to the next | |
| provider. Never raises β a fallback that can crash the chain is worse | |
| than one that declines.""" | |
| if not self.enabled or not self.client: | |
| return None | |
| try: | |
| response = self.client.invoke_model( | |
| modelId=self.model, | |
| body=json.dumps(self._build_body(query, alu_context, history)), | |
| contentType="application/json", | |
| accept="application/json", | |
| ) | |
| payload = json.loads(response["body"].read()) | |
| parts = [ | |
| block.get("text", "") | |
| for block in payload.get("content", []) | |
| if block.get("type") == "text" | |
| ] | |
| answer = "".join(parts).strip() | |
| logger.info("[OK] Bedrock response generated (%d chars)", len(answer)) | |
| return answer or None | |
| except Exception as e: | |
| logger.error("[FAIL] Bedrock API error: %s", e) | |
| return None | |
| def probe(self) -> dict: | |
| """Make one tiny real call and report what actually happened. | |
| The chain deliberately swallows errors so one dead provider cannot | |
| break a student's request. That is right for serving and useless for | |
| diagnosis β an expired key, a missing model grant and a bad model id | |
| all look identical from outside. This exposes the difference. | |
| """ | |
| if not self.enabled or not self.client: | |
| return { | |
| "ok": False, | |
| "reason": self.disabled_reason or "not enabled", | |
| "fix": "Provide AWS credentials (task role on AWS, or access keys elsewhere).", | |
| } | |
| try: | |
| self.client.invoke_model( | |
| modelId=self.model, | |
| body=json.dumps({ | |
| "anthropic_version": "bedrock-2023-05-31", | |
| "max_tokens": 8, | |
| "messages": [{"role": "user", "content": [{"type": "text", "text": "ping"}]}], | |
| }), | |
| contentType="application/json", | |
| accept="application/json", | |
| ) | |
| return {"ok": True, "model": self.model, "region": self.region} | |
| except Exception as e: | |
| name = type(e).__name__ | |
| text = str(e) | |
| fix = "See the error text." | |
| if "AccessDenied" in name or "AccessDenied" in text: | |
| fix = ( | |
| f"Model access not granted. In the Bedrock console for " | |
| f"{self.region}, request access to this model, then retry." | |
| ) | |
| elif "ValidationException" in name or "ValidationException" in text: | |
| fix = ( | |
| "Model id rejected. Most regions need a cross-region " | |
| "inference profile id (starting 'us.'), not a bare model id." | |
| ) | |
| elif "ExpiredToken" in text or "InvalidSignature" in text: | |
| fix = "AWS credentials are expired or invalid." | |
| elif "ThrottlingException" in name or "Throttling" in text: | |
| fix = "Rate limited by Bedrock. Retry, or request a quota increase." | |
| return { | |
| "ok": False, | |
| "error_type": name, | |
| "error": text[:400], | |
| "fix": fix, | |
| "model": self.model, | |
| "region": self.region, | |
| } | |
| def get_status(self) -> dict: | |
| return { | |
| "enabled": self.enabled, | |
| "model": self.model if self.enabled else None, | |
| "region": self.region, | |
| "priority": self.priority, | |
| "credentials": "iam-role-or-keys" if self.enabled else None, | |
| "disabled_reason": self.disabled_reason, | |
| } | |
| # Module-level singleton, matching the claude_engine / groq_fallback pattern. | |
| bedrock_fallback = BedrockFallback() | |