Spaces:
Sleeping
Sleeping
| """Editable, illustrative System One requests; these are not benchmark results.""" | |
| MODEL = 'apertus-decide' | |
| def request(state, **questions): | |
| return {'model': MODEL, 'state': state, 'questions': questions} | |
| def choice(instructions, **criteria): | |
| return {'type': 'choice', 'instructions': instructions, 'criteria': criteria} | |
| def score(instructions, *criteria): | |
| return {'type': 'score', 'instructions': instructions, 'criteria': list(criteria)} | |
| def noul(instructions, criteria=None): | |
| question = {'type': 'noul', 'instructions': instructions} | |
| if criteria is not None: | |
| question['criteria'] = criteria | |
| return question | |
| EXAMPLES = { | |
| 'Customer support 路 combined decisions': request( | |
| 'My order arrived with a cracked screen. I would like my money back, please.', | |
| intent=choice('What is the primary customer intent?', | |
| refund='Request a refund for a damaged item', tracking='Track a delayed parcel', | |
| information='Ask about product specifications'), | |
| urgency=score('How urgent is the customer request?', | |
| 'Routine product enquiry', 'Customer needs help with a damaged purchase', | |
| 'Critical service outage affecting many customers'), | |
| refund_requested=noul('Is the customer asking for a refund?')), | |
| 'Agent routing 路 Choice': request( | |
| 'Find the opening hours on the museum website and include a source link.', | |
| route=choice('Which capability is most appropriate for this request?', | |
| web_search='Find current information on websites and cite sources', | |
| calculator='Perform numerical calculations', | |
| document_search='Search only the uploaded private documents', | |
| direct_answer='Answer a general question without external evidence')), | |
| 'Incident triage 路 ordered Score': request( | |
| {'service': 'checkout', 'symptom': 'All payment attempts fail', | |
| 'scope': 'All customers', 'workaround': 'None available'}, | |
| severity=score('Rate the operational severity of this incident.', | |
| 'Cosmetic issue with no functional impact', 'Minor issue with a working workaround', | |
| 'Major feature impaired for some customers', 'Critical outage blocking all purchases')), | |
| 'Retrieval gate 路 Noul': request( | |
| 'According to the uploaded annual report, what were the operating expenses in 2025?', | |
| needs_documents=noul('Does answering this request require consulting the uploaded document?')), | |
| 'Evidence check 路 explicit false / true': request( | |
| {'claim': 'The invoice has been paid.', | |
| 'evidence': 'Invoice 1042 is marked overdue. No payment is recorded.'}, | |
| supported=noul('Does the evidence support the claim?', | |
| {'false': 'The evidence does not support the claim', | |
| 'true': 'The evidence supports the claim'})), | |
| 'Inbox routing 路 structured object': request( | |
| {'subject': 'Cannot sign in', 'message': 'My password reset link has expired.', | |
| 'channel': 'email'}, | |
| team=choice('Which team should handle this message?', | |
| account='Login, passwords and account access', | |
| billing='Invoices, subscriptions and payments', | |
| shipping='Parcel tracking and delivery')), | |
| 'Review analysis 路 mixed questions': request( | |
| 'The battery lasts all day and the camera is excellent, but the case scratches easily.', | |
| sentiment=choice('What is the overall sentiment?', | |
| positive='Mostly satisfied with the product', negative='Mostly dissatisfied', | |
| neutral='No clear positive or negative opinion'), | |
| satisfaction=score('Rate the expressed product satisfaction.', | |
| 'Very dissatisfied', 'Dissatisfied', 'Mixed', 'Satisfied', 'Very satisfied'), | |
| durability_issue=noul('Does the review mention a durability problem?')), | |
| 'Conversation handoff 路 array state': request( | |
| [{'role': 'user', 'content': 'I cannot access my account.'}, | |
| {'role': 'agent', 'content': 'Please try the password reset link.'}, | |
| {'role': 'user', 'content': 'I tried twice. Can I speak with a person?'}], | |
| handoff=noul('Is the user explicitly requesting a human agent?'), | |
| next_action=choice('What should happen next?', | |
| human='Offer a handoff to a human support agent', | |
| repeat='Send the same password reset instructions again', | |
| close='Close the conversation as resolved')), | |
| 'Content operations 路 nullable descriptions': request( | |
| 'A practical tutorial showing how to install a Python package and run a first script.', | |
| category=choice('Choose the best content category.', | |
| tutorial=None, announcement=None, opinion=None)), | |
| 'Answer quality 路 five-level rubric': request( | |
| {'question': 'What is photosynthesis?', | |
| 'answer': 'Plants use light energy to convert carbon dioxide and water into sugars, releasing oxygen.'}, | |
| quality=score('Rate how well the answer addresses the question.', | |
| 'Irrelevant or incorrect', 'Major factual errors', 'Partly correct but incomplete', | |
| 'Correct and relevant', 'Correct, relevant and clearly explained')), | |
| 'Document analysis 路 several Choice questions': request( | |
| 'This research article presents a new text encoder and compares it with existing models on retrieval tasks.', | |
| document_type=choice('What type of document is this?', | |
| research='Academic research article', manual='User manual', news='News report'), | |
| subject=choice('What is the main subject?', | |
| machine_learning='Text representation and information retrieval', | |
| economics='Markets and economic policy', ecology='Ecosystems and biodiversity')), | |
| 'Evidence review 路 several Noul questions': request( | |
| {'note': 'The package arrived on Tuesday. The outer box was damaged, but the contents were intact.'}, | |
| delivered=noul('Has the package been delivered?'), | |
| packaging_damaged=noul('Was the outer packaging damaged?'), | |
| contents_damaged=noul('Were the contents damaged?')), | |
| 'Writing assessment 路 several Score questions': request( | |
| 'The application stores files locally. To export a file, select Export from the File menu.', | |
| clarity=score('How clearly is the procedure explained?', | |
| 'Unclear', 'Partially clear', 'Clear', 'Very clear'), | |
| detail=score('How detailed is the procedure?', | |
| 'No actionable detail', 'Basic instructions', 'Detailed instructions', 'Comprehensive instructions')), | |
| 'Research screening 路 nested object state': request( | |
| {'study': {'title': 'Evaluating sentence embeddings for document search', | |
| 'method': 'Comparison of dense retrieval models', | |
| 'evaluation': {'datasets': 'Three public retrieval collections', 'metric': 'nDCG@10'}}}, | |
| relevant=noul('Is the study relevant to embedding-based information retrieval?'), | |
| method=choice('Which methodological category best describes the study?', | |
| empirical='Empirical comparison using benchmark datasets', | |
| theoretical='Theoretical analysis without experiments', | |
| review='Literature review without original evaluation')), | |
| 'Meeting records 路 array of observations': request( | |
| ['The team agreed to update the documentation.', | |
| 'No owner was assigned.', 'The completion date remains open.'], | |
| action_exists=noul('Was a follow-up action agreed?'), | |
| owner_assigned=noul('Was a responsible person assigned?'), | |
| completeness=score('How complete is the action record?', | |
| 'No action recorded', 'Action only', 'Action and owner', 'Action, owner and deadline')), | |
| 'Document relationship 路 structured instructions': request( | |
| {'document_a': 'The service is available Monday to Friday.', | |
| 'document_b': 'The service is closed on Saturdays and Sundays.'}, | |
| relation={'type': 'choice', | |
| 'instructions': {'task': 'Classify the relationship between the two statements.', | |
| 'scope': 'Consider only their stated availability.'}, | |
| 'criteria': {'consistent': 'The statements can both be true', | |
| 'contradictory': 'The statements cannot both be true', | |
| 'unrelated': 'The statements address unrelated subjects'}}), | |
| 'Workflow routing 路 structured option descriptions': request( | |
| {'message': 'Please correct the spelling in the introduction.', 'attachment': 'draft.txt'}, | |
| workflow={'type': 'choice', 'instructions': 'Select the appropriate workflow.', | |
| 'criteria': {'copy_edit': {'purpose': 'Correct spelling, grammar and punctuation'}, | |
| 'summarize': {'purpose': 'Produce a shorter account of the content'}, | |
| 'translate': {'purpose': 'Convert the text to another language'}}}), | |
| 'Rubric evaluation 路 structured Score levels': request( | |
| 'The release note identifies the issue, affected version and available workaround.', | |
| completeness={'type': 'score', 'instructions': 'Evaluate the completeness of the release note.', | |
| 'criteria': [{'level': 'Insufficient', 'description': 'No issue or affected version'}, | |
| {'level': 'Partial', 'description': 'Issue described but no workaround'}, | |
| {'level': 'Complete', 'description': 'Issue, affected version and workaround included'}]}), | |
| 'Binary classification 路 two-option Choice': request( | |
| 'The service stopped responding immediately after the configuration update.', | |
| category=choice('Does this message report an operational incident?', | |
| incident='A failure or interruption of a service', | |
| other='An informational request without a reported failure')), | |
| 'Schema variation 路 model alias omitted': { | |
| 'state': 'The article describes how to configure a document search index.', | |
| 'questions': {'technical': noul('Is this technical documentation?')}}, | |
| 'Schema variation 路 full model identifier': { | |
| **request('The user wants to locate a passage in an uploaded report.', | |
| route=choice('Which capability should be used?', | |
| retrieval='Search the uploaded documents', | |
| calculation='Perform an arithmetic calculation')), | |
| 'model': 'andreasmartin/apertus-v1.1-swiss-decide-0.4b-bidir'}, | |
| 'Content review 路 uncertainty as a Choice option': request( | |
| 'The note states that a delivery is scheduled but does not report whether it arrived.', | |
| delivery_status=choice('What delivery status is supported by the note?', | |
| delivered='The note explicitly confirms arrival', | |
| not_delivered='The note explicitly confirms a failed delivery', | |
| insufficient_evidence='The note does not establish whether delivery occurred')), | |
| } | |
| # The actions below are candidate decisions, not operations executed by this service. | |
| APPLICATION_EXAMPLES = { | |
| 'Email triage': request('I was charged twice.', | |
| queue=choice('Which queue should handle this email?', | |
| billing='Charges, invoices and payments', account='Login and account access', | |
| shipping='Orders and delivery'), | |
| urgency=score('How urgently should this email be reviewed?', | |
| 'Routine informational inquiry', 'Time-sensitive customer billing problem', | |
| 'Critical widespread service outage'), | |
| next_action=choice('Which follow-up is most appropriate?', | |
| billing_queue='Route to the billing queue for review', | |
| documentation='Send general product documentation', | |
| order_lookup='Look up the delivery status')), | |
| 'Support routing': request('Nobody can log in. All employees receive a service unavailable error.', | |
| team=choice('Which team should investigate?', | |
| it_operations='Widespread service availability and authentication incidents', | |
| billing='Payment and invoice queries', sales='Product demonstrations and pricing'), | |
| severity=score('How severe is the reported incident?', | |
| 'Minor inconvenience', 'Some users affected', 'Critical outage affecting all users'), | |
| next_action=choice('Select an appropriate response.', | |
| on_call='Assign the on-call operations team', | |
| routine_queue='Add to the routine information queue', | |
| close='Close as resolved')), | |
| 'Lead scoring': request('Can we book a demo? Our team is evaluating your product for rollout next month.', | |
| purchase_intent=score('How strong is the expressed purchase intent?', | |
| 'No commercial interest', 'General exploration', 'Active evaluation', | |
| 'High intent with a concrete purchasing timeframe'), | |
| demo_requested=noul('Is a product demonstration explicitly requested?'), | |
| next_action=choice('Which follow-up fits this inquiry?', | |
| sales_follow_up='Prioritize a sales follow-up and arrange a demo', | |
| newsletter='Offer a general newsletter subscription', | |
| technical_incident='Assign to incident response')), | |
| 'Model routing': request({'task': 'Summarize a short FAQ', | |
| 'content': 'Office hours are 9 to 5. Support is available by email. The FAQ contains six short entries.'}, | |
| model_tier=choice('Which model tier is proportionate to this task?', | |
| small='Small model for short, straightforward text transformations', | |
| large='Large model for complex multi-step synthesis across long documents', | |
| specialized='Specialized model for image or audio analysis'), | |
| next_action=choice('Which dispatch decision follows from the task?', | |
| small_model='Dispatch to the configured small text model', | |
| vision_model='Dispatch to an image analysis model', | |
| request_image='Ask the user to upload an image')), | |
| 'Tool selection': request('Where is order #482?', | |
| tool=choice('Which available capability best answers this request?', | |
| order_lookup='Retrieve order status and tracking by order number', | |
| calculator='Calculate numerical expressions', | |
| product_search='Find product descriptions'), | |
| next_action=choice('Which action should the calling application consider?', | |
| order_status_api='Call the order-status API using the provided order number', | |
| calculate='Perform an arithmetic calculation', | |
| product_catalog='Search the product catalog')), | |
| 'Safety gate': request({'request': 'Delete the customer records from the archive.', | |
| 'policy': 'Deleting records requires explicit approval by a human operator.', | |
| 'approval': 'No human approval has been recorded.'}, | |
| approval_required=noul('According to the supplied policy, does this request require human approval?'), | |
| next_action=choice('Which next action is consistent with the supplied policy and approval status?', | |
| pause='Pause and request human approval', | |
| proceed='Delete the records immediately', | |
| report_complete='Report that deletion is complete')), | |
| 'Answer evaluation': request({'source': 'The standard warranty lasts one year.', | |
| 'draft_answer': 'The product has a five-year standard warranty.'}, | |
| support=choice('How does the draft answer relate to the source?', | |
| supported='The draft is supported by the source', | |
| contradicted='The draft conflicts with the source', | |
| insufficient_evidence='The source does not address the claim'), | |
| next_action=choice('Which next step should the application consider?', | |
| regenerate='Retry answer generation using the source', | |
| accept='Accept the draft unchanged', | |
| calculate='Use a calculator')), | |
| 'Agent verification': request({'task': 'Export the report to a file.', | |
| 'agent_message': 'The export is complete.', | |
| 'observations': {'file_exists': False, 'export_log': 'Export failed: destination unavailable.'}}, | |
| completion=choice('Does the observed state establish task completion?', | |
| complete='The required export file exists and the export succeeded', | |
| incomplete='The required export file is missing or export failed', | |
| unknown='There is no observation of the export outcome'), | |
| next_action=choice('Which next step follows from the observations?', | |
| retry_export='Retry the export step after addressing the destination error', | |
| deliver='Deliver the successfully exported file', | |
| close='Close the task as completed')), | |
| 'RAG grounding': request({'question': 'What were operating expenses in 2025?', | |
| 'retrieved_passages': ['The company opened a new office in 2025.', | |
| 'The annual report includes a financial statements section.']}, | |
| evidence=choice('Do the retrieved passages support answering the question?', | |
| sufficient='The passages state the requested operating expenses', | |
| insufficient='The passages do not provide the requested operating expenses', | |
| conflicting='The passages give contradictory expense figures'), | |
| next_action=choice('Which next step is justified by the available evidence?', | |
| retrieve_more='Retrieve more relevant financial passages', | |
| answer='Answer with an expense figure from the supplied passages', | |
| calculate='Calculate the expenses from numeric values in the passages')), | |
| } | |
| EXAMPLES = {**APPLICATION_EXAMPLES, **EXAMPLES} | |