Update Operational_Instructions/create-submission-result.ipynb

Operational_Instructions/create-submission-result.ipynb

@@ -1,68 +1,98 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "a5e875f5",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import pandas as pd\n",
-    
-    "\n# Code that generates the output\n\n",
-    "answers = pd.DataFrame({\n",
-    "    \"id\": [1, 2],\n",
-    "    \"question\": [\"What is the capital of France?\", \"What is the capital of Germany?\"],\n",
-    "    \"passages\": [\n",
-    "        [\n",
-    "            {\"passage\": \"Paris is the capital and most populous city of France.\", \n",
-    "             \"doc_IDs\": [\"<urn:uuid:1234abcd-5678-efgh-9101-ijklmnopqrst>\", \"<urn:uuid:1234abcd-5678-efgh-9202-ijklmnopqrst>\"]},\n",
-    "            {\"passage\": \"France is located in Western Europe.\", \n",
-    "             \"doc_IDs\": [\"<urn:uuid:1234abcd-5678-efgh-9101-ijklmnopqrst>\"]}\n",
-    "        ],\n",
-    "        [\n",
-    "            {\"passage\": \"Berlin is the capital of Germany.\", \n",
-    "             \"doc_IDs\": [\"<urn:uuid:1234abcd-5678-efgh-9101-ijklmnopqrst>\"]}\n",
-    "        ]\n",
-    "    ],\n",
-    "    \"final_prompt\": [\n",
-    "        \"Using the following - Paris is the capital and most populous city of France - and - France is located in Western Europe - answer the question: What is the capital of France?\",\n",
-    "        \"Using the following - Berlin is the capital of Germany - answer the question: What is the capital of Germany?\"\n",
-    "    ],\n",
-    "    \"answer\": [\"Paris\", \"Berlin\"]\n",
-    "})\n\n",
-    "# Convert to JSON format\n",
-    "answers_json = answers.to_json(orient='records', lines=True, force_ascii=False)\n",
-    "\n",
-    "# Or just save to a file\n",
-    "answers.to_json(\"answers.jsonl\", orient='records', lines=True, force_ascii=False)\n",
-    "\n",
-    "# Load the file to make sure it is correct\n",
-    "loaded_answers = pd.read_json(\"answers.jsonl\", lines=True)\n\n",
-    "# Print the loaded answers\n",
-    "print(loaded_answers)\n"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": ".venv",
-   "language": "python",
-   "name": "python3"
+import pandas as pd
+import jsonschema
+from jsonschema import validate
+import json
+
+# LiveRAG Answer JSON schema: 
+json_schema = """
+{ 
+"$schema": "http://json-schema.org/draft-07/schema#", 
+
+  "title": "Answer file schema", 
+  "type": "object", 
+  "properties": { 
+    "id": { 
+      "type": "integer", 
+      "description": "Question ID" 
+    }, 
+    "question": { 
+      "type": "string", 
+      "description": "The question" 
+    }, 
+    "passages": { 
+      "type": "array", 
+      "description": "Passages used and related FineWeb doc IDs, ordered by decreasing importance", 
+      "items": { 
+        "type": "object", 
+        "properties": {
+          "passage": { 
+            "type": "string", 
+            "description": "Passage text" 
+          }, 
+          "doc_IDs": {
+            "type": "array", 
+            "description": "Passage related FineWeb doc IDs, ordered by decreasing importance", 
+            "items": { 
+              "type": "string", 
+              "description": "FineWeb doc ID, e.g., <urn:uuid:d69cbebc-133a-4ebe-9378-68235ec9f091>"
+            } 
+          } 
+        },
+        "required": ["passage", "doc_IDs"]
+      }
+    }, 
+    "final_prompt": {
+      "type": "string",
+      "description": "Final prompt, as submitted to Falcon LLM"
+    },
+    "answer": {
+      "type": "string",
+      "description": "Your answer"
+    }
   },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.12.8"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
+  "required": ["id", "question", "passages", "final_prompt", "answer"]
 }
+"""
+
+# Code that generates the output
+answers = pd.DataFrame({
+    "id": [1, 2],
+    "question": ["What is the capital of France?", "What is the capital of Germany?"],
+    "passages": [
+        [
+            {"passage": "Paris is the capital and most populous city of France.",
+             "doc_IDs": ["<urn:uuid:1234abcd-5678-efgh-9101-ijklmnopqrst>", "<urn:uuid:1234abcd-5678-efgh-9202-ijklmnopqrst>"]},
+            {"passage": "France is located in Western Europe.",
+             "doc_IDs": ["<urn:uuid:1234abcd-5678-efgh-9101-ijklmnopqrst>"]}
+        ],
+        [
+            {"passage": "Berlin is the capital of Germany.",
+             "doc_IDs": ["<urn:uuid:1234abcd-5678-efgh-9101-ijklmnopqrst>"]}
+        ]
+    ],
+    "final_prompt": [
+        "Using the following - Paris is the capital and most populous city of France - and - France is located in Western Europe - answer the question: What is the capital of France?",
+        "Using the following - Berlin is the capital of Germany - answer the question: What is the capital of Germany?"
+    ],
+    "answer": ["Paris", "Berlin"]
+})
+
+# Convert to JSON format
+answers_json = answers.to_json(orient='records', lines=True, force_ascii=False)
+
+# Or just save to a file
+answers.to_json("answers.jsonl", orient='records', lines=True, force_ascii=False)
+
+# Load the file to make sure it is ok
+loaded_answers = pd.read_json("answers.jsonl", lines=True)
+
+# Load the JSON schema
+schema = json.loads(json_schema)
+
+# Validate each Answer JSON object against the schema
+for answer in loaded_answers.to_dict(orient='records'):
+    try:
+        validate(instance=answer, schema=schema)
+        print(f"Answer {answer['id']} is valid.")
+    except jsonschema.exceptions.ValidationError as e:
+        print(f"Answer {answer['id']} is invalid: {e.message}")