{ "examples": 100, "base": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.15, "present_accuracy": 0.76, "extraction_f1": 0.7275 }, "fine_tuned": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.92, "present_accuracy": 0.91, "extraction_f1": 0.9059 }, "details": [ { "index": 1, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"license_grant\",\n \"present\": true,\n \"extracted_language\": [\n \"License. Subject to the terms and conditions of this Agreement, Licensor hereby grants to Licensee, and Licensee hereby accepts from Licensor, a personal, non-exclusive, royalty-free right and license to use the Licensed Mark solely and exclusively as a component of Licensee's own corporate name and in connection with marketing the investment management, investment consultation and investment advisory services that Investment Advisor may provide to Licensee.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 12.54, "input_tokens": 3343, "output_tokens": 122 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Uncapped Liability\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 7.071, "input_tokens": 3343, "output_tokens": 31 } } }, { "index": 2, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"volume_restriction\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 6.726, "input_tokens": 3573, "output_tokens": 36 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Volume Restriction\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 7.459, "input_tokens": 3573, "output_tokens": 31 } } }, { "index": 3, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"statement_of_work\",\n \"present\": true,\n \"extracted_language\": [\n \"outlined in Attachment 1: Statement of Work, to the extent owned or transferable by MSL, and required by IBM to perform such functions.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 5.154, "input_tokens": 184, "output_tokens": 66 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Ip Ownership Assignment\",\n \"present\": true,\n \"extracted_language\": [\n \"outlined in Attachment 1: Statement of Work, to the extent owned or transferable by MSL, and required by IBM to perform such functions.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 7.891, "input_tokens": 184, "output_tokens": 62 } } }, { "index": 4, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"liquidated_damages\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 38.324, "input_tokens": 3594, "output_tokens": 38 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Liquidated Damages\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 38.499, "input_tokens": 3594, "output_tokens": 32 } } }, { "index": 5, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"franchise_rights\",\n \"present\": true,\n \"extracted_language\": [\n \"We grant to you, upon the terms and conditions contained in this Agreement, the exclusive right to establish and operate a Franchised Business and a license to use the methods, procedures and products developed by us in the business of selling and servicing Unit Franchises in the territory described on Attachment A attached to this Agreement and incorporated into this Agreement by reference (the 'Master Territory')\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 8.801, "input_tokens": 229, "output_tokens": 113 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Exclusivity\",\n \"present\": true,\n \"extracted_language\": [\n \"We grant to you, upon the terms and conditions contained in this Agreement, the exclusive right to establish and operate a Franchised Business and a license to use the methods, procedures and products developed by us in the business of selling and servicing Unit Franchises in the territory described on Attachment A attached to this Agreement and incorporated into this Agreement by reference (the \\\"Master Territory\\\").\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 13.593, "input_tokens": 229, "output_tokens": 108 } } }, { "index": 6, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"non_disparagement\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 5.564, "input_tokens": 3181, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Non-Disparagement\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 7.078, "input_tokens": 3181, "output_tokens": 33 } } }, { "index": 7, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"none\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 2.987, "input_tokens": 771, "output_tokens": 34 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Competitive Restriction Exception\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 4.568, "input_tokens": 771, "output_tokens": 33 } } }, { "index": 8, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"unlimited_all-you-can-eat_license\",\n \"present\": true,\n \"extracted_language\": [\n \"In consideration of the payments, through the issuance of securities to AJR as provided for in Section 3 hereof, AJR agrees to designate XC as an associate sponsor and the 'exclusive technology sponsor' for wearable computer technology of the Team for the Season and grants to XC the rights and benefits of such sponsorship as more fully set forth herein.\",\n \"XC shall have approval rights over use of its logo by AJR and shall designate which XC trademark identifications are utilized by AJR and how they are used in connection with the logos.\",\n \"The costs and expenses of placing the XC logos on the Cars as contemplated by subsections (ii), (ii) and (iii of this Section 1(b) shall be borne exclusively by AJR.\",\n \"XC shall receive and AJR shall provide, at AJR's cost and expense (except as otherwise expressly provided), the following: (i) AJR shall make one of the Cars available for display at one (1) promotional event to be designated by XC.\",\n \"If requested by XC, XC shall have the right to host XC's employees and invitees at a hospitality tent at the various race venues during the Season, the details of each hospitality event to be agreed to by XC and AJR (such details to include, without limitation, the amount of costs to be paid by XC, the drinks and food service at the event, seating, the availability of VIP parking, etc.).\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 75.05, "input_tokens": 4065, "output_tokens": 338 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Unlimited/All-You-Can-Eat-License\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 49.557, "input_tokens": 4065, "output_tokens": 39 } } }, { "index": 9, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"confidentiality\",\n \"present\": true,\n \"extracted_language\": [\n \"During the term of this Agreement, except in the performance of its obligations or exercise of its rights under this Agreement, neither OntoChem nor any of its Affiliates will discover, research, develop, manufacture or commercialize any compound or product directed to any Target, either independently or for or in collaboration with a third party (including the grant of a license to any third party), or have any of the foregoing activities performed on behalf of OntoChem or any of its Affiliates by a third party.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 10.708, "input_tokens": 256, "output_tokens": 139 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Non-Compete\",\n \"present\": true,\n \"extracted_language\": [\n \"During the term of this Agreement, except in the performance of its obligations or exercise of its rights under this Agreement, neither OntoChem nor any of its Affiliates will discover, research, develop, manufacture or commercialize any compound or product directed to any Target, either independently or for or in collaboration with a third party (including the grant of a license to any third party), or have any of the foregoing activities performed on behalf of OntoChem or any of its Affiliates by a third party.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 17.257, "input_tokens": 256, "output_tokens": 135 } } }, { "index": 10, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"confidentiality\",\n \"present\": true,\n \"extracted_language\": [\n \"During the term of this Agreement, XENCOR hereby grants to BII and BII hereby accepts for the purpose of pursuing the Project a non-exclusive, non-sub-licensable (except to Affiliated Companies), royalty-free, license to use the XENCOR Confidential Information and Know-How, the Material, the XENCOR Intellectual Property and/or any part of the Other Improvements for the sole purpose to develop the Process, and for the manufacturing of the Product for clinical purposes in accordance with this Agreement\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 10.772, "input_tokens": 256, "output_tokens": 140 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"License Grant\",\n \"present\": true,\n \"extracted_language\": [\n \"During the term of this Agreement, XENCOR hereby grants to BII and BII hereby accepts for the purpose of pursuing the Project a non-exclusive, non-sub-licensable (except to Affiliated Companies), royalty-free, license to use the XENCOR Confidential Information and Know-How, the Material, the XENCOR Intellectual Property and/or any part of the Other Improvements for the sole purpose to develop the Process, and for the manufacturing of the Product for clinical purposes in accordance with this Agreement\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 16.962, "input_tokens": 256, "output_tokens": 133 } } }, { "index": 11, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"definitions\",\n \"present\": true,\n \"extracted_language\": [\n \"Definition of 'Copyright Assignment Agreement' has the meaning set forth in Section 2.01.\",\n \"Definition of 'Copyrights' means copyrights, works of authorship (including all translations, adaptations, derivations and combinations thereof), mask works, designs and database rights, including, in each case, any registrations and applications therefor.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 57.692, "input_tokens": 4220, "output_tokens": 107 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Audit Rights\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 50.634, "input_tokens": 4220, "output_tokens": 30 } } }, { "index": 12, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"rof\",\n \"present\": true,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 39.338, "input_tokens": 3714, "output_tokens": 35 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Rofr/Rofo/Rofn\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 41.79, "input_tokens": 3714, "output_tokens": 37 } } }, { "index": 13, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"anti_assignment\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 44.676, "input_tokens": 3909, "output_tokens": 35 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Anti-Assignment\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 44.938, "input_tokens": 3909, "output_tokens": 31 } } }, { "index": 14, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"customer\",\n \"present\": true,\n \"extracted_language\": [\n \"Customer\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 0.5 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 3.028, "input_tokens": 160, "output_tokens": 36 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Parties\",\n \"present\": true,\n \"extracted_language\": [\n \"\\\"Customer\\\" or \\\"you\\\"\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 4.766, "input_tokens": 160, "output_tokens": 37 } } }, { "index": 15, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"exclusivity\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 15.714, "input_tokens": 4226, "output_tokens": 36 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Exclusivity\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 8.566, "input_tokens": 4226, "output_tokens": 31 } } }, { "index": 16, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"license\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 50.958, "input_tokens": 4505, "output_tokens": 34 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Affiliate License-Licensee\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 52.337, "input_tokens": 4505, "output_tokens": 33 } } }, { "index": 17, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"non-transferable_license\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 3.893, "input_tokens": 1904, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Non-Transferable License\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 5.328, "input_tokens": 1904, "output_tokens": 33 } } }, { "index": 18, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"exclusivity\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 3.129, "input_tokens": 955, "output_tokens": 36 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Exclusivity\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 4.323, "input_tokens": 955, "output_tokens": 31 } } }, { "index": 19, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"undefined\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 2.65, "input_tokens": 162, "output_tokens": 34 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Parties\",\n \"present\": true,\n \"extracted_language\": [\n \"Bluerock Residential Holdings, L.P.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 4.929, "input_tokens": 162, "output_tokens": 39 } } }, { "index": 20, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"assignment\",\n \"present\": true,\n \"extracted_language\": [\n \"Neither party shall assign or delegate this Agreement or any rights, duties or obligations hereunder to any other person and/or entity without prior express written approval of the other party.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 5.263, "input_tokens": 187, "output_tokens": 68 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Anti-Assignment\",\n \"present\": true,\n \"extracted_language\": [\n \"Neither party shall assign or delegate this Agreement or any rights, duties or obligations hereunder to any other person and/or entity without prior express written approval of the other party.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 8.184, "input_tokens": 187, "output_tokens": 65 } } }, { "index": 21, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"none\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 3.306, "input_tokens": 1341, "output_tokens": 34 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"No-Solicit Of Employees\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 4.932, "input_tokens": 1341, "output_tokens": 33 } } }, { "index": 22, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"competitive_restriction_exception\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 4.7, "input_tokens": 2726, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Competitive Restriction Exception\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 6.41, "input_tokens": 2726, "output_tokens": 33 } } }, { "index": 23, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"grant_of_license_and_exclusivity\",\n \"present\": true,\n \"extracted_language\": [\n \"North hereby grants to Company the right and privilege to use North's Likeness and North's Endorsement during the Term and within the Contract Territory in all reasonable forms of advertising including, but not limited to television (including the infomercial format), radio, print advertising, brochures, pamphlets, product packaging, point-of-purchase materials, Company's web-site and a demonstration video (hereinafter referred to as 'Advertising Materials') in connection with Company's advertisement and sale of the Endorsed Product only.\",\n \"Upon the expiration or termination of this Agreement, Company shall be prohibited from using or distributing any new or existing Advertising Materials using North's Likeness and North's Endorsement except for Endorsed Product with North's Likeness and/or North's Endorsement on the packaging that is already in the channel of retail distribution.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 64.199, "input_tokens": 4340, "output_tokens": 217 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Affiliate License-Licensee\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 48.625, "input_tokens": 4340, "output_tokens": 33 } } }, { "index": 24, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"irrevocable_perpetual_license\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 5.047, "input_tokens": 2784, "output_tokens": 41 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Irrevocable Or Perpetual License\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 7.045, "input_tokens": 2784, "output_tokens": 37 } } }, { "index": 25, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"verbal\",\n \"present\": true,\n \"extracted_language\": [\n \"It is expressly understood by the parties that CONSULTANT may play [* ****] clubs in the bag other than ADAMS GOLF clubs including, but not limited to, a putter by a manufacturer other than ADAMS GOLF but may not endorse those clubs and/or putter.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 7.199, "input_tokens": 210, "output_tokens": 92 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Exclusivity\",\n \"present\": true,\n \"extracted_language\": [\n \"(It is expressly understood by the parties that CONSULTANT may play [* ****] clubs in the bag other than ADAMS GOLF clubs including, but not limited to, a putter by a manufacturer other than ADAMS GOLF but may not endorse those clubs and/or putter.)\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 11.618, "input_tokens": 210, "output_tokens": 89 } } }, { "index": 26, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"non_compete\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 8.295, "input_tokens": 3573, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Non-Compete\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 9.505, "input_tokens": 3573, "output_tokens": 32 } } }, { "index": 27, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"definition\",\n \"present\": true,\n \"extracted_language\": [\n \"1.2 'Affiliate' shall mean...\",\n \"1.3 'Astellas Indemnitees' shall have the meaning as set forth in Section 17.3 below.\",\n \"1.4 'Astellas Territory' shall mean the country of Japan.\",\n \"1.5 'Authorized Designee' shall mean an officer of FG or Astellas, as the case may be, designated by the Chief Executive Officer of the respective corporation, that has been granted full authority to resolve a dispute arising between FG and Astellas as required under Section 2.4 or Section 19.1 hereof.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 81.764, "input_tokens": 5303, "output_tokens": 168 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Most Favored Nation\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 59.894, "input_tokens": 5303, "output_tokens": 32 } } }, { "index": 28, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"reporting_requirements\",\n \"present\": true,\n \"extracted_language\": [\n \"Both parties are entitled to reports of sales and to conduct periodic audits to ensure accuracy of Payments as follows: a. Each party will provide to the other a quarterly report (in hard copy and electronic copy (if applicable)) showing the AMR Product Suite sales including the Gross Proceeds and the Production Costs.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 7.337, "input_tokens": 222, "output_tokens": 97 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Audit Rights\",\n \"present\": true,\n \"extracted_language\": [\n \"Both parties are entitled to reports of sales and to conduct periodic audits to ensure accuracy of Payments as follows:\\n\\n a. Each party will provide to the other a quarterly report (in hard copy and electronic copy (if applicable)) showing the AMR Product Suite sales including the Gross Proceeds and the Production Costs.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 12.739, "input_tokens": 222, "output_tokens": 101 } } }, { "index": 29, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"merchandise_transfer\",\n \"present\": true,\n \"extracted_language\": [\n \"To the extent that there is Merchandise remaining at the Sale Termination Date (the 'Remaining Merchandise'), such Remaining Merchandise shall be deemed automatically transferred to Agent free and clear of all liens, claims, and encumbrances.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 0.625 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 6.614, "input_tokens": 250, "output_tokens": 86 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Post-Termination Services\",\n \"present\": true,\n \"extracted_language\": [\n \"To the extent that there is Merchandise remaining at the Sale Termination Date (the \\\"Remaining Merchandise\\\"), such Remaining Merchandise shall be deemed automatically transferred to Agent free and clear of all liens, claims, and encumbrances. Agent and its affiliates shall be authorized to sell or otherwise dispose of the Remaining Merchandise with all logos, brand names, and other Intellectual Property intact, and shall be authorized to advertise the sale of the Remaining Merchandise using the Intellectual Property.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 16.623, "input_tokens": 250, "output_tokens": 131 } } }, { "index": 30, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"irrevocable_perpetual_license\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 51.874, "input_tokens": 3905, "output_tokens": 41 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Irrevocable Or Perpetual License\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 54.73, "input_tokens": 3905, "output_tokens": 37 } } }, { "index": 31, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"post-termination_services\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 55.474, "input_tokens": 3853, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Post-Termination Services\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 61.54, "input_tokens": 3853, "output_tokens": 33 } } }, { "index": 32, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"source_code_escrow\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 30.719, "input_tokens": 2508, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Source Code Escrow\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 39.412, "input_tokens": 2508, "output_tokens": 32 } } }, { "index": 33, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"renewal_term\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 52.523, "input_tokens": 3997, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Renewal Term\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 59.561, "input_tokens": 3997, "output_tokens": 32 } } }, { "index": 34, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"non-disparagement\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 31.209, "input_tokens": 2581, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Non-Disparagement\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 41.396, "input_tokens": 2581, "output_tokens": 33 } } }, { "index": 35, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"volume_restriction\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 52.436, "input_tokens": 3622, "output_tokens": 36 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Volume Restriction\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 54.681, "input_tokens": 3622, "output_tokens": 31 } } }, { "index": 36, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"negotiation\",\n \"present\": true,\n \"extracted_language\": [\n \"Excite will negotiate exclusively with Sponsor for the next [****] in good faith effort to negotiate and execute a written sponsorship renewal agreement.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 4.761, "input_tokens": 183, "output_tokens": 63 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Exclusivity\",\n \"present\": true,\n \"extracted_language\": [\n \"Excite will negotiate exclusively with Sponsor for the next [****] in good faith effort to negotiate and execute a written sponsorship renewal agreement.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 7.578, "input_tokens": 183, "output_tokens": 61 } } }, { "index": 37, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"anti_assignment\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 68.018, "input_tokens": 4265, "output_tokens": 35 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Anti-Assignment\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 70.194, "input_tokens": 4265, "output_tokens": 31 } } }, { "index": 38, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"license_grant\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 30.51, "input_tokens": 2240, "output_tokens": 36 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"License Grant\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 34.639, "input_tokens": 2240, "output_tokens": 30 } } }, { "index": 39, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"affiliate_license\",\n \"present\": true,\n \"extracted_language\": [\n \"2.2 (a) Provide guidance and/or lead initiatives related to the development of pharmaceutical forms of the EHS cannabinoid portfolio including methods to enhance bioavailability or delivery of these compounds.\",\n \"2.3 (a) Provide advice and general assistance in EHN's business efforts primarily involving product development, but also including guidance on marketing, sales, and product education.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 64.841, "input_tokens": 4007, "output_tokens": 114 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Affiliate License-Licensor\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 56.696, "input_tokens": 4007, "output_tokens": 34 } } }, { "index": 40, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"term_definition\",\n \"present\": true,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 0.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 71.307, "input_tokens": 4386, "output_tokens": 35 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Renewal Term\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 67.168, "input_tokens": 4386, "output_tokens": 32 } } }, { "index": 41, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"termination_rights\",\n \"present\": true,\n \"extracted_language\": [\n \"We will have the right to terminate the Ad Fund by giving you thirty (30) days' advance written notice.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 4.449, "input_tokens": 177, "output_tokens": 59 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Termination For Convenience\",\n \"present\": true,\n \"extracted_language\": [\n \"We will have the right to terminate the Ad Fund by giving you thirty (30) days' advance written notice.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 7.068, "input_tokens": 177, "output_tokens": 56 } } }, { "index": 42, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"renewal_term\",\n \"present\": true,\n \"extracted_language\": [\n \"This Agreement shall commence on the Effective Date of this Agreement and shall remain in effect for a period of 5 years ('Initial Term'). Thereafter, this Agreement shall be renewed automatically on a five (5) year basis, unless one party notifies the other of its desire to terminate this Agreement at least sixty (60) days prior to the expiration of the Initial Term or then current renewal term, as applicable, or unless a new Agreement is signed between the USA MCO and IMedicor which will then invalidate this Agreement.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 0.8311688311688311 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 65.527, "input_tokens": 3714, "output_tokens": 147 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Renewal Term\",\n \"present\": true,\n \"extracted_language\": [\n \"This Agreement shall commence on the Effective Date of this Agreement and shall remain in effect for a period of 5 years (\\\"Initial Term\\\"). Thereafter, this Agreement shall be renewed automatically on a five (5) year basis, unless one party notifies the other of its desire to terminate this Agreement at least sixty 90 days prior to the expiration of the Initial Term or then current renewal term, as applicable, or unless a new Agreement is signed between the USA MCO and IMedicor which will then invalidate this Agreement.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 0.8441558441558442 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 72.425, "input_tokens": 3714, "output_tokens": 139 } } }, { "index": 43, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"indemnity\",\n \"present\": true,\n \"extracted_language\": [\n \"The Company shall indemnify and hold CNET harmless from and against any Losses that CNET may suffer, incur or be subjected to by reason of any legal action, proceeding, arbitration or other claim by a third party, whether commenced or threatened, arising out of or as a result of (a) any breach or alleged breach by the Company of its representations, warranties or covenants hereunder; (b) the use by CNET of the Company Marks or any content provided by the Company to CNET expressly for display in connection with or as part of the Promotions, including claims of infringement or misappropriation of intellectual property rights; or (c) the operation of the Company Site or the offer or sale of the Products by the Company or through the Company Site\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 0.9922480620155039 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 14.518, "input_tokens": 321, "output_tokens": 194 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Cap On Liability\",\n \"present\": true,\n \"extracted_language\": [\n \"The Company shall indemnify and hold CNET harmless from and against any Losses that CNET may suffer, incur or be subjected to by reason of any legal action, proceeding,\\n\\n\\n\\n\\n\\n arbitration or other claim by a third party, whether commenced or threatened, arising out of or as a result of (a) any breach or alleged breach by the Company of its representations, warranties or covenants hereunder; (b) the use by CNET of the Company Marks or any content provided by the Company to CNET expressly for display in connection with or as part of the Promotions, including claims of infringement or misappropriation of intellectual property rights; or (c) the operation of the Company Site or the offer or sale of the Products by the Company or through the Company Sit\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 25.538, "input_tokens": 321, "output_tokens": 204 } } }, { "index": 44, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"termination\",\n \"present\": true,\n \"extracted_language\": [\n \"ENERGOUS may, at any time after the third anniversary of the Effective Date, terminate this Agreement with or without cause upon not less than one hundred and eighty (180) days prior written notice to DIALOG.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 6.04, "input_tokens": 198, "output_tokens": 79 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Termination For Convenience\",\n \"present\": true,\n \"extracted_language\": [\n \"ENERGOUS may, at any time after the third anniversary of the Effective Date, terminate this Agreement with or without cause upon not less than one hundred and eighty (180) days prior written notice to DIALOG.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 9.729, "input_tokens": 198, "output_tokens": 77 } } }, { "index": 45, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"date\",\n \"present\": true,\n \"extracted_language\": [\n \"March 27, 2019\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 3.573, "input_tokens": 163, "output_tokens": 45 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Agreement Date\",\n \"present\": true,\n \"extracted_language\": [\n \"March 27, 2019\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 5.23, "input_tokens": 163, "output_tokens": 41 } } }, { "index": 46, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"non-transferable-license\",\n \"present\": true,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 0.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 48.462, "input_tokens": 3556, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Non-Transferable License\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 58.985, "input_tokens": 3556, "output_tokens": 33 } } }, { "index": 47, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"verbal_agreement\",\n \"present\": true,\n \"extracted_language\": [\n \"Promptly upon receipt of notice from Pfizer, Exact and Pfizer shall engage in exclusive good faith negotiations to enter into a definitive written agreement for the OB/Gyn Commercial Rights.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 5.344, "input_tokens": 187, "output_tokens": 71 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Rofr/Rofo/Rofn\",\n \"present\": true,\n \"extracted_language\": [\n \"Promptly upon receipt of notice from Pfizer, Exact and Pfizer shall engage in exclusive good faith negotiations to enter into a definitive written agreement for the OB/Gyn Commercial Rights.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 9.05, "input_tokens": 187, "output_tokens": 71 } } }, { "index": 48, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"none\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 72.055, "input_tokens": 4226, "output_tokens": 34 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Most Favored Nation\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 61.633, "input_tokens": 4226, "output_tokens": 32 } } }, { "index": 49, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"no_solicit_of_employees\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 50.153, "input_tokens": 4940, "output_tokens": 39 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"No-Solicit Of Employees\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 54.542, "input_tokens": 4940, "output_tokens": 33 } } }, { "index": 50, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"joint_ip_ownership\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 68.383, "input_tokens": 4136, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Joint Ip Ownership\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 66.275, "input_tokens": 4136, "output_tokens": 31 } } }, { "index": 51, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"warranty_duration\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 33.512, "input_tokens": 1655, "output_tokens": 36 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Warranty Duration\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 34.966, "input_tokens": 1655, "output_tokens": 31 } } }, { "index": 52, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"license_grant\",\n \"present\": true,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 52.473, "input_tokens": 3622, "output_tokens": 36 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"License Grant\",\n \"present\": true,\n \"extracted_language\": [\n \"TrueLink hereby grants to Client a non-exclusive license to use the Interface in the ordinary course of its business of the origination, underwriting, processing and funding of consumer finance receivables in accordance with this Agreement.\",\n \"The license granted to Client pursuant to section 2(a) consists of the following rights:\\n\\ni. Use and execution of the Interface on a compatible software platform (as such compatibility specifications may be issued by TrueLink from time to time); and\\n\\nii. Access to the Interface from multiple computer located at those sites listed on Exhibit \\\"A.\\\" Client may amend Exhibit \\\"A\\\" by giving TrueLink written notice of the new sites.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 0.8288288288288288 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 76.642, "input_tokens": 3622, "output_tokens": 166 } } }, { "index": 53, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"undefined\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 2.639, "input_tokens": 159, "output_tokens": 34 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Parties\",\n \"present\": true,\n \"extracted_language\": [\n \"BellSouth International, Inc.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 4.498, "input_tokens": 159, "output_tokens": 36 } } }, { "index": 54, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"breach_of_agreement\",\n \"present\": true,\n \"extracted_language\": [\n \"Any Transfer that is made without our approval will constitute a breach of this Agreement and convey no rights to or interests in this Agreement, you, the Franchise, or any other The Joint Corp. franchise.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 6.154, "input_tokens": 194, "output_tokens": 79 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Anti-Assignment\",\n \"present\": true,\n \"extracted_language\": [\n \"Any Transfer that is made without our approval will constitute a breach of this Agreement and convey no rights to or interests in this Agreement, you, the Franchise, or any other The Joint Corp. franchise.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 9.022, "input_tokens": 194, "output_tokens": 72 } } }, { "index": 55, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"breach_of_agreement\",\n \"present\": true,\n \"extracted_language\": [\n \"Failure to secure such insurance as of the date of execution of this Agreement shall constitute a breach of this Agreement.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 4.586, "input_tokens": 175, "output_tokens": 60 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Insurance\",\n \"present\": true,\n \"extracted_language\": [\n \"Failure to secure such insurance as of the date of execution of this Agreement shall constitute a breach of this Agreement.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 6.404, "input_tokens": 175, "output_tokens": 51 } } }, { "index": 56, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"effective_date\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 44.567, "input_tokens": 4634, "output_tokens": 35 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Effective Date\",\n \"present\": true,\n \"extracted_language\": [\n \"May 6, 2019\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 61.614, "input_tokens": 4634, "output_tokens": 39 } } }, { "index": 57, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"location\",\n \"present\": true,\n \"extracted_language\": [\n \"This Agreement has been executed and delivered in Los Angeles County in the State of California\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 0.6037735849056604 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 3.897, "input_tokens": 193, "output_tokens": 51 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Governing Law\",\n \"present\": true,\n \"extracted_language\": [\n \"This Agreement has been executed and delivered in Los Angeles County in the State of California, and its interpretation, validity and performance shall be construed and enforced in accordance with the laws of the State of California.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 9.323, "input_tokens": 193, "output_tokens": 72 } } }, { "index": 58, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"change_of_control\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 53.492, "input_tokens": 3840, "output_tokens": 36 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Change Of Control\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 60.286, "input_tokens": 3840, "output_tokens": 31 } } }, { "index": 59, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"network\",\n \"present\": true,\n \"extracted_language\": [\n \"Network\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 2.752, "input_tokens": 154, "output_tokens": 36 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Parties\",\n \"present\": true,\n \"extracted_language\": [\n \"Network\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 3.907, "input_tokens": 154, "output_tokens": 31 } } }, { "index": 60, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"parties\",\n \"present\": true,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 32.363, "input_tokens": 1947, "output_tokens": 35 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Parties\",\n \"present\": true,\n \"extracted_language\": [\n \"The Associated Press\",\n \"Kubient\",\n \"Customer\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 0.9166666666666666 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 38.246, "input_tokens": 1947, "output_tokens": 41 } } }, { "index": 61, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"confidentiality\",\n \"present\": true,\n \"extracted_language\": [\n \"Supplier shall permit Bank of America to inspect the physical system equipment, operational environment, and Confidential Information handling procedures.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 4.404, "input_tokens": 175, "output_tokens": 58 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Audit Rights\",\n \"present\": true,\n \"extracted_language\": [\n \"Supplier shall permit Bank of America to inspect the physical system equipment, operational environment, and Confidential Information handling procedures.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 6.797, "input_tokens": 175, "output_tokens": 52 } } }, { "index": 62, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"renewal_term\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 3.911, "input_tokens": 536, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Renewal Term\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 5.021, "input_tokens": 536, "output_tokens": 32 } } }, { "index": 63, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"source_code_escrow\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 51.017, "input_tokens": 3913, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Source Code Escrow\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 54.559, "input_tokens": 3913, "output_tokens": 32 } } }, { "index": 64, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"no_solicit_of_employees\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 12.373, "input_tokens": 738, "output_tokens": 39 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"No-Solicit Of Employees\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 11.885, "input_tokens": 738, "output_tokens": 33 } } }, { "index": 65, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"joint_ip_ownership\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 47.301, "input_tokens": 3354, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Joint Ip Ownership\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 53.896, "input_tokens": 3354, "output_tokens": 31 } } }, { "index": 66, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"no_solicit_of_employees\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 49.711, "input_tokens": 3730, "output_tokens": 39 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"No-Solicit Of Employees\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 59.397, "input_tokens": 3730, "output_tokens": 33 } } }, { "index": 67, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"assignment\",\n \"present\": true,\n \"extracted_language\": [\n \"No Party shall assign this Agreement or any rights or obligations hereunder...\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 0.20689655172413793 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 3.816, "input_tokens": 279, "output_tokens": 49 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Anti-Assignment\",\n \"present\": true,\n \"extracted_language\": [\n \"No Party shall assign this Agreement or any rights or obligations hereunder or, except as expressly set forth in the Agreement with respect to the PHL Services and Investors Capital Services, delegate any of their respective duties and obligations hereunder, without the prior written consent of the other Parties, which, in view of the unique and specialized nature of each Party's obligations hereunder, may be declined by any Investors Capital Party on the one hand or any PHL Party, on the other hand, as the case may be, for any reason. Any attempted assignment or delegation in violation of this Section shall be void.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 19.633, "input_tokens": 279, "output_tokens": 157 } } }, { "index": 68, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"liability_for_consequential_damages\",\n \"present\": true,\n \"extracted_language\": [\n \"Except for claims arising out of Articles 4.3 and 7.0, or as may be set forth in a SOW, neither Party will be liable for any consequential damages, lost profits, lost savings, loss of anticipated revenue, or any exemplary, punitive, special or indirect damages, even if advised of their possibility.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 8.142, "input_tokens": 220, "output_tokens": 108 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Cap On Liability\",\n \"present\": true,\n \"extracted_language\": [\n \"Except for claims arising out of Articles 4.3 and 7.0, or as may be set forth in a SOW, neither Party will be liable for any consequential damages, lost profits, lost savings, loss of anticipated revenue, or any exemplary, punitive, special or indirect damages, even if advised of their possibility.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 12.299, "input_tokens": 220, "output_tokens": 98 } } }, { "index": 69, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"volume_restriction\",\n \"present\": true,\n \"extracted_language\": [\n \"mPhase may apply only the exact logo shown on Schedule A of this Agreement (the 'Lucent Co-Branding Logo') in Approved Uses for mPhase's Goods.\",\n \"mPhase agrees to abide by the guidelines appearing on Schedule B of this Agreement specifying the dimensions, requirements and specifications, and the review process related to use of the Lucent Co-Branding Logo.\",\n \"In the event that mPhase's use of the Lucent Co-Branding Logo, in the sole judgment of Lucent, may adversely affect Lucent's rights to the mark shown on Schedule A or the marks and names LUCENT, LUCENT TECHNOLOGIES or LUCENT INNOVATION RING DESIGN, Lucent may upon written notification to mPhase, terminate this Agreement.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 93.602, "input_tokens": 2023, "output_tokens": 200 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Volume Restriction\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 32.928, "input_tokens": 2023, "output_tokens": 31 } } }, { "index": 70, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"license\",\n \"present\": true,\n \"extracted_language\": [\n \"A 10 year exclusive, worldwide license to use, modify, reproduce, distribute, display and transmit any and all PACIFICAP ENTERTAINMENT nostalgic television show library Content.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 0.5853658536585366 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 5.62, "input_tokens": 247, "output_tokens": 75 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"License Grant\",\n \"present\": true,\n \"extracted_language\": [\n \"Subject to the terms and conditions of this Agreement, PACIFICAP ENTERTAINMENT hereby grants to THE HENRY FILM AND ENTERTAINMENT CORPORATION, under PACIFICAP ENTERTAINMENT'S full ownership and or fully authorized licensing Rights of Content\\n\\n (a) A 10 year exclusive, worldwide license to use, modify, reproduce, distribute, display and transmit any and all PACIFICAP ENTERTAINMENT nostalgic television show library Content.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 15.679, "input_tokens": 247, "output_tokens": 125 } } }, { "index": 71, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"insurance_coverage\",\n \"present\": true,\n \"extracted_language\": [\n \"Metavante currently maintains and, if available at a reasonable cost, Metavante shall continue to pay for, and maintain in full force and effect during the Term insurance as follows: A. Workers' compensation and employers' liability insurance...\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 0.3063829787234043 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 6.749, "input_tokens": 445, "output_tokens": 84 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Insurance\",\n \"present\": true,\n \"extracted_language\": [\n \"Metavante currently maintains and, if available at a reasonable cost, Metavante shall continue to pay for, and maintain in full force and effect during the Term insurance as follows:\\n\\nA. Workers' compensation and employers' liability insurance with limits to conform with the greater of the amount required by Wisconsin applicable state statutory law or one million dollars ($1,000,000) each accident, including occupational disease coverage;\\n\\nB. Commercial general liability insurance with limits not less than three million dollars ($3,000,000) combined single limit for bodily injury, death, and property damage, including personal injury, contractual liability, independent contractors, broad-form property damage, and products and completed operations coverage;\\n\\nC. Commercial automobile liability insurance with limits not less than one million dollars ($1,000,000) each occurrence combined single limit of liability for bodily injury, death, and property damage, including owned and non-owned and hired automobile coverages, as applicable;\\n\\nD. Commercial Blanket Bond, including Electronic & Computer Crime or Unauthorized Computer Access coverage, in the amount of not less than ten million dollars ($10,000,000); and\\n\\nE. Professional liability insurance (Errors and Omissions) with limits not less than three million dollars ($3,000,000) annual aggregate for all claims each policy year for computer programming and electronic data processing services.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 49.364, "input_tokens": 445, "output_tokens": 330 } } }, { "index": 72, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"audit_rights\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 57.775, "input_tokens": 4087, "output_tokens": 35 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Audit Rights\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 59.77, "input_tokens": 4087, "output_tokens": 30 } } }, { "index": 73, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"confidentiality\",\n \"present\": true,\n \"extracted_language\": [\n \"During the term hereof and for a period of five (5) years thereafter, each Party agrees not to, either directly or indirectly, for itself or on behalf of any other person, firm, partnership, corporation or other entity hire, solicit, contract for, attempt to solicit, or cause to be solicited, the employment or services of any current or previous employee of the other Party (unless a period of sixty months has elapsed from the last date that such employee was employed by such party) without the prior written consent of such other Party.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 10.867, "input_tokens": 263, "output_tokens": 146 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"No-Solicit Of Employees\",\n \"present\": true,\n \"extracted_language\": [\n \"During the term hereof and for a period of five (5) years thereafter, each Party agrees not to, either directly or indirectly, for itself or on behalf of any other person, firm, partnership, corporation or other entity hire, solicit, contract for, attempt to solicit, or cause to be solicited, the employment or services of any current or previous employee of the other Party (unless a period of sixty months has elapsed from the last date that such employee was employed by such party) without the prior written consent of such other Party.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 18.02, "input_tokens": 263, "output_tokens": 143 } } }, { "index": 74, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"term\",\n \"present\": true,\n \"extracted_language\": [\n \"The term of this Agreement (the \\\"Term\\\") shall commence on the date first above written and shall terminate on 31 December 2006, unless sooner terminated in accordance with the provisions hereof.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 5.89, "input_tokens": 198, "output_tokens": 77 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Expiration Date\",\n \"present\": true,\n \"extracted_language\": [\n \"The term of this Agreement (the \\\"Term\\\") shall commence on the date first above written and shall terminate on 31 December 2006, unless sooner terminated in accordance with the provisions hereof.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 9.437, "input_tokens": 198, "output_tokens": 75 } } }, { "index": 75, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"undefined\",\n \"present\": true,\n \"extracted_language\": [\n \"SUNTRON GCO, L.P.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 3.476, "input_tokens": 163, "output_tokens": 44 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Parties\",\n \"present\": true,\n \"extracted_language\": [\n \"SUNTRON GCO, L.P.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 5.121, "input_tokens": 163, "output_tokens": 40 } } }, { "index": 76, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"product_information_sales_video\",\n \"present\": true,\n \"extracted_language\": [\n \"KI will participate in the production of up to one (1) product information/sales video per License Year during the period of this Agreement on behalf of Diplomat at a mutually acceptable time and place.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 5.874, "input_tokens": 193, "output_tokens": 77 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Volume Restriction\",\n \"present\": true,\n \"extracted_language\": [\n \"KI will participate in the production of up to one (1) product information/sales video per License Year during the period of this Agreement on behalf of Diplomat at a mutually acceptable time and place.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 8.901, "input_tokens": 193, "output_tokens": 71 } } }, { "index": 77, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"renewal_term\",\n \"present\": true,\n \"extracted_language\": [\n \"2.2 Term: The license hereby granted shall commence on the date the Franchised Restaurant opens for business (the 'Commencement Date'), and, unless sooner terminated in accordance with the terms and provisions of this Agreement, shall continue for the period of years set forth on SCHEDULE 1 hereto (the 'Term').\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 84.572, "input_tokens": 4442, "output_tokens": 109 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Renewal Term\",\n \"present\": true,\n \"extracted_language\": [\n \"The license hereby granted shall commence on the date the Franchised Restaurant opens for business (the \\\"Commencement Date\\\"), and, unless sooner terminated in accordance with the terms and provisions of this Agreement, shall continue for the period of years set forth on SCHEDULE 1 hereto (the \\\"Term\\\").\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 93.587, "input_tokens": 4442, "output_tokens": 107 } } }, { "index": 78, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"insurance_coverage\",\n \"present\": true,\n \"extracted_language\": [\n \"The insurance coverages and limits required to be maintained by Supplier and its Subcontractors shall be primary and non-contributory to insurance coverage, if any, maintained by Bank of America.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 0.6516853932584269 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 5.421, "input_tokens": 233, "output_tokens": 72 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Insurance\",\n \"present\": true,\n \"extracted_language\": [\n \"The insurance coverages and limits required to be maintained by Supplier and its Subcontractors shall be primary and non-contributory to insurance coverage, if any, maintained by Bank of America. Supplier and Proprietary to Bank of America its Subcontractors and their underwriters shall waive subrogation against Bank of America and shall cause their insurer(s) to waive subrogation against Bank of America.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 15.052, "input_tokens": 233, "output_tokens": 109 } } }, { "index": 79, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"commission\",\n \"present\": true,\n \"extracted_language\": [\n \"After the Initial Term, Neoforma shall pay to VerticalNet commissions equal to [*] of any Medical Products Net Revenues during such Contract Year resulting from (a) any VerticalNet Medical Products Listing or (b) any Neoforma Medical Products Listing for which VerticalNet was the Transaction Origination Party.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 7.375, "input_tokens": 213, "output_tokens": 94 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Revenue/Profit Sharing\",\n \"present\": true,\n \"extracted_language\": [\n \"After the Initial Term, Neoforma shall pay to VerticalNet commissions equal to [*] of any Medical Products Net Revenues during such Contract Year resulting from (a) any VerticalNet Medical Products Listing or (b) any Neoforma Medical Products Listing for which VerticalNet was the Transaction Origination Party.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 11.75, "input_tokens": 213, "output_tokens": 92 } } }, { "index": 80, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"anti_assignment\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 2.972, "input_tokens": 477, "output_tokens": 35 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Anti-Assignment\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 4.077, "input_tokens": 477, "output_tokens": 31 } } }, { "index": 81, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"unlimited_all-you-can-eat_license\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 50.787, "input_tokens": 3711, "output_tokens": 41 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Unlimited/All-You-Can-Eat-License\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 55.758, "input_tokens": 3711, "output_tokens": 39 } } }, { "index": 82, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"royalty-bearing-license\",\n \"present\": true,\n \"extracted_language\": [\n \"The license granted by Licensee pursuant to Section 9.2.2 shall be extended to also include the Development, Manufacture, sale, import or other Commercialization of Licensed Products in the Field in the Territory, and, unless this Agreement is terminated by Bioeq pursuant to pursuant to Sections 15.2.1, 15.2.2, 15.2.3, 15.2.4, 15.2.8 or 15.2.9, or by Licensee pursuant to Section 15.2.5 (in [***]), such license shall thereafter be royalty-bearing on Bioeq on Net Sales (applied mutatis mutandis as if Bioeq were Licensee, and additionally applying to sales by sublicensees of Bioeq) by Bioeq, its Affiliates, and its sublicensees of Licensed Products in the Field in the Territory which have [***] Licensee Improvement, at [***].\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 18.08, "input_tokens": 355, "output_tokens": 240 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"License Grant\",\n \"present\": true,\n \"extracted_language\": [\n \"The license granted by Licensee pursuant to Section 9.2.2 shall be extended to also include the Development, Manufacture, sale, import or other Commercialization of Licensed Products in the Field in the Territory, and, unless this Agreement is terminated by Bioeq pursuant to pursuant to Sections 15.2.1, 15.2.2, 15.2.3, 15.2.4, 15.2.8 or 15.2.9, or by Licensee pursuant to Section 15.2.5 (in [***]), such license shall thereafter be royalty-bearing on Bioeq on Net Sales (applied mutatis mutandis as if Bioeq were Licensee, and additionally applying to sales by sublicensees of Bioeq) by Bioeq, its Affiliates, and its sublicensees of Licensed Products in the Field in the Territory which have [***] Licensee Improvement, at [***].\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 29.237, "input_tokens": 355, "output_tokens": 232 } } }, { "index": 83, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"post-termination_services\",\n \"present\": true,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 0.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 57.616, "input_tokens": 4061, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Post-Termination Services\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 59.73, "input_tokens": 4061, "output_tokens": 33 } } }, { "index": 84, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"joint_ip_ownership\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 47.695, "input_tokens": 3466, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Joint Ip Ownership\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 56.051, "input_tokens": 3466, "output_tokens": 31 } } }, { "index": 85, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"undefined\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 0.0, "extraction_f1": 0.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 2.647, "input_tokens": 161, "output_tokens": 34 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Parties\",\n \"present\": true,\n \"extracted_language\": [\n \"Care G. B. Plus Ltd.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 4.854, "input_tokens": 161, "output_tokens": 38 } } }, { "index": 86, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"none\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 52.318, "input_tokens": 3657, "output_tokens": 34 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"No-Solicit Of Customers\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 53.548, "input_tokens": 3657, "output_tokens": 33 } } }, { "index": 87, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"date\",\n \"present\": true,\n \"extracted_language\": [\n \"January 22, 2007\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 3.483, "input_tokens": 163, "output_tokens": 45 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Agreement Date\",\n \"present\": true,\n \"extracted_language\": [\n \"January 22, 2007\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 5.154, "input_tokens": 163, "output_tokens": 41 } } }, { "index": 88, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"definitions\",\n \"present\": true,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 0.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 56.202, "input_tokens": 3782, "output_tokens": 34 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Non-Disparagement\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 73.963, "input_tokens": 3782, "output_tokens": 33 } } }, { "index": 89, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"termination\",\n \"present\": true,\n \"extracted_language\": [\n \"Either Party may terminate this Agreement for any reason upon providing 180 days prior written notice to the other Party prior to the expiration of the then-current term or unless sooner terminated in accordance with the terms of this Agreement.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 6.076, "input_tokens": 198, "output_tokens": 79 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Termination For Convenience\",\n \"present\": true,\n \"extracted_language\": [\n \"Either Party may terminate this Agreement for any reason upon providing 180 days prior written notice to the other Party prior to the expiration of the then-current term or unless sooner terminated in accordance with the terms of this Agreement.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 10.257, "input_tokens": 198, "output_tokens": 77 } } }, { "index": 90, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"post-termination-services\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 56.565, "input_tokens": 3920, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Post-Termination Services\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 60.572, "input_tokens": 3920, "output_tokens": 33 } } }, { "index": 91, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"renewal_term\",\n \"present\": true,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 0.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 59.919, "input_tokens": 3939, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Renewal Term\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 68.877, "input_tokens": 3939, "output_tokens": 32 } } }, { "index": 92, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"technology_rights\",\n \"present\": true,\n \"extracted_language\": [\n \"SFJ shall cause each employee and individual consultant of such SFJ or its Affiliates...\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 0.3943661971830986 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 4.365, "input_tokens": 236, "output_tokens": 55 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Ip Ownership Assignment\",\n \"present\": true,\n \"extracted_language\": [\n \"SFJ shall cause each employee and individual consultant of such SFJ or its Affiliates (but excluding Permitted Third Parties of SFJ and its Affiliates, which are separately addressed in Section 11.1.1.3(c)) who conceives, discovers, develops or otherwise makes any Trial Invention to be under an obligation to assign to PB their rights in any such Trial Invention.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 15.28, "input_tokens": 236, "output_tokens": 114 } } }, { "index": 93, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"insurance\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 10.749, "input_tokens": 660, "output_tokens": 34 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Insurance\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 10.276, "input_tokens": 660, "output_tokens": 29 } } }, { "index": 94, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"governing law\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 64.705, "input_tokens": 3863, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Governing Law\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 78.853, "input_tokens": 3863, "output_tokens": 32 } } }, { "index": 95, "task": "cuad_classification", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"term_definition\",\n \"present\": true,\n \"extracted_language\": [\n \"Term means...\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 0.05714285714285715 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 3.188, "input_tokens": 246, "output_tokens": 39 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Expiration Date\",\n \"present\": true,\n \"extracted_language\": [\n \"\\\"Term\\\" means, on a Country-by-Country and Collaboration Product-by-Collaboration Product basis, the period from the Effective Date until the later of (a) the expiration or termination of the last Valid Claim of a Patent Right covering the Pooled Compound in such Collaboration Product in such Country, and (b) fifteen (15) years from First Commercial Sale in such Country, unless this Agreement is terminated earlier in accordance with Article 14.\"\n ]\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 16.217, "input_tokens": 246, "output_tokens": 123 } } }, { "index": 96, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"minimum_commitment\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 66.383, "input_tokens": 3864, "output_tokens": 36 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Minimum Commitment\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 76.218, "input_tokens": 3864, "output_tokens": 31 } } }, { "index": 97, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"post-termination-services\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 62.012, "input_tokens": 3996, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Post-Termination Services\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 72.06, "input_tokens": 3996, "output_tokens": 33 } } }, { "index": 98, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"competitive_restriction_exception\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 0.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 62.545, "input_tokens": 3664, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Competitive Restriction Exception\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 69.724, "input_tokens": 3664, "output_tokens": 33 } } }, { "index": 99, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"non_compete\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 3.119, "input_tokens": 443, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Non-Compete\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 4.272, "input_tokens": 443, "output_tokens": 32 } } }, { "index": 100, "task": "cuad_extraction", "source": "CUAD", "base": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"non-disparagement\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "base", "latency_seconds": 103.102, "input_tokens": 3967, "output_tokens": 37 } }, "fine_tuned": { "output": "{\n \"task\": \"cuad_clause_extraction\",\n \"clause_type\": \"Non-Disparagement\",\n \"present\": false,\n \"extracted_language\": []\n}", "scores": { "json_valid": 1.0, "label_accuracy": 0.0, "clause_type_accuracy": 1.0, "present_accuracy": 1.0, "extraction_f1": 1.0 }, "diagnostics": { "base_model": "Qwen/Qwen3-1.7B", "adapter": "outputs\\legal-qwen3-1.7b-qlora", "loaded": true, "cuda": true, "device": "cuda:0", "four_bit": true, "path": "fine_tuned", "latency_seconds": 89.339, "input_tokens": 3967, "output_tokens": 33 } } } ] }