Revision 5: negative-result model card, report rev. 5 (PDF/MD/figure), CITATION.cff, iterations 020-021 code, logs, sealed tests and checkpoints
Browse files- 020/cuerdas020-sealed.log +114 -0
- 020/eval020.py +127 -0
- 020/kaggle_kernel.py +11 -0
- 020/make_sealed020.py +37 -0
- 020/predictions020.jsonl +0 -0
- 020/preregistro020.md +30 -0
- 020/resultados020.md +66 -0
- 020/results020.jsonl +99 -0
- 020/sealed020.json +1 -0
- 021/c021.py +106 -0
- 021/cuerdas021-a.log +106 -0
- 021/cuerdas021-b.log +152 -0
- 021/evalA.py +55 -0
- 021/evalB.py +35 -0
- 021/kaggle_kernel_A.py +11 -0
- 021/kaggle_kernel_B.py +11 -0
- 021/make_sealed021.py +50 -0
- 021/predictionsA.jsonl +0 -0
- 021/predictionsB.jsonl +0 -0
- 021/preregistro021.md +28 -0
- 021/resultados021.md +59 -0
- 021/resultsA.jsonl +88 -0
- 021/resultsB.jsonl +134 -0
- 021/sealed021.json +1 -0
- 021/stats021.py +27 -0
- 021/stats021.txt +67 -0
- CITATION.cff +17 -13
- README.md +130 -70
- checkpoints/k2_s17.pt +3 -0
- checkpoints/k2_s29.pt +3 -0
- checkpoints/k2_s43.pt +3 -0
- checkpoints/k2v_s17.pt +3 -0
- checkpoints/k2v_s29.pt +3 -0
- checkpoints/k2v_s43.pt +3 -0
- checkpoints/sp_kv_s17.pt +3 -0
- checkpoints/sp_kv_s29.pt +3 -0
- checkpoints/sp_kv_s43.pt +3 -0
- checkpoints/sp_values_csv_s17.pt +3 -0
- checkpoints/sp_values_csv_s29.pt +3 -0
- checkpoints/sp_values_csv_s43.pt +3 -0
- checkpoints/sp_values_s17.pt +3 -0
- checkpoints/sp_values_s29.pt +3 -0
- checkpoints/sp_values_s43.pt +3 -0
- paper/cuerdas-technical-report.md +0 -0
- paper/cuerdas-technical-report.pdf +2 -2
- paper/fig_architecture.svg +45 -0
020/cuerdas020-sealed.log
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[{"stream_name":"stdout","time":3.388341406,"data":" ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 44.1/44.1 kB 1.6 MB/s eta 0:00:00\n"}
|
| 2 |
+
,{"stream_name":"stdout","time":4.420337612,"data":" ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 10.0/10.0 MB 79.0 MB/s eta 0:00:00\n"}
|
| 3 |
+
,{"stream_name":"stdout","time":4.454446033,"data":" ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 566.4/566.4 kB 28.4 MB/s eta 0:00:00\n"}
|
| 4 |
+
,{"stream_name":"stdout","time":4.530811111,"data":" ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.0/3.0 MB 48.9 MB/s eta 0:00:00\n"}
|
| 5 |
+
,{"stream_name":"stderr","time":17.960055603,"data":"ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n"}
|
| 6 |
+
,{"stream_name":"stderr","time":17.960079725,"data":"litellm 1.85.7 requires tokenizers\u003c1.0,\u003e=0.21.0, but you have tokenizers 0.20.3 which is incompatible.\n"}
|
| 7 |
+
,{"stream_name":"stderr","time":17.960084532,"data":"gradio 6.26.0 requires huggingface-hub\u003c2.0,\u003e=1.16.0, but you have huggingface-hub 0.36.2 which is incompatible.\n"}
|
| 8 |
+
,{"stream_name":"stderr","time":17.960088477,"data":"diffusers 0.40.0 requires huggingface-hub\u003c2.0,\u003e=1.23.0, but you have huggingface-hub 0.36.2 which is incompatible.\n"}
|
| 9 |
+
,{"stream_name":"stdout","time":65.837648993,"data":"{\"env\": {\"python\": \"3.13.15\", \"torch\": \"2.11.0+cu128\", \"transformers\": \"4.46.3\", \"device\": \"Tesla T4\"}, \"sealed_sha256\": \"4c7737f66364a44786b4d9acb4686a6f31829d78cb5914f19eca1df74fe73393\"}\n"}
|
| 10 |
+
,{"stream_name":"stdout","time":68.610543429,"data":"{\"route\": \"text_template\", \"set\": \"sealed_id\", \"n\": 200, \"acc\": 0.995, \"wilson95\": [97.2, 99.9], \"positions_per_event\": 35.0}\n"}
|
| 11 |
+
,{"stream_name":"stdout","time":72.844882846,"data":"{\"route\": \"text_template\", \"set\": \"sealed_ood\", \"n\": 400, \"acc\": 0.9925, \"wilson95\": [97.8, 99.7], \"positions_per_event\": 35.0}\n"}
|
| 12 |
+
,{"stream_name":"stdout","time":74.885158342,"data":"{\"route\": \"text_template\", \"set\": \"sealed_1_new\", \"n\": 192, \"acc\": 0.9635416666666666, \"wilson95\": [92.7, 98.2], \"positions_per_event\": 35.0}\n"}
|
| 13 |
+
,{"stream_name":"stdout","time":76.948103187,"data":"{\"route\": \"text_template\", \"set\": \"sealed_2_new\", \"n\": 192, \"acc\": 0.984375, \"wilson95\": [95.5, 99.5], \"positions_per_event\": 35.0}\n"}
|
| 14 |
+
,{"stream_name":"stdout","time":79.000557751,"data":"{\"route\": \"text_template\", \"set\": \"sealed_4_new\", \"n\": 192, \"acc\": 0.9427083333333334, \"wilson95\": [90.0, 96.8], \"positions_per_event\": 35.0}\n"}
|
| 15 |
+
,{"stream_name":"stdout","time":81.022912645,"data":"{\"route\": \"text_kv\", \"set\": \"sealed_id\", \"n\": 200, \"acc\": 0.065, \"wilson95\": [3.8, 10.8], \"positions_per_event\": 31.0, \"diff_vs_template\": -93.0, \"boot95\": [-96.5, -89.0]}\n"}
|
| 16 |
+
,{"stream_name":"stdout","time":85.009245483,"data":"{\"route\": \"text_kv\", \"set\": \"sealed_ood\", \"n\": 400, \"acc\": 0.025, \"wilson95\": [1.4, 4.5], \"positions_per_event\": 31.0, \"diff_vs_template\": -96.75, \"boot95\": [-98.25, -95.0]}\n"}
|
| 17 |
+
,{"stream_name":"stdout","time":86.926660201,"data":"{\"route\": \"text_kv\", \"set\": \"sealed_1_new\", \"n\": 192, \"acc\": 0.010416666666666666, \"wilson95\": [0.3, 3.7], \"positions_per_event\": 31.0, \"diff_vs_template\": -95.31, \"boot95\": [-98.44, -91.67]}\n"}
|
| 18 |
+
,{"stream_name":"stdout","time":88.836022647,"data":"{\"route\": \"text_kv\", \"set\": \"sealed_2_new\", \"n\": 192, \"acc\": 0.052083333333333336, \"wilson95\": [2.9, 9.3], \"positions_per_event\": 31.0, \"diff_vs_template\": -93.23, \"boot95\": [-96.35, -89.58]}\n"}
|
| 19 |
+
,{"stream_name":"stdout","time":90.758205317,"data":"{\"route\": \"text_kv\", \"set\": \"sealed_4_new\", \"n\": 192, \"acc\": 0.08333333333333333, \"wilson95\": [5.2, 13.1], \"positions_per_event\": 31.0, \"diff_vs_template\": -85.94, \"boot95\": [-90.62, -80.73]}\n"}
|
| 20 |
+
,{"stream_name":"stdout","time":92.80039212,"data":"{\"route\": \"text_abbrev\", \"set\": \"sealed_id\", \"n\": 200, \"acc\": 0.035, \"wilson95\": [1.7, 7.0], \"positions_per_event\": 27.67, \"diff_vs_template\": -96.0, \"boot95\": [-98.5, -93.0]}\n"}
|
| 21 |
+
,{"stream_name":"stdout","time":96.629012409,"data":"{\"route\": \"text_abbrev\", \"set\": \"sealed_ood\", \"n\": 400, \"acc\": 0.0, \"wilson95\": [0.0, 1.0], \"positions_per_event\": 27.63, \"diff_vs_template\": -99.25, \"boot95\": [-100.0, -98.25]}\n"}
|
| 22 |
+
,{"stream_name":"stdout","time":98.586372404,"data":"{\"route\": \"text_abbrev\", \"set\": \"sealed_1_new\", \"n\": 192, \"acc\": 0.0, \"wilson95\": [0.0, 2.0], \"positions_per_event\": 29.1, \"diff_vs_template\": -96.35, \"boot95\": [-98.96, -93.23]}\n"}
|
| 23 |
+
,{"stream_name":"stdout","time":100.684886105,"data":"{\"route\": \"text_abbrev\", \"set\": \"sealed_2_new\", \"n\": 192, \"acc\": 0.0, \"wilson95\": [0.0, 2.0], \"positions_per_event\": 30.47, \"diff_vs_template\": -98.44, \"boot95\": [-100.0, -96.35]}\n"}
|
| 24 |
+
,{"stream_name":"stdout","time":102.909738537,"data":"{\"route\": \"text_abbrev\", \"set\": \"sealed_4_new\", \"n\": 192, \"acc\": 0.0, \"wilson95\": [0.0, 2.0], \"positions_per_event\": 33.37, \"diff_vs_template\": -94.27, \"boot95\": [-97.4, -90.62]}\n"}
|
| 25 |
+
,{"stream_name":"stdout","time":105.034201776,"data":"{\"route\": \"text_csv\", \"set\": \"sealed_id\", \"n\": 200, \"acc\": 0.03, \"wilson95\": [1.4, 6.4], \"positions_per_event\": 30.67, \"diff_vs_template\": -96.5, \"boot95\": [-99.0, -93.5]}\n"}
|
| 26 |
+
,{"stream_name":"stdout","time":109.355365425,"data":"{\"route\": \"text_csv\", \"set\": \"sealed_ood\", \"n\": 400, \"acc\": 0.005, \"wilson95\": [0.1, 1.8], \"positions_per_event\": 30.63, \"diff_vs_template\": -98.75, \"boot95\": [-99.75, -97.5]}\n"}
|
| 27 |
+
,{"stream_name":"stdout","time":111.556024189,"data":"{\"route\": \"text_csv\", \"set\": \"sealed_1_new\", \"n\": 192, \"acc\": 0.005208333333333333, \"wilson95\": [0.1, 2.9], \"positions_per_event\": 32.1, \"diff_vs_template\": -95.83, \"boot95\": [-98.44, -92.71]}\n"}
|
| 28 |
+
,{"stream_name":"stdout","time":113.84502117,"data":"{\"route\": \"text_csv\", \"set\": \"sealed_2_new\", \"n\": 192, \"acc\": 0.005208333333333333, \"wilson95\": [0.1, 2.9], \"positions_per_event\": 33.47, \"diff_vs_template\": -97.92, \"boot95\": [-99.48, -95.83]}\n"}
|
| 29 |
+
,{"stream_name":"stdout","time":116.331066973,"data":"{\"route\": \"text_csv\", \"set\": \"sealed_4_new\", \"n\": 192, \"acc\": 0.0, \"wilson95\": [0.0, 2.0], \"positions_per_event\": 36.37, \"diff_vs_template\": -94.27, \"boot95\": [-97.4, -90.62]}\n"}
|
| 30 |
+
,{"stream_name":"stdout","time":117.764029661,"data":"{\"route\": \"text_values_csv\", \"set\": \"sealed_id\", \"n\": 200, \"acc\": 0.03, \"wilson95\": [1.4, 6.4], \"positions_per_event\": 22.0, \"diff_vs_template\": -96.5, \"boot95\": [-99.0, -93.5]}\n"}
|
| 31 |
+
,{"stream_name":"stdout","time":120.623199264,"data":"{\"route\": \"text_values_csv\", \"set\": \"sealed_ood\", \"n\": 400, \"acc\": 0.0025, \"wilson95\": [0.0, 1.4], \"positions_per_event\": 22.0, \"diff_vs_template\": -99.0, \"boot95\": [-99.75, -98.0]}\n"}
|
| 32 |
+
,{"stream_name":"stdout","time":122.201757359,"data":"{\"route\": \"text_values_csv\", \"set\": \"sealed_1_new\", \"n\": 192, \"acc\": 0.0, \"wilson95\": [0.0, 2.0], \"positions_per_event\": 22.46, \"diff_vs_template\": -96.35, \"boot95\": [-98.96, -93.23]}\n"}
|
| 33 |
+
,{"stream_name":"stdout","time":123.764041327,"data":"{\"route\": \"text_values_csv\", \"set\": \"sealed_2_new\", \"n\": 192, \"acc\": 0.0, \"wilson95\": [0.0, 2.0], \"positions_per_event\": 22.76, \"diff_vs_template\": -98.44, \"boot95\": [-100.0, -96.35]}\n"}
|
| 34 |
+
,{"stream_name":"stdout","time":125.356988219,"data":"{\"route\": \"text_values_csv\", \"set\": \"sealed_4_new\", \"n\": 192, \"acc\": 0.0, \"wilson95\": [0.0, 2.0], \"positions_per_event\": 23.56, \"diff_vs_template\": -94.27, \"boot95\": [-97.4, -90.62]}\n"}
|
| 35 |
+
,{"stream_name":"stdout","time":126.613655256,"data":"{\"route\": \"text_values\", \"set\": \"sealed_id\", \"n\": 200, \"acc\": 0.005, \"wilson95\": [0.1, 2.8], \"positions_per_event\": 19.0, \"diff_vs_template\": -99.0, \"boot95\": [-100.0, -97.5]}\n"}
|
| 36 |
+
,{"stream_name":"stdout","time":129.126717252,"data":"{\"route\": \"text_values\", \"set\": \"sealed_ood\", \"n\": 400, \"acc\": 0.0025, \"wilson95\": [0.0, 1.4], \"positions_per_event\": 19.0, \"diff_vs_template\": -99.0, \"boot95\": [-99.75, -98.0]}\n"}
|
| 37 |
+
,{"stream_name":"stdout","time":130.498097875,"data":"{\"route\": \"text_values\", \"set\": \"sealed_1_new\", \"n\": 192, \"acc\": 0.0, \"wilson95\": [0.0, 2.0], \"positions_per_event\": 19.46, \"diff_vs_template\": -96.35, \"boot95\": [-98.96, -93.23]}\n"}
|
| 38 |
+
,{"stream_name":"stdout","time":131.90200467,"data":"{\"route\": \"text_values\", \"set\": \"sealed_2_new\", \"n\": 192, \"acc\": 0.0, \"wilson95\": [0.0, 2.0], \"positions_per_event\": 19.76, \"diff_vs_template\": -98.44, \"boot95\": [-100.0, -96.35]}\n"}
|
| 39 |
+
,{"stream_name":"stdout","time":133.250639158,"data":"{\"route\": \"text_values\", \"set\": \"sealed_4_new\", \"n\": 192, \"acc\": 0.0, \"wilson95\": [0.0, 2.0], \"positions_per_event\": 20.56, \"diff_vs_template\": -94.27, \"boot95\": [-97.4, -90.62]}\n"}
|
| 40 |
+
,{"stream_name":"stdout","time":134.808868904,"data":"{\"route\": \"k2_s17\", \"set\": \"sealed_id\", \"n\": 200, \"acc\": 0.985, \"wilson95\": [95.7, 99.5], \"positions_per_event\": 17, \"diff_vs_template\": -1.0, \"boot95\": [-3.0, 1.0]}\n"}
|
| 41 |
+
,{"stream_name":"stdout","time":137.377613413,"data":"{\"route\": \"k2_s17\", \"set\": \"sealed_ood\", \"n\": 400, \"acc\": 0.99, \"wilson95\": [97.5, 99.6], \"positions_per_event\": 17, \"diff_vs_template\": -0.25, \"boot95\": [-1.5, 1.0]}\n"}
|
| 42 |
+
,{"stream_name":"stdout","time":138.581706175,"data":"{\"route\": \"k2_s17\", \"set\": \"sealed_1_new\", \"n\": 192, \"acc\": 0.9583333333333334, \"wilson95\": [92.0, 97.9], \"positions_per_event\": 17, \"diff_vs_template\": -0.52, \"boot95\": [-4.17, 3.12]}\n"}
|
| 43 |
+
,{"stream_name":"stdout","time":139.806274618,"data":"{\"route\": \"k2_s17\", \"set\": \"sealed_2_new\", \"n\": 192, \"acc\": 0.90625, \"wilson95\": [85.7, 94.0], \"positions_per_event\": 17, \"diff_vs_template\": -7.81, \"boot95\": [-12.5, -3.12]}\n"}
|
| 44 |
+
,{"stream_name":"stdout","time":141.049152533,"data":"{\"route\": \"k2_s17\", \"set\": \"sealed_4_new\", \"n\": 192, \"acc\": 0.828125, \"wilson95\": [76.8, 87.5], \"positions_per_event\": 17, \"diff_vs_template\": -11.46, \"boot95\": [-17.19, -5.73]}\n"}
|
| 45 |
+
,{"stream_name":"stdout","time":142.511667421,"data":"{\"route\": \"k2_s29\", \"set\": \"sealed_id\", \"n\": 200, \"acc\": 1.0, \"wilson95\": [98.1, 100.0], \"positions_per_event\": 17, \"diff_vs_template\": 0.5, \"boot95\": [0.0, 1.5]}\n"}
|
| 46 |
+
,{"stream_name":"stdout","time":145.062978489,"data":"{\"route\": \"k2_s29\", \"set\": \"sealed_ood\", \"n\": 400, \"acc\": 0.9975, \"wilson95\": [98.6, 100.0], \"positions_per_event\": 17, \"diff_vs_template\": 0.5, \"boot95\": [-0.5, 1.5]}\n"}
|
| 47 |
+
,{"stream_name":"stdout","time":146.304692095,"data":"{\"route\": \"k2_s29\", \"set\": \"sealed_1_new\", \"n\": 192, \"acc\": 0.984375, \"wilson95\": [95.5, 99.5], \"positions_per_event\": 17, \"diff_vs_template\": 2.08, \"boot95\": [-1.04, 5.21]}\n"}
|
| 48 |
+
,{"stream_name":"stdout","time":147.545631656,"data":"{\"route\": \"k2_s29\", \"set\": \"sealed_2_new\", \"n\": 192, \"acc\": 0.9791666666666666, \"wilson95\": [94.8, 99.2], \"positions_per_event\": 17, \"diff_vs_template\": -0.52, \"boot95\": [-3.12, 2.08]}\n"}
|
| 49 |
+
,{"stream_name":"stdout","time":148.784008133,"data":"{\"route\": \"k2_s29\", \"set\": \"sealed_4_new\", \"n\": 192, \"acc\": 0.875, \"wilson95\": [82.1, 91.5], \"positions_per_event\": 17, \"diff_vs_template\": -6.77, \"boot95\": [-11.98, -2.08]}\n"}
|
| 50 |
+
,{"stream_name":"stdout","time":150.214376809,"data":"{\"route\": \"k2_s43\", \"set\": \"sealed_id\", \"n\": 200, \"acc\": 0.98, \"wilson95\": [95.0, 99.2], \"positions_per_event\": 17, \"diff_vs_template\": -1.5, \"boot95\": [-4.0, 0.5]}\n"}
|
| 51 |
+
,{"stream_name":"stdout","time":152.814730981,"data":"{\"route\": \"k2_s43\", \"set\": \"sealed_ood\", \"n\": 400, \"acc\": 0.9825, \"wilson95\": [96.4, 99.1], \"positions_per_event\": 17, \"diff_vs_template\": -1.0, \"boot95\": [-2.5, 0.5]}\n"}
|
| 52 |
+
,{"stream_name":"stdout","time":154.044099749,"data":"{\"route\": \"k2_s43\", \"set\": \"sealed_1_new\", \"n\": 192, \"acc\": 0.859375, \"wilson95\": [80.3, 90.2], \"positions_per_event\": 17, \"diff_vs_template\": -10.42, \"boot95\": [-15.62, -5.21]}\n"}
|
| 53 |
+
,{"stream_name":"stdout","time":155.266531092,"data":"{\"route\": \"k2_s43\", \"set\": \"sealed_2_new\", \"n\": 192, \"acc\": 0.8697916666666666, \"wilson95\": [81.5, 91.0], \"positions_per_event\": 17, \"diff_vs_template\": -11.46, \"boot95\": [-16.67, -6.77]}\n"}
|
| 54 |
+
,{"stream_name":"stdout","time":156.500330949,"data":"{\"route\": \"k2_s43\", \"set\": \"sealed_4_new\", \"n\": 192, \"acc\": 0.8229166666666666, \"wilson95\": [76.3, 87.0], \"positions_per_event\": 17, \"diff_vs_template\": -11.98, \"boot95\": [-18.23, -6.25]}\n"}
|
| 55 |
+
,{"stream_name":"stdout","time":156.539517256,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_id\", \"acc\": 0.9883333333333333, \"vs\": \"text_template\", \"diff\": -0.67, \"boot95\": [-2.0, 0.83]}\n"}
|
| 56 |
+
,{"stream_name":"stdout","time":156.579250879,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_id\", \"acc\": 0.9883333333333333, \"vs\": \"text_kv\", \"diff\": 92.33, \"boot95\": [88.67, 95.67]}\n"}
|
| 57 |
+
,{"stream_name":"stdout","time":156.616884816,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_id\", \"acc\": 0.9883333333333333, \"vs\": \"text_abbrev\", \"diff\": 95.33, \"boot95\": [92.5, 97.83]}\n"}
|
| 58 |
+
,{"stream_name":"stdout","time":156.65477194,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_id\", \"acc\": 0.9883333333333333, \"vs\": \"text_csv\", \"diff\": 95.83, \"boot95\": [93.17, 98.17]}\n"}
|
| 59 |
+
,{"stream_name":"stdout","time":156.693577489,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_id\", \"acc\": 0.9883333333333333, \"vs\": \"text_values_csv\", \"diff\": 95.83, \"boot95\": [93.17, 98.17]}\n"}
|
| 60 |
+
,{"stream_name":"stdout","time":156.732202027,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_id\", \"acc\": 0.9883333333333333, \"vs\": \"text_values\", \"diff\": 98.33, \"boot95\": [96.83, 99.5]}\n"}
|
| 61 |
+
,{"stream_name":"stdout","time":156.826159533,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_ood\", \"acc\": 0.99, \"vs\": \"text_template\", \"diff\": -0.25, \"boot95\": [-1.25, 0.92]}\n"}
|
| 62 |
+
,{"stream_name":"stdout","time":156.919513567,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_ood\", \"acc\": 0.99, \"vs\": \"text_kv\", \"diff\": 96.5, \"boot95\": [94.75, 98.0]}\n"}
|
| 63 |
+
,{"stream_name":"stdout","time":157.011904627,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_ood\", \"acc\": 0.99, \"vs\": \"text_abbrev\", \"diff\": 99.0, \"boot95\": [98.33, 99.58]}\n"}
|
| 64 |
+
,{"stream_name":"stdout","time":157.103279626,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_ood\", \"acc\": 0.99, \"vs\": \"text_csv\", \"diff\": 98.5, \"boot95\": [97.5, 99.33]}\n"}
|
| 65 |
+
,{"stream_name":"stdout","time":157.195875214,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_ood\", \"acc\": 0.99, \"vs\": \"text_values_csv\", \"diff\": 98.75, \"boot95\": [97.83, 99.5]}\n"}
|
| 66 |
+
,{"stream_name":"stdout","time":157.288146417,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_ood\", \"acc\": 0.99, \"vs\": \"text_values\", \"diff\": 98.75, \"boot95\": [97.83, 99.5]}\n"}
|
| 67 |
+
,{"stream_name":"stdout","time":157.329154262,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_1_new\", \"acc\": 0.9340277777777778, \"vs\": \"text_template\", \"diff\": -2.95, \"boot95\": [-6.08, 0.35]}\n"}
|
| 68 |
+
,{"stream_name":"stdout","time":157.365357174,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_1_new\", \"acc\": 0.9340277777777778, \"vs\": \"text_kv\", \"diff\": 92.36, \"boot95\": [89.41, 94.97]}\n"}
|
| 69 |
+
,{"stream_name":"stdout","time":157.402069447,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_1_new\", \"acc\": 0.9340277777777778, \"vs\": \"text_abbrev\", \"diff\": 93.4, \"boot95\": [90.97, 95.49]}\n"}
|
| 70 |
+
,{"stream_name":"stdout","time":157.438547212,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_1_new\", \"acc\": 0.9340277777777778, \"vs\": \"text_csv\", \"diff\": 92.88, \"boot95\": [90.28, 95.14]}\n"}
|
| 71 |
+
,{"stream_name":"stdout","time":157.476708162,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_1_new\", \"acc\": 0.9340277777777778, \"vs\": \"text_values_csv\", \"diff\": 93.4, \"boot95\": [90.97, 95.49]}\n"}
|
| 72 |
+
,{"stream_name":"stdout","time":157.513927911,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_1_new\", \"acc\": 0.9340277777777778, \"vs\": \"text_values\", \"diff\": 93.4, \"boot95\": [90.97, 95.49]}\n"}
|
| 73 |
+
,{"stream_name":"stdout","time":157.551306283,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_2_new\", \"acc\": 0.9184027777777778, \"vs\": \"text_template\", \"diff\": -6.6, \"boot95\": [-9.72, -3.47]}\n"}
|
| 74 |
+
,{"stream_name":"stdout","time":157.587669908,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_2_new\", \"acc\": 0.9184027777777778, \"vs\": \"text_kv\", \"diff\": 86.63, \"boot95\": [82.47, 90.62]}\n"}
|
| 75 |
+
,{"stream_name":"stdout","time":157.625047843,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_2_new\", \"acc\": 0.9184027777777778, \"vs\": \"text_abbrev\", \"diff\": 91.84, \"boot95\": [89.06, 94.44]}\n"}
|
| 76 |
+
,{"stream_name":"stdout","time":157.66255768,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_2_new\", \"acc\": 0.9184027777777778, \"vs\": \"text_csv\", \"diff\": 91.32, \"boot95\": [88.37, 94.1]}\n"}
|
| 77 |
+
,{"stream_name":"stdout","time":157.700952112,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_2_new\", \"acc\": 0.9184027777777778, \"vs\": \"text_values_csv\", \"diff\": 91.84, \"boot95\": [89.06, 94.44]}\n"}
|
| 78 |
+
,{"stream_name":"stdout","time":157.739386102,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_2_new\", \"acc\": 0.9184027777777778, \"vs\": \"text_values\", \"diff\": 91.84, \"boot95\": [89.06, 94.44]}\n"}
|
| 79 |
+
,{"stream_name":"stdout","time":157.776225545,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_4_new\", \"acc\": 0.8420138888888888, \"vs\": \"text_template\", \"diff\": -10.07, \"boot95\": [-15.1, -5.38]}\n"}
|
| 80 |
+
,{"stream_name":"stdout","time":157.813558255,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_4_new\", \"acc\": 0.8420138888888888, \"vs\": \"text_kv\", \"diff\": 75.87, \"boot95\": [70.14, 81.25]}\n"}
|
| 81 |
+
,{"stream_name":"stdout","time":157.850692098,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_4_new\", \"acc\": 0.8420138888888888, \"vs\": \"text_abbrev\", \"diff\": 84.2, \"boot95\": [79.51, 88.37]}\n"}
|
| 82 |
+
,{"stream_name":"stdout","time":157.890013899,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_4_new\", \"acc\": 0.8420138888888888, \"vs\": \"text_csv\", \"diff\": 84.2, \"boot95\": [79.51, 88.37]}\n"}
|
| 83 |
+
,{"stream_name":"stdout","time":157.926849507,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_4_new\", \"acc\": 0.8420138888888888, \"vs\": \"text_values_csv\", \"diff\": 84.2, \"boot95\": [79.51, 88.37]}\n"}
|
| 84 |
+
,{"stream_name":"stdout","time":157.965024562,"data":"{\"route\": \"k2_mean3\", \"set\": \"sealed_4_new\", \"acc\": 0.8420138888888888, \"vs\": \"text_values\", \"diff\": 84.2, \"boot95\": [79.51, 88.37]}\n"}
|
| 85 |
+
,{"stream_name":"stdout","time":223.255143598,"data":"{\"route\": \"probes_k2_s17\", \"ridge_aug:sealed_ood\": 1.0, \"mlp_aug:sealed_ood\": 1.0, \"mlp_ctrl_shuffled:sealed_ood\": 0.0, \"mlp_cls_fit:sealed_ood\": 0.995, \"knn1:sealed_ood\": 0.0, \"knn5:sealed_ood\": 0.1325, \"ridge_aug:sealed_4_new\": 0.5833333333333334, \"mlp_aug:sealed_4_new\": 0.546875, \"mlp_ctrl_shuffled:sealed_4_new\": 0.0, \"mlp_cls_fit:sealed_4_new\": 0.0, \"knn1:sealed_4_new\": 0.0, \"knn5:sealed_4_new\": 0.0, \"ridge_aug:sealed_1_new\": 0.9322916666666666, \"mlp_aug:sealed_1_new\": 1.0, \"mlp_ctrl_shuffled:sealed_1_new\": 0.0, \"mlp_cls_fit:sealed_1_new\": 0.0, \"knn1:sealed_1_new\": 0.0, \"knn5:sealed_1_new\": 0.0}\n"}
|
| 86 |
+
,{"stream_name":"stdout","time":276.767804206,"data":"{\"route\": \"probes_k2_s29\", \"ridge_aug:sealed_ood\": 1.0, \"mlp_aug:sealed_ood\": 1.0, \"mlp_ctrl_shuffled:sealed_ood\": 0.0, \"mlp_cls_fit:sealed_ood\": 0.9975, \"knn1:sealed_ood\": 0.0, \"knn5:sealed_ood\": 0.14, \"ridge_aug:sealed_4_new\": 0.6822916666666666, \"mlp_aug:sealed_4_new\": 0.6458333333333334, \"mlp_ctrl_shuffled:sealed_4_new\": 0.0, \"mlp_cls_fit:sealed_4_new\": 0.0, \"knn1:sealed_4_new\": 0.0, \"knn5:sealed_4_new\": 0.0, \"ridge_aug:sealed_1_new\": 0.9270833333333334, \"mlp_aug:sealed_1_new\": 1.0, \"mlp_ctrl_shuffled:sealed_1_new\": 0.0, \"mlp_cls_fit:sealed_1_new\": 0.0, \"knn1:sealed_1_new\": 0.0, \"knn5:sealed_1_new\": 0.0}\n"}
|
| 87 |
+
,{"stream_name":"stdout","time":328.503478024,"data":"{\"route\": \"probes_k2_s43\", \"ridge_aug:sealed_ood\": 1.0, \"mlp_aug:sealed_ood\": 1.0, \"mlp_ctrl_shuffled:sealed_ood\": 0.0, \"mlp_cls_fit:sealed_ood\": 1.0, \"knn1:sealed_ood\": 0.0, \"knn5:sealed_ood\": 0.175, \"ridge_aug:sealed_4_new\": 0.671875, \"mlp_aug:sealed_4_new\": 0.5, \"mlp_ctrl_shuffled:sealed_4_new\": 0.0, \"mlp_cls_fit:sealed_4_new\": 0.0, \"knn1:sealed_4_new\": 0.0, \"knn5:sealed_4_new\": 0.0, \"ridge_aug:sealed_1_new\": 0.96875, \"mlp_aug:sealed_1_new\": 1.0, \"mlp_ctrl_shuffled:sealed_1_new\": 0.0, \"mlp_cls_fit:sealed_1_new\": 0.0, \"knn1:sealed_1_new\": 0.0, \"knn5:sealed_1_new\": 0.0}\n"}
|
| 88 |
+
,{"stream_name":"stdout","time":328.621673763,"data":"{\"probe_mean3\": \"ridge_aug\", \"set\": \"sealed_ood\", \"acc\": 1.0, \"llm_k2_acc\": 0.99, \"diff_vs_llm\": 1.0, \"boot95\": [0.42, 1.67]}\n"}
|
| 89 |
+
,{"stream_name":"stdout","time":328.726904848,"data":"{\"probe_mean3\": \"mlp_aug\", \"set\": \"sealed_ood\", \"acc\": 1.0, \"llm_k2_acc\": 0.99, \"diff_vs_llm\": 1.0, \"boot95\": [0.42, 1.67]}\n"}
|
| 90 |
+
,{"stream_name":"stdout","time":328.82347804,"data":"{\"probe_mean3\": \"mlp_ctrl_shuffled\", \"set\": \"sealed_ood\", \"acc\": 0.0, \"llm_k2_acc\": 0.99, \"diff_vs_llm\": -99.0, \"boot95\": [-99.58, -98.33]}\n"}
|
| 91 |
+
,{"stream_name":"stdout","time":328.914063195,"data":"{\"probe_mean3\": \"mlp_cls_fit\", \"set\": \"sealed_ood\", \"acc\": 0.9975, \"llm_k2_acc\": 0.99, \"diff_vs_llm\": 0.75, \"boot95\": [0.08, 1.5]}\n"}
|
| 92 |
+
,{"stream_name":"stdout","time":329.008640798,"data":"{\"probe_mean3\": \"knn1\", \"set\": \"sealed_ood\", \"acc\": 0.0, \"llm_k2_acc\": 0.99, \"diff_vs_llm\": -99.0, \"boot95\": [-99.58, -98.33]}\n"}
|
| 93 |
+
,{"stream_name":"stdout","time":329.105495539,"data":"{\"probe_mean3\": \"knn5\", \"set\": \"sealed_ood\", \"acc\": 0.14916666666666667, \"llm_k2_acc\": 0.99, \"diff_vs_llm\": -84.08, \"boot95\": [-87.0, -81.08]}\n"}
|
| 94 |
+
,{"stream_name":"stdout","time":329.145507064,"data":"{\"probe_mean3\": \"ridge_aug\", \"set\": \"sealed_4_new\", \"acc\": 0.6458333333333334, \"llm_k2_acc\": 0.8420138888888888, \"diff_vs_llm\": -19.62, \"boot95\": [-26.74, -12.5]}\n"}
|
| 95 |
+
,{"stream_name":"stdout","time":329.183671603,"data":"{\"probe_mean3\": \"mlp_aug\", \"set\": \"sealed_4_new\", \"acc\": 0.564236111111111, \"llm_k2_acc\": 0.8420138888888888, \"diff_vs_llm\": -27.78, \"boot95\": [-35.24, -20.14]}\n"}
|
| 96 |
+
,{"stream_name":"stdout","time":329.222185721,"data":"{\"probe_mean3\": \"mlp_ctrl_shuffled\", \"set\": \"sealed_4_new\", \"acc\": 0.0, \"llm_k2_acc\": 0.8420138888888888, \"diff_vs_llm\": -84.2, \"boot95\": [-88.54, -79.69]}\n"}
|
| 97 |
+
,{"stream_name":"stdout","time":329.259354621,"data":"{\"probe_mean3\": \"mlp_cls_fit\", \"set\": \"sealed_4_new\", \"acc\": 0.0, \"llm_k2_acc\": 0.8420138888888888, \"diff_vs_llm\": -84.2, \"boot95\": [-88.54, -79.69]}\n"}
|
| 98 |
+
,{"stream_name":"stdout","time":329.296126324,"data":"{\"probe_mean3\": \"knn1\", \"set\": \"sealed_4_new\", \"acc\": 0.0, \"llm_k2_acc\": 0.8420138888888888, \"diff_vs_llm\": -84.2, \"boot95\": [-88.54, -79.69]}\n"}
|
| 99 |
+
,{"stream_name":"stdout","time":329.334319863,"data":"{\"probe_mean3\": \"knn5\", \"set\": \"sealed_4_new\", \"acc\": 0.0, \"llm_k2_acc\": 0.8420138888888888, \"diff_vs_llm\": -84.2, \"boot95\": [-88.54, -79.69]}\n"}
|
| 100 |
+
,{"stream_name":"stdout","time":329.37332952,"data":"{\"probe_mean3\": \"ridge_aug\", \"set\": \"sealed_1_new\", \"acc\": 0.9427083333333334, \"llm_k2_acc\": 0.9340277777777778, \"diff_vs_llm\": 0.87, \"boot95\": [-2.95, 4.51]}\n"}
|
| 101 |
+
,{"stream_name":"stdout","time":329.411147075,"data":"{\"probe_mean3\": \"mlp_aug\", \"set\": \"sealed_1_new\", \"acc\": 1.0, \"llm_k2_acc\": 0.9340277777777778, \"diff_vs_llm\": 6.6, \"boot95\": [4.51, 8.85]}\n"}
|
| 102 |
+
,{"stream_name":"stdout","time":329.44810494,"data":"{\"probe_mean3\": \"mlp_ctrl_shuffled\", \"set\": \"sealed_1_new\", \"acc\": 0.0, \"llm_k2_acc\": 0.9340277777777778, \"diff_vs_llm\": -93.4, \"boot95\": [-95.49, -91.15]}\n"}
|
| 103 |
+
,{"stream_name":"stdout","time":329.485734684,"data":"{\"probe_mean3\": \"mlp_cls_fit\", \"set\": \"sealed_1_new\", \"acc\": 0.0, \"llm_k2_acc\": 0.9340277777777778, \"diff_vs_llm\": -93.4, \"boot95\": [-95.49, -91.15]}\n"}
|
| 104 |
+
,{"stream_name":"stdout","time":329.523359846,"data":"{\"probe_mean3\": \"knn1\", \"set\": \"sealed_1_new\", \"acc\": 0.0, \"llm_k2_acc\": 0.9340277777777778, \"diff_vs_llm\": -93.4, \"boot95\": [-95.49, -91.15]}\n"}
|
| 105 |
+
,{"stream_name":"stdout","time":329.561016123,"data":"{\"probe_mean3\": \"knn5\", \"set\": \"sealed_1_new\", \"acc\": 0.0, \"llm_k2_acc\": 0.9340277777777778, \"diff_vs_llm\": -93.4, \"boot95\": [-95.49, -91.15]}\n"}
|
| 106 |
+
,{"stream_name":"stdout","time":342.977856481,"data":"{\"control\": \"random_receiver\", \"set\": \"sealed_ood\", \"text_template\": 0.0, \"k2_s17\": 0.0, \"k2_s29\": 0.0, \"k2_s43\": 0.0}\n"}
|
| 107 |
+
,{"stream_name":"stdout","time":349.383162888,"data":"{\"control\": \"random_receiver\", \"set\": \"sealed_4_new\", \"text_template\": 0.0, \"k2_s17\": 0.0, \"k2_s29\": 0.0, \"k2_s43\": 0.0}\n"}
|
| 108 |
+
,{"stream_name":"stderr","time":357.249388448,"data":"/usr/local/lib/python3.13/dist-packages/mistune.py:435: SyntaxWarning: invalid escape sequence '\\|'\n"}
|
| 109 |
+
,{"stream_name":"stderr","time":357.249427362,"data":" cells[i][c] = re.sub('\\\\\\\\\\|', '|', cell)\n"}
|
| 110 |
+
,{"stream_name":"stderr","time":357.476570044,"data":"/usr/local/lib/python3.13/dist-packages/nbconvert/filters/filter_links.py:36: SyntaxWarning: invalid escape sequence '\\_'\n"}
|
| 111 |
+
,{"stream_name":"stderr","time":357.47660111,"data":" text = re.sub(r'_', '\\_', text) # Escape underscores in display text\n"}
|
| 112 |
+
,{"stream_name":"stderr","time":358.244424653,"data":"[NbConvertApp] Converting notebook __script__.ipynb to html\n"}
|
| 113 |
+
,{"stream_name":"stderr","time":358.758158185,"data":"[NbConvertApp] Writing 278746 bytes to __results__.html\n"}
|
| 114 |
+
]
|
020/eval020.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CUERDAS-020: sealed test (evaluated once), compact text baselines, stronger no-LLM decoders.
|
| 2 |
+
import os, sys, json, hashlib, random, math, platform
|
| 3 |
+
import torch, torch.nn as nn, torch.nn.functional as F, transformers
|
| 4 |
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 5 |
+
import run018 as R
|
| 6 |
+
from transformers import AutoModelForCausalLM, AutoConfig
|
| 7 |
+
INP = os.environ['INP']; OD = os.environ['OD']; NBOOT = 10000; SEEDS = [17, 29, 43]
|
| 8 |
+
raw = open(f'{INP}/sealed020.json').read()
|
| 9 |
+
H = hashlib.sha256(raw.encode()).hexdigest(); assert H == '4c7737f66364a44786b4d9acb4686a6f31829d78cb5914f19eca1df74fe73393', H
|
| 10 |
+
S = json.loads(raw); NEW = S['new_words']
|
| 11 |
+
SUMM = open(f'{OD}/results020.jsonl', 'w'); PRED = open(f'{OD}/predictions020.jsonl', 'w')
|
| 12 |
+
def log(d): SUMM.write(json.dumps(d) + '\n'); SUMM.flush(); print(json.dumps(d), flush=True)
|
| 13 |
+
log({'env': {'python': platform.python_version(), 'torch': torch.__version__, 'transformers': transformers.__version__, 'device': torch.cuda.get_device_name() if R.DEV == 'cuda' else 'cpu'}, 'sealed_sha256': H})
|
| 14 |
+
for j in range(4):
|
| 15 |
+
for w in NEW[j]: assert len(R.t1(w)) == 1, w
|
| 16 |
+
CAND = [torch.tensor(sorted({R.t1(w)[0] for w in R.V[j] + NEW[j]})) for j in range(4)]
|
| 17 |
+
assert all(len(CAND[j]) == len(R.V[j]) + 8 for j in range(4))
|
| 18 |
+
def gidx(j, w): return int((CAND[j] == R.t1(w)[0]).nonzero()[0])
|
| 19 |
+
SETS = ['sealed_id', 'sealed_ood', 'sealed_1_new', 'sealed_2_new', 'sealed_4_new']
|
| 20 |
+
FMT = {'template': R.TPL, 'kv': "action: {action}; target: {target}; status: {status}; window: {window}.",
|
| 21 |
+
'abbrev': "act={action} tgt={target} st={status} win={window}", 'csv': "action,target,status,window\n{action},{target},{status},{window}",
|
| 22 |
+
'values_csv': "{action}, {target}, {status}, {window}", 'values': "{action} {target} {status} {window}"}
|
| 23 |
+
def text_logits(msgs, fmt):
|
| 24 |
+
embs, mt = [], []
|
| 25 |
+
for i, m in enumerate(msgs):
|
| 26 |
+
for j in range(4):
|
| 27 |
+
embs.append(R.Emb[R.tok.encode(fmt.format(**dict(zip(R.FIELDS, m))) + R.SUF_MAIN.format(field=R.FIELDS[j]), add_special_tokens=False)]); mt.append((i, j))
|
| 28 |
+
return R.run_rows(embs, None), mt
|
| 29 |
+
@torch.no_grad()
|
| 30 |
+
def per_ex(msgs, fmt=None, cod=None, bs=24):
|
| 31 |
+
out = []
|
| 32 |
+
for b in range(0, len(msgs), bs):
|
| 33 |
+
mb = msgs[b:b+bs]; lg, mt = text_logits(mb, fmt) if cod is None else R.codec_logits(cod, mb, R.SUF_MAIN)
|
| 34 |
+
good = [True] * len(mb)
|
| 35 |
+
for r, (i, j) in enumerate(mt): good[i] &= int(lg[r][CAND[j]].argmax()) == gidx(j, mb[i][j])
|
| 36 |
+
out += good
|
| 37 |
+
return [float(g) for g in out]
|
| 38 |
+
def wilson(k, n, z=1.96):
|
| 39 |
+
p = k / n; d = 1 + z*z/n; c = (p + z*z/(2*n)) / d; h = z * math.sqrt(p*(1-p)/n + z*z/(4*n*n)) / d
|
| 40 |
+
return [round(100*(c-h), 1), round(100*(c+h), 1)]
|
| 41 |
+
def boot(a, b, seed=0):
|
| 42 |
+
g = torch.Generator().manual_seed(seed); a = torch.tensor(a, dtype=torch.float64, device='cpu'); b = torch.tensor(b, dtype=torch.float64, device='cpu')
|
| 43 |
+
idx = torch.randint(0, len(a), (NBOOT, len(a)), generator=g, device='cpu'); d = (a[idx].mean(1) - b[idx].mean(1)) * 100
|
| 44 |
+
return round(float((a.mean()-b.mean())*100), 2), [round(float(d.quantile(.025)), 2), round(float(d.quantile(.975)), 2)]
|
| 45 |
+
def positions(msgs, fmt): return sum(len(R.tok.encode(fmt.format(**dict(zip(R.FIELDS, m))), add_special_tokens=False)) for m in msgs) / len(msgs) + 15
|
| 46 |
+
# ---- text formats ----
|
| 47 |
+
TXT = {}
|
| 48 |
+
for fn, fmt in FMT.items():
|
| 49 |
+
for s in SETS:
|
| 50 |
+
r = per_ex(S[s], fmt); TXT[(fn, s)] = r; k = int(sum(r))
|
| 51 |
+
for i, c in enumerate(r): PRED.write(json.dumps({'route': 'text_' + fn, 'set': s, 'i': i, 'correct': bool(c)}) + '\n')
|
| 52 |
+
e = {'route': 'text_' + fn, 'set': s, 'n': len(r), 'acc': k/len(r), 'wilson95': wilson(k, len(r)), 'positions_per_event': round(positions(S[s], fmt), 2)}
|
| 53 |
+
if fn != 'template': e['diff_vs_template'], e['boot95'] = boot(r, TXT[('template', s)], seed=7)
|
| 54 |
+
log(e)
|
| 55 |
+
# ---- K=2, sealed, evaluated once ----
|
| 56 |
+
cods = {}; K2 = {}
|
| 57 |
+
for sd in SEEDS:
|
| 58 |
+
cod = R.PackK(2, sd); cod.load_state_dict(torch.load(f'{INP}/k2long_s{sd}.pt', map_location=R.DEV)); cod.eval().requires_grad_(False); cods[sd] = cod
|
| 59 |
+
for s in SETS:
|
| 60 |
+
r = per_ex(S[s], cod=cod); K2[(sd, s)] = r; k = int(sum(r))
|
| 61 |
+
for i, c in enumerate(r): PRED.write(json.dumps({'route': f'k2_s{sd}', 'set': s, 'i': i, 'correct': bool(c)}) + '\n')
|
| 62 |
+
d, ci = boot(r, TXT[('template', s)], seed=sd)
|
| 63 |
+
log({'route': f'k2_s{sd}', 'set': s, 'n': len(r), 'acc': k/len(r), 'wilson95': wilson(k, len(r)), 'positions_per_event': 17, 'diff_vs_template': d, 'boot95': ci})
|
| 64 |
+
for s in SETS:
|
| 65 |
+
a = [sum(K2[(sd, s)][i] for sd in SEEDS)/3 for i in range(len(S[s]))]
|
| 66 |
+
for fn in FMT:
|
| 67 |
+
d, ci = boot(a, TXT[(fn, s)], seed=1); log({'route': 'k2_mean3', 'set': s, 'acc': sum(a)/len(a), 'vs': 'text_' + fn, 'diff': d, 'boot95': ci})
|
| 68 |
+
# ---- no-LLM decoders on K=2 slots ----
|
| 69 |
+
def slots(cod, ms):
|
| 70 |
+
with torch.no_grad(): return torch.cat([cod(torch.stack([torch.stack([R.lex(w) for w in m]) for m in ms[i:i+512]])).flatten(1) for i in range(0, len(ms), 512)])
|
| 71 |
+
g = random.Random(2020); AUG = []
|
| 72 |
+
for _ in range(20000):
|
| 73 |
+
m = list(g.choice(R.FIT))
|
| 74 |
+
for j in range(4):
|
| 75 |
+
if g.random() < .4: m[j] = g.choice(R.POOL)
|
| 76 |
+
AUG.append(m)
|
| 77 |
+
YA = torch.stack([torch.stack([R.lex(w) for w in m]) for m in AUG]) # [N,4,D]
|
| 78 |
+
CE = [F.normalize(R.Emb[CAND[j]], dim=-1) for j in range(4)]
|
| 79 |
+
def dec_emb(P, ms): # P [n,4,D] predicted value embeddings -> exact-match
|
| 80 |
+
ok = torch.ones(len(ms), dtype=torch.bool, device='cpu')
|
| 81 |
+
for j in range(4): ok &= ((F.normalize(P[:, j], dim=-1) @ CE[j].T).argmax(1).cpu() == torch.tensor([gidx(j, m[j]) for m in ms], device='cpu'))
|
| 82 |
+
return ok.float().tolist()
|
| 83 |
+
def train_mlp(X, Y, shuffle=False, steps=4000, seed=0):
|
| 84 |
+
torch.manual_seed(seed); Y = Y[torch.randperm(len(Y))] if shuffle else Y
|
| 85 |
+
mu, sdv = X.mean(0), X.std(0) + 1e-6
|
| 86 |
+
net = nn.Sequential(nn.Linear(X.shape[1], 1024), nn.GELU(), nn.Linear(1024, 1024), nn.GELU(), nn.Linear(1024, 4*R.D)); opt = torch.optim.Adam(net.parameters(), 1e-3)
|
| 87 |
+
for _ in range(steps):
|
| 88 |
+
b = torch.randint(0, len(X), (256,)); p = net((X[b]-mu)/sdv).view(-1, 4, R.D)
|
| 89 |
+
loss = (1 - F.cosine_similarity(p, Y[b], -1)).mean(); opt.zero_grad(); loss.backward(); opt.step()
|
| 90 |
+
net.eval(); return lambda Z: net((Z-mu)/sdv).view(-1, 4, R.D).detach()
|
| 91 |
+
PROBE = {}
|
| 92 |
+
for sd in SEEDS:
|
| 93 |
+
cod = cods[sd]; XA = slots(cod, AUG); XF = slots(cod, R.FIT); res = {'route': f'probes_k2_s{sd}'}
|
| 94 |
+
A = torch.linalg.solve(XA.T@XA + 10*torch.eye(XA.shape[1]), XA.T@YA.flatten(1)); ridge = lambda Z: (Z@A).view(-1, 4, R.D)
|
| 95 |
+
mlp = train_mlp(XA, YA, seed=sd); ctrl = train_mlp(XA, YA, shuffle=True, seed=sd)
|
| 96 |
+
YF = torch.tensor([[R.V[j].index(m[j]) for j in range(4)] for m in R.FIT])
|
| 97 |
+
torch.manual_seed(sd); cls = nn.Sequential(nn.Linear(XF.shape[1], 1024), nn.GELU(), nn.Linear(1024, sum(len(v) for v in R.V))); opt = torch.optim.Adam(cls.parameters(), 1e-3)
|
| 98 |
+
off = [0, 12, 24, 40, 50]; muF, sdF = XF.mean(0), XF.std(0)+1e-6
|
| 99 |
+
for _ in range(2000):
|
| 100 |
+
b = torch.randint(0, len(XF), (128,)); o = cls((XF[b]-muF)/sdF); loss = sum(F.cross_entropy(o[:, off[j]:off[j+1]], YF[b, j]) for j in range(4)); opt.zero_grad(); loss.backward(); opt.step()
|
| 101 |
+
XFn = F.normalize(XF, dim=-1)
|
| 102 |
+
for s in ('sealed_ood', 'sealed_4_new', 'sealed_1_new'):
|
| 103 |
+
ms = S[s]; Z = slots(cod, ms)
|
| 104 |
+
for name, fn in (('ridge_aug', ridge), ('mlp_aug', mlp), ('mlp_ctrl_shuffled', ctrl)):
|
| 105 |
+
r = dec_emb(fn(Z), ms); PROBE[(sd, name, s)] = r; res[f'{name}:{s}'] = sum(r)/len(r)
|
| 106 |
+
with torch.no_grad(): o = cls((Z-muF)/sdF)
|
| 107 |
+
okc = []; okk1 = []; okk5 = []; sim = F.normalize(Z, dim=-1) @ XFn.T; top = sim.topk(5, dim=1).indices.cpu()
|
| 108 |
+
for i, m in enumerate(ms):
|
| 109 |
+
okc.append(float(all(m[j] in R.V[j] and int(o[i, off[j]:off[j+1]].argmax()) == R.V[j].index(m[j]) for j in range(4))))
|
| 110 |
+
n1 = R.FIT[int(top[i, 0])]; okk1.append(float(all(n1[j] == m[j] for j in range(4))))
|
| 111 |
+
vote = [max(set(R.FIT[int(t)][j] for t in top[i]), key=lambda v: sum(R.FIT[int(t)][j] == v for t in top[i])) for j in range(4)]
|
| 112 |
+
okk5.append(float(all(vote[j] == m[j] for j in range(4))))
|
| 113 |
+
for name, r in (('mlp_cls_fit', okc), ('knn1', okk1), ('knn5', okk5)): PROBE[(sd, name, s)] = r; res[f'{name}:{s}'] = sum(r)/len(r)
|
| 114 |
+
log(res)
|
| 115 |
+
for s in ('sealed_ood', 'sealed_4_new', 'sealed_1_new'):
|
| 116 |
+
for name in ('ridge_aug', 'mlp_aug', 'mlp_ctrl_shuffled', 'mlp_cls_fit', 'knn1', 'knn5'):
|
| 117 |
+
a = [sum(PROBE[(sd, name, s)][i] for sd in SEEDS)/3 for i in range(len(S[s]))]; l = [sum(K2[(sd, s)][i] for sd in SEEDS)/3 for i in range(len(S[s]))]
|
| 118 |
+
d, ci = boot(a, l, seed=3); log({'probe_mean3': name, 'set': s, 'acc': sum(a)/len(a), 'llm_k2_acc': sum(l)/len(l), 'diff_vs_llm': d, 'boot95': ci})
|
| 119 |
+
# ---- random-weight receiver ----
|
| 120 |
+
torch.manual_seed(0)
|
| 121 |
+
rnd = AutoModelForCausalLM.from_config(AutoConfig.from_pretrained(R.MP)).float().eval().requires_grad_(False).to(R.DEV)
|
| 122 |
+
rnd.model.embed_tokens.weight.data.copy_(R.Emb.data); R.lm = rnd
|
| 123 |
+
for s in ('sealed_ood', 'sealed_4_new'):
|
| 124 |
+
e = {'control': 'random_receiver', 'set': s, 'text_template': sum(per_ex(S[s], R.TPL))/len(S[s])}
|
| 125 |
+
for sd in SEEDS: e[f'k2_s{sd}'] = sum(per_ex(S[s], cod=cods[sd]))/len(S[s])
|
| 126 |
+
log(e)
|
| 127 |
+
SUMM.close(); PRED.close()
|
020/kaggle_kernel.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os,subprocess,sys,glob,shutil
|
| 2 |
+
subprocess.run([sys.executable,'-m','pip','install','-q','transformers==4.46.3'],check=True)
|
| 3 |
+
inp=os.path.dirname(glob.glob('/kaggle/input/**/k2long_s17.pt',recursive=True)[0])
|
| 4 |
+
S='/kaggle/working/src'; os.makedirs(S+'/data',exist_ok=True); shutil.copy(f'{inp}/expanded.json',f'{S}/data/')
|
| 5 |
+
for f in ('run018.py','eval020.py'): shutil.copy(f'{inp}/{f}','/kaggle/working/')
|
| 6 |
+
shutil.copy(f'{inp}/sealed020.json','/kaggle/working/')
|
| 7 |
+
from transformers import AutoTokenizer,AutoModelForCausalLM
|
| 8 |
+
mp='/kaggle/working/m135'; R='12fd25f77366fa6b3b4b768ec3050bf629380bac'
|
| 9 |
+
AutoTokenizer.from_pretrained('HuggingFaceTB/SmolLM2-135M-Instruct',revision=R).save_pretrained(mp); AutoModelForCausalLM.from_pretrained('HuggingFaceTB/SmolLM2-135M-Instruct',revision=R).save_pretrained(mp)
|
| 10 |
+
subprocess.run([sys.executable,'/kaggle/working/eval020.py'],env=dict(os.environ,SRC=S,MP=mp,OD='/kaggle/working',INP=inp),check=True)
|
| 11 |
+
shutil.rmtree(mp); shutil.rmtree(S); os.remove('/kaggle/working/run018.py')
|
020/make_sealed020.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CUERDAS-020: sealed test generator. Fresh seed, never seen by any earlier iteration.
|
| 2 |
+
import json, random, itertools, re, hashlib, sys
|
| 3 |
+
from transformers import AutoTokenizer
|
| 4 |
+
tok = AutoTokenizer.from_pretrained(sys.argv[1]); SEED = 20261016
|
| 5 |
+
meta = json.load(open(sys.argv[2])); V = meta['vocabulary']; rows = meta['rows']; SPL = meta['splits']
|
| 6 |
+
def t1(w): return tok.encode(' ' + w, add_special_tokens=False)
|
| 7 |
+
used = {tuple(r['values']) for r in rows}
|
| 8 |
+
fit = [tuple(rows[i]['values']) for i in SPL['fit']]; seen_at = {v[:2] for v in fit}; seen_sw = {v[2:] for v in fit}
|
| 9 |
+
# replicate the 015-019 fresh test (seed 15015) so it is excluded
|
| 10 |
+
rng = random.Random(15015); uni = [v for v in itertools.product(*[range(len(x)) for x in V]) if v not in used]; rng.shuffle(uni)
|
| 11 |
+
isid = lambda v: v[:2] in seen_at and v[2:] in seen_sw
|
| 12 |
+
fresh = set([v for v in uni if isid(v)][:200] + [v for v in uni if not isid(v)][:400])
|
| 13 |
+
S013 = (7, 7, 8, 5)
|
| 14 |
+
elig = [v for v in itertools.product(*[range(len(x)) for x in V]) if v not in used and v not in fresh and not all(v[j] < S013[j] for j in range(4))]
|
| 15 |
+
r = random.Random(SEED); r.shuffle(elig)
|
| 16 |
+
sid = [v for v in elig if isid(v)][:200]; sood = [v for v in elig if not isid(v)][:400]
|
| 17 |
+
W = lambda v: [V[j][v[j]] for j in range(4)]
|
| 18 |
+
# unseen words: single-token, >=10 lowercase letters (the 018 training pool only has 3-9 letters, so these can never have been trained on)
|
| 19 |
+
OLD_NEW = 'inspect repair cancel approve archive scan print send sensor battery printer server truck door screen cable broken stable late empty full warm missing urgent night evening weekend noon spring summer winter midnight'.split()
|
| 20 |
+
cand = sorted(t[1:] for t in tok.get_vocab() if t.startswith('Ġ') and re.fullmatch(r'[a-z]{10,14}', t[1:]))
|
| 21 |
+
r.shuffle(cand); usedtok = {t1(w)[0] for vs in V for w in vs}; NEW = [[] for _ in range(4)]
|
| 22 |
+
for w in cand:
|
| 23 |
+
j = min(range(4), key=lambda k: len(NEW[k]))
|
| 24 |
+
if len(NEW[j]) >= 8: break
|
| 25 |
+
if len(t1(w)) == 1 and t1(w)[0] not in usedtok and w not in OLD_NEW: NEW[j].append(w); usedtok.add(t1(w)[0])
|
| 26 |
+
def novel(n_new, n=192):
|
| 27 |
+
out = []
|
| 28 |
+
for k in range(n):
|
| 29 |
+
b = W(sood[k % 400]); js = r.sample(range(4), n_new)
|
| 30 |
+
for j in js: b[j] = r.choice(NEW[j])
|
| 31 |
+
out.append(b)
|
| 32 |
+
return out
|
| 33 |
+
SEALED = {'seed': SEED, 'n_eligible_universe': len(elig) + len(sid) * 0, 'new_words': NEW,
|
| 34 |
+
'sealed_id': [W(v) for v in sid], 'sealed_ood': [W(v) for v in sood],
|
| 35 |
+
'sealed_1_new': novel(1), 'sealed_2_new': novel(2), 'sealed_4_new': novel(4)}
|
| 36 |
+
s = json.dumps(SEALED, sort_keys=True); open(sys.argv[3], 'w').write(s)
|
| 37 |
+
print('sha256', hashlib.sha256(s.encode()).hexdigest(), 'eligible', len(elig), {k: len(v) for k, v in SEALED.items() if isinstance(v, list)})
|
020/predictions020.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
020/preregistro020.md
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CUERDAS-020 — Pre-registro (escrito ANTES de cualquier evaluación)
|
| 2 |
+
|
| 3 |
+
Fecha: 2026-10-16 (America/Mexico_City). Autor: Erik Sanchez (con Grok Bot).
|
| 4 |
+
|
| 5 |
+
## Conjunto sellado
|
| 6 |
+
- Generado por `make_sealed020.py` con semilla nueva **20261016** (nunca usada antes).
|
| 7 |
+
- Universo elegible: combinaciones del vocabulario 014 que NO están en filas 014, NO están en el test fresco 015–019 (semilla 15015) y NO están en el universo 013 → 17,494 combinaciones.
|
| 8 |
+
- `sealed_id` (200), `sealed_ood` (400, con al menos un par retenido), `sealed_1_new` / `sealed_2_new` / `sealed_4_new` (192 c/u; palabras nuevas de 1 token con ≥10 letras, imposibles en el pool de entrenamiento 018 que solo tiene 3–9 letras).
|
| 9 |
+
- Archivo `kd/sealed020.json`, **SHA-256 4c7737f66364a44786b4d9acb4686a6f31829d78cb5914f19eca1df74fe73393**. El kernel verifica el hash antes de evaluar.
|
| 10 |
+
- Opciones por campo (cerradas): vocabulario 014 del campo + las 8 palabras nuevas del campo.
|
| 11 |
+
- Se evalúa **una sola vez**, con los checkpoints 018 `k2long_s{17,29,43}.pt` (elegidos solo con validación en 018). Sin reentrenar, sin elegir nada después.
|
| 12 |
+
|
| 13 |
+
## Métricas
|
| 14 |
+
- Exactitud exact-match (los 4 campos), lectura cerrada de primer token, consulta por defecto. IC Wilson 95%.
|
| 15 |
+
- Diferencia K=2 − plantilla de texto (pp), bootstrap pareado 10,000 remuestreos, por semilla y media de 3 semillas.
|
| 16 |
+
- Posiciones por evento = tokens del mensaje + 15 (7 compartidos + 4×2 colas); K=2 = 17.
|
| 17 |
+
|
| 18 |
+
## Experimento 2: baselines de texto compactos (receptor congelado, zero-shot)
|
| 19 |
+
`template` (20 tokens de 014), `kv` = "action: a; target: t; status: s; window: w.", `abbrev` = "act=a tgt=t st=s win=w", `csv` = "action,target,status,window\na,t,s,w", `values_csv` = "a, t, s, w", `values` = "a t s w". Mismas consultas, opciones y conjuntos sellados.
|
| 20 |
+
|
| 21 |
+
## Experimento 3: decodificadores sin LLM sobre las ranuras K=2 (hiperparámetros fijados aquí, sin ajuste)
|
| 22 |
+
- Entrenamiento: 20,000 mensajes de FIT con aumento de palabras aleatorias del pool de entrenamiento 018 (p=0.4 por campo, igual que el codificador). Objetivo: embedding léxico del valor de cada campo. Decodificación: coseno contra las opciones.
|
| 23 |
+
- `ridge_aug` (λ=10); `mlp_aug` (1152→1024→GELU→1024→GELU→4×576, Adam lr 1e-3, 4,000 pasos, lote 256, pérdida coseno); `mlp_ctrl` = igual pero con objetivos barajados (control de Hewitt & Liang); `knn1`/`knn5` = vecinos más cercanos (coseno) sobre las ranuras de FIT (solo pueden emitir valores conocidos); `mlp_cls_fit` = clasificador MLP sobre FIT sin aumento (solo valores conocidos).
|
| 24 |
+
- Receptor con pesos aleatorios (semilla 0, mismo léxico y cabeza): texto y K=2 en `sealed_ood` y `sealed_4_new`.
|
| 25 |
+
|
| 26 |
+
## Reglas de decisión (fijadas antes)
|
| 27 |
+
1. **Test sellado**: la ventaja "K=2 ≈ texto" sobrevive si la media de 3 semillas en `sealed_ood` está a ≥ −2 pp de la plantilla Y el límite inferior del bootstrap de la media es > −3 pp.
|
| 28 |
+
2. **Baselines compactos**: la ventaja en posiciones "sobrevive" frente al mejor formato de texto que esté a ≥ −2 pp de la plantilla en `sealed_ood`; se reporta el ahorro K=2 vs ese formato (posiciones y %). Si ese formato también gana en palabras nuevas, se dice.
|
| 29 |
+
3. **Lookup sin LLM**: si algún decodificador sin LLM (excepto el control) alcanza ≥ la exactitud del LLM con K=2 en `sealed_4_new` (media de semillas), se concluye que la información es recuperable sin el LLM y se retira la afirmación de "no-lookup". Si queda muy por debajo, solo se dice que *estos* decodificadores fallan.
|
| 30 |
+
Todo resultado se reporta, sea bueno o malo.
|
020/resultados020.md
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CUERDAS-020 — Test sellado, baselines de texto compactos y decodificadores más fuertes
|
| 2 |
+
|
| 3 |
+
Fecha: 16 oct 2026 (hora CDMX). Kernel privado de Kaggle `cuerdas020-sealed` (Tesla T4, PyTorch 2.11.0+cu128, transformers 4.46.3, Python 3.13.15), ≈6 min de GPU. Sin entrenamiento: se usaron los checkpoints 018 `k2long_s{17,29,43}.pt`, elegidos en 018 solo con validación.
|
| 4 |
+
|
| 5 |
+
Pre-registro: `preregistro020.md` (escrito antes de correr; SHA-256 5bc64fb6…). Conjunto sellado `sealed020.json`, SHA-256 `4c7737f6…3393`, verificado por el kernel antes de evaluar. **Se evaluó una sola vez.** Datos crudos: `gpu/results020.jsonl`, `gpu/predictions020.jsonl`.
|
| 6 |
+
|
| 7 |
+
## 1. Test sellado (semilla nueva 20261016, nunca vista)
|
| 8 |
+
|
| 9 |
+
Exactitud exact-match (%), lectura cerrada de primer token, consulta por defecto. IC Wilson 95% entre corchetes. Opciones por campo = vocabulario 014 + 8 palabras nuevas.
|
| 10 |
+
|
| 11 |
+
| Conjunto (n) | Texto plantilla (35 pos.) | K=2 s17 | K=2 s29 | K=2 s43 | K=2 media | Δ media vs texto [bootstrap 95%] |
|
| 12 |
+
|---|---|---|---|---|---|---|
|
| 13 |
+
| sealed_id (200) | 99.5 [97.2, 99.9] | 98.5 [95.7, 99.5] | 100 [98.1, 100] | 98.0 [95.0, 99.2] | 98.83 | −0.67 [−2.00, +0.83] |
|
| 14 |
+
| **sealed_ood (400)** | **99.25 [97.8, 99.7]** | 99.0 [97.5, 99.6] | 99.75 [98.6, 100] | 98.25 [96.4, 99.1] | **99.0** | **−0.25 [−1.25, +0.92]** |
|
| 15 |
+
| sealed_1_new (192) | 96.4 [92.7, 98.2] | 95.8 | 98.4 | 85.9 | 93.4 | −2.95 [−6.08, +0.35] |
|
| 16 |
+
| sealed_2_new (192) | 98.4 [95.5, 99.5] | 90.6 | 97.9 | 87.0 | 91.8 | −6.60 [−9.72, −3.47] |
|
| 17 |
+
| sealed_4_new (192) | 94.3 [90.0, 96.8] | 82.8 [76.8, 87.5] | 87.5 [82.1, 91.5] | 82.3 [76.3, 87.0] | 84.2 | −10.07 [−15.10, −5.38] |
|
| 18 |
+
|
| 19 |
+
Por semilla en sealed_ood: s17 −0.25 [−1.5, +1.0]; s29 +0.5 [−0.5, +1.5]; s43 −1.0 [−2.5, +0.5] pp.
|
| 20 |
+
|
| 21 |
+
**Regla 1 (pre-registrada): se cumple.** La media en sealed_ood está a −0.25 pp del texto (≥ −2 pp) y el límite inferior del bootstrap (−1.25) es > −3 pp. La paridad con la plantilla en combinaciones nuevas de palabras conocidas **sobrevive** en un test que nadie había visto. Sigue sin ser una prueba de equivalencia formal (modelos fijos, 3 semillas).
|
| 22 |
+
|
| 23 |
+
Palabras nuevas: la brecha sigue siendo real (−3 a −10 pp), aunque menor que en desarrollo (4 nuevas: 84% sellado vs 55–73% en dev). Las palabras nuevas selladas son largas (≥10 letras) y de otro tipo que las de desarrollo, así que no se debe leer como mejora.
|
| 24 |
+
|
| 25 |
+
## 2. Baselines de texto compactos (receptor congelado, zero-shot)
|
| 26 |
+
|
| 27 |
+
| Formato | Posiciones/evento (sealed_ood) | sealed_ood | sealed_id | sealed_4_new |
|
| 28 |
+
|---|---:|---:|---:|---:|
|
| 29 |
+
| plantilla 014 ("The action is …") | 35.0 | **99.25** | 99.5 | 94.3 |
|
| 30 |
+
| kv ("action: …; target: …") | 31.0 | 2.5 | 6.5 | 8.3 |
|
| 31 |
+
| csv con encabezado | 30.6 | 0.5 | 3.0 | 0.0 |
|
| 32 |
+
| abbrev ("act=… tgt=…") | 27.6 | 0.0 | 3.5 | 0.0 |
|
| 33 |
+
| values_csv ("a, t, s, w") | 22.0 | 0.25 | 3.0 | 0.0 |
|
| 34 |
+
| values ("a t s w") | 19.0 | 0.25 | 0.5 | 0.0 |
|
| 35 |
+
| **K=2 (media 3 semillas)** | **17** | **99.0** | 98.8 | 84.2 |
|
| 36 |
+
|
| 37 |
+
**Regla 2: ningún formato compacto queda a ≤2 pp de la plantilla**, así que el ahorro de posiciones de K=2 se mide contra la plantilla (35 → 17, −51%). Pero hay que decirlo claro: los formatos compactos fallan porque este receptor de 135M, zero-shot y con la consulta "According to the record, the {field} is", no sabe leerlos; no porque la información falte. La comparación **no es justa en entrenamiento**: el codificador K=2 se entrenó, el texto no. Un formato compacto con ejemplos (few-shot), con una consulta que use la misma etiqueta ("action:") o con un prompt aprendido podría funcionar con 19–23 posiciones. Eso no se probó. Conclusión honesta: K=2 le gana a texto compacto *zero-shot*, pero falta el baseline compacto *adaptado*.
|
| 38 |
+
|
| 39 |
+
## 3. Decodificadores sin LLM sobre las ranuras K=2
|
| 40 |
+
|
| 41 |
+
Entrenados en 20,000 mensajes de FIT con palabras aleatorias del pool de entrenamiento (p=0.4 por campo), objetivo = embedding del valor, decodificación por coseno contra las opciones. Media de 3 semillas (por semilla entre paréntesis).
|
| 42 |
+
|
| 43 |
+
| Decodificador | sealed_ood (400) | sealed_1_new (192) | sealed_4_new (192) |
|
| 44 |
+
|---|---|---|---|
|
| 45 |
+
| LLM congelado (K=2) | 99.0 | 93.4 | **84.2** |
|
| 46 |
+
| ridge_aug (λ=10) | 100 (100/100/100) | 94.3 (93.2/92.7/96.9) | **64.6** (58.3/68.2/67.2) |
|
| 47 |
+
| mlp_aug (2 capas ocultas) | 100 | **100** (100/100/100) | 56.4 (54.7/64.6/50.0) |
|
| 48 |
+
| mlp con objetivos barajados (control) | 0 | 0 | 0 |
|
| 49 |
+
| mlp clasificador solo FIT | 99.75 | 0 | 0 |
|
| 50 |
+
| kNN-1 / kNN-5 sobre ranuras de FIT | 0 / 14.9 | 0 / 0 | 0 / 0 |
|
| 51 |
+
| Receptor con pesos aleatorios: texto / K=2 | 0 / 0 | — | 0 / 0 |
|
| 52 |
+
|
| 53 |
+
Δ vs LLM en sealed_4_new: ridge_aug −19.6 [−26.7, −12.5] pp; mlp_aug −27.8 [−35.2, −20.1] pp. En sealed_1_new: mlp_aug **+6.6 [+4.5, +8.9] pp** (mejor que el LLM).
|
| 54 |
+
|
| 55 |
+
(kNN no puede emitir palabras nuevas ni combinaciones fuera de FIT, por construcción; su 0% no dice nada sobre lookup.)
|
| 56 |
+
|
| 57 |
+
**Regla 3: ningún decodificador sin LLM alcanza al LLM en sealed_4_new**, así que formalmente no se retira la afirmación. Pero el resultado cambia la historia: el "0–2%" de la ridge/coseno de 018 era un artefacto de sondas que nunca vieron palabras nuevas. Con aumento de vocabulario, una ridge lineal recupera 58–68% de los mensajes con 4 palabras nuevas, y un MLP lee el 100% de los mensajes con 1 palabra nueva. **Las ranuras son en gran parte decodificables sin el LLM**: la información de las palabras viaja casi lineal en las ranuras. El LLM agrega ≈20 pp solo en el caso más difícil (4 nuevas). La afirmación "no es un lookup" debe bajarse a: "no es un lookup *simple*; un decodificador lineal entrenado con aumento recupera la mayoría".
|
| 58 |
+
|
| 59 |
+
## 4. Veredicto honesto
|
| 60 |
+
|
| 61 |
+
- **Sobrevive:** la paridad K=2 ≈ plantilla de texto en combinaciones nuevas de palabras conocidas, ahora en un test sellado y pre-registrado (99.0% vs 99.25%, Δ −0.25 pp [−1.25, +0.92]) con 17 vs 35 posiciones.
|
| 62 |
+
- **No sobrevive tal cual:** la idea de que solo el LLM puede leer las ranuras. Una sonda lineal con aumento lee 58–68% con 4 palabras nuevas y ~93–100% con 1.
|
| 63 |
+
- **Sin resolver:** el ahorro de posiciones frente a un texto compacto *bien adaptado*. Zero-shot, todos los formatos compactos colapsan (0–8%), pero eso mide al receptor, no al formato.
|
| 64 |
+
- **Sigue débil:** transferencia a palabras nuevas (−3 a −10 pp vs texto).
|
| 65 |
+
|
| 66 |
+
Siguiente paso sugerido: baseline compacto few-shot o con prompt aprendido (mismo presupuesto de entrenamiento que K=2), generación abierta, y 10 semillas.
|
020/results020.jsonl
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"env": {"python": "3.13.15", "torch": "2.11.0+cu128", "transformers": "4.46.3", "device": "Tesla T4"}, "sealed_sha256": "4c7737f66364a44786b4d9acb4686a6f31829d78cb5914f19eca1df74fe73393"}
|
| 2 |
+
{"route": "text_template", "set": "sealed_id", "n": 200, "acc": 0.995, "wilson95": [97.2, 99.9], "positions_per_event": 35.0}
|
| 3 |
+
{"route": "text_template", "set": "sealed_ood", "n": 400, "acc": 0.9925, "wilson95": [97.8, 99.7], "positions_per_event": 35.0}
|
| 4 |
+
{"route": "text_template", "set": "sealed_1_new", "n": 192, "acc": 0.9635416666666666, "wilson95": [92.7, 98.2], "positions_per_event": 35.0}
|
| 5 |
+
{"route": "text_template", "set": "sealed_2_new", "n": 192, "acc": 0.984375, "wilson95": [95.5, 99.5], "positions_per_event": 35.0}
|
| 6 |
+
{"route": "text_template", "set": "sealed_4_new", "n": 192, "acc": 0.9427083333333334, "wilson95": [90.0, 96.8], "positions_per_event": 35.0}
|
| 7 |
+
{"route": "text_kv", "set": "sealed_id", "n": 200, "acc": 0.065, "wilson95": [3.8, 10.8], "positions_per_event": 31.0, "diff_vs_template": -93.0, "boot95": [-96.5, -89.0]}
|
| 8 |
+
{"route": "text_kv", "set": "sealed_ood", "n": 400, "acc": 0.025, "wilson95": [1.4, 4.5], "positions_per_event": 31.0, "diff_vs_template": -96.75, "boot95": [-98.25, -95.0]}
|
| 9 |
+
{"route": "text_kv", "set": "sealed_1_new", "n": 192, "acc": 0.010416666666666666, "wilson95": [0.3, 3.7], "positions_per_event": 31.0, "diff_vs_template": -95.31, "boot95": [-98.44, -91.67]}
|
| 10 |
+
{"route": "text_kv", "set": "sealed_2_new", "n": 192, "acc": 0.052083333333333336, "wilson95": [2.9, 9.3], "positions_per_event": 31.0, "diff_vs_template": -93.23, "boot95": [-96.35, -89.58]}
|
| 11 |
+
{"route": "text_kv", "set": "sealed_4_new", "n": 192, "acc": 0.08333333333333333, "wilson95": [5.2, 13.1], "positions_per_event": 31.0, "diff_vs_template": -85.94, "boot95": [-90.62, -80.73]}
|
| 12 |
+
{"route": "text_abbrev", "set": "sealed_id", "n": 200, "acc": 0.035, "wilson95": [1.7, 7.0], "positions_per_event": 27.67, "diff_vs_template": -96.0, "boot95": [-98.5, -93.0]}
|
| 13 |
+
{"route": "text_abbrev", "set": "sealed_ood", "n": 400, "acc": 0.0, "wilson95": [0.0, 1.0], "positions_per_event": 27.63, "diff_vs_template": -99.25, "boot95": [-100.0, -98.25]}
|
| 14 |
+
{"route": "text_abbrev", "set": "sealed_1_new", "n": 192, "acc": 0.0, "wilson95": [0.0, 2.0], "positions_per_event": 29.1, "diff_vs_template": -96.35, "boot95": [-98.96, -93.23]}
|
| 15 |
+
{"route": "text_abbrev", "set": "sealed_2_new", "n": 192, "acc": 0.0, "wilson95": [0.0, 2.0], "positions_per_event": 30.47, "diff_vs_template": -98.44, "boot95": [-100.0, -96.35]}
|
| 16 |
+
{"route": "text_abbrev", "set": "sealed_4_new", "n": 192, "acc": 0.0, "wilson95": [0.0, 2.0], "positions_per_event": 33.37, "diff_vs_template": -94.27, "boot95": [-97.4, -90.62]}
|
| 17 |
+
{"route": "text_csv", "set": "sealed_id", "n": 200, "acc": 0.03, "wilson95": [1.4, 6.4], "positions_per_event": 30.67, "diff_vs_template": -96.5, "boot95": [-99.0, -93.5]}
|
| 18 |
+
{"route": "text_csv", "set": "sealed_ood", "n": 400, "acc": 0.005, "wilson95": [0.1, 1.8], "positions_per_event": 30.63, "diff_vs_template": -98.75, "boot95": [-99.75, -97.5]}
|
| 19 |
+
{"route": "text_csv", "set": "sealed_1_new", "n": 192, "acc": 0.005208333333333333, "wilson95": [0.1, 2.9], "positions_per_event": 32.1, "diff_vs_template": -95.83, "boot95": [-98.44, -92.71]}
|
| 20 |
+
{"route": "text_csv", "set": "sealed_2_new", "n": 192, "acc": 0.005208333333333333, "wilson95": [0.1, 2.9], "positions_per_event": 33.47, "diff_vs_template": -97.92, "boot95": [-99.48, -95.83]}
|
| 21 |
+
{"route": "text_csv", "set": "sealed_4_new", "n": 192, "acc": 0.0, "wilson95": [0.0, 2.0], "positions_per_event": 36.37, "diff_vs_template": -94.27, "boot95": [-97.4, -90.62]}
|
| 22 |
+
{"route": "text_values_csv", "set": "sealed_id", "n": 200, "acc": 0.03, "wilson95": [1.4, 6.4], "positions_per_event": 22.0, "diff_vs_template": -96.5, "boot95": [-99.0, -93.5]}
|
| 23 |
+
{"route": "text_values_csv", "set": "sealed_ood", "n": 400, "acc": 0.0025, "wilson95": [0.0, 1.4], "positions_per_event": 22.0, "diff_vs_template": -99.0, "boot95": [-99.75, -98.0]}
|
| 24 |
+
{"route": "text_values_csv", "set": "sealed_1_new", "n": 192, "acc": 0.0, "wilson95": [0.0, 2.0], "positions_per_event": 22.46, "diff_vs_template": -96.35, "boot95": [-98.96, -93.23]}
|
| 25 |
+
{"route": "text_values_csv", "set": "sealed_2_new", "n": 192, "acc": 0.0, "wilson95": [0.0, 2.0], "positions_per_event": 22.76, "diff_vs_template": -98.44, "boot95": [-100.0, -96.35]}
|
| 26 |
+
{"route": "text_values_csv", "set": "sealed_4_new", "n": 192, "acc": 0.0, "wilson95": [0.0, 2.0], "positions_per_event": 23.56, "diff_vs_template": -94.27, "boot95": [-97.4, -90.62]}
|
| 27 |
+
{"route": "text_values", "set": "sealed_id", "n": 200, "acc": 0.005, "wilson95": [0.1, 2.8], "positions_per_event": 19.0, "diff_vs_template": -99.0, "boot95": [-100.0, -97.5]}
|
| 28 |
+
{"route": "text_values", "set": "sealed_ood", "n": 400, "acc": 0.0025, "wilson95": [0.0, 1.4], "positions_per_event": 19.0, "diff_vs_template": -99.0, "boot95": [-99.75, -98.0]}
|
| 29 |
+
{"route": "text_values", "set": "sealed_1_new", "n": 192, "acc": 0.0, "wilson95": [0.0, 2.0], "positions_per_event": 19.46, "diff_vs_template": -96.35, "boot95": [-98.96, -93.23]}
|
| 30 |
+
{"route": "text_values", "set": "sealed_2_new", "n": 192, "acc": 0.0, "wilson95": [0.0, 2.0], "positions_per_event": 19.76, "diff_vs_template": -98.44, "boot95": [-100.0, -96.35]}
|
| 31 |
+
{"route": "text_values", "set": "sealed_4_new", "n": 192, "acc": 0.0, "wilson95": [0.0, 2.0], "positions_per_event": 20.56, "diff_vs_template": -94.27, "boot95": [-97.4, -90.62]}
|
| 32 |
+
{"route": "k2_s17", "set": "sealed_id", "n": 200, "acc": 0.985, "wilson95": [95.7, 99.5], "positions_per_event": 17, "diff_vs_template": -1.0, "boot95": [-3.0, 1.0]}
|
| 33 |
+
{"route": "k2_s17", "set": "sealed_ood", "n": 400, "acc": 0.99, "wilson95": [97.5, 99.6], "positions_per_event": 17, "diff_vs_template": -0.25, "boot95": [-1.5, 1.0]}
|
| 34 |
+
{"route": "k2_s17", "set": "sealed_1_new", "n": 192, "acc": 0.9583333333333334, "wilson95": [92.0, 97.9], "positions_per_event": 17, "diff_vs_template": -0.52, "boot95": [-4.17, 3.12]}
|
| 35 |
+
{"route": "k2_s17", "set": "sealed_2_new", "n": 192, "acc": 0.90625, "wilson95": [85.7, 94.0], "positions_per_event": 17, "diff_vs_template": -7.81, "boot95": [-12.5, -3.12]}
|
| 36 |
+
{"route": "k2_s17", "set": "sealed_4_new", "n": 192, "acc": 0.828125, "wilson95": [76.8, 87.5], "positions_per_event": 17, "diff_vs_template": -11.46, "boot95": [-17.19, -5.73]}
|
| 37 |
+
{"route": "k2_s29", "set": "sealed_id", "n": 200, "acc": 1.0, "wilson95": [98.1, 100.0], "positions_per_event": 17, "diff_vs_template": 0.5, "boot95": [0.0, 1.5]}
|
| 38 |
+
{"route": "k2_s29", "set": "sealed_ood", "n": 400, "acc": 0.9975, "wilson95": [98.6, 100.0], "positions_per_event": 17, "diff_vs_template": 0.5, "boot95": [-0.5, 1.5]}
|
| 39 |
+
{"route": "k2_s29", "set": "sealed_1_new", "n": 192, "acc": 0.984375, "wilson95": [95.5, 99.5], "positions_per_event": 17, "diff_vs_template": 2.08, "boot95": [-1.04, 5.21]}
|
| 40 |
+
{"route": "k2_s29", "set": "sealed_2_new", "n": 192, "acc": 0.9791666666666666, "wilson95": [94.8, 99.2], "positions_per_event": 17, "diff_vs_template": -0.52, "boot95": [-3.12, 2.08]}
|
| 41 |
+
{"route": "k2_s29", "set": "sealed_4_new", "n": 192, "acc": 0.875, "wilson95": [82.1, 91.5], "positions_per_event": 17, "diff_vs_template": -6.77, "boot95": [-11.98, -2.08]}
|
| 42 |
+
{"route": "k2_s43", "set": "sealed_id", "n": 200, "acc": 0.98, "wilson95": [95.0, 99.2], "positions_per_event": 17, "diff_vs_template": -1.5, "boot95": [-4.0, 0.5]}
|
| 43 |
+
{"route": "k2_s43", "set": "sealed_ood", "n": 400, "acc": 0.9825, "wilson95": [96.4, 99.1], "positions_per_event": 17, "diff_vs_template": -1.0, "boot95": [-2.5, 0.5]}
|
| 44 |
+
{"route": "k2_s43", "set": "sealed_1_new", "n": 192, "acc": 0.859375, "wilson95": [80.3, 90.2], "positions_per_event": 17, "diff_vs_template": -10.42, "boot95": [-15.62, -5.21]}
|
| 45 |
+
{"route": "k2_s43", "set": "sealed_2_new", "n": 192, "acc": 0.8697916666666666, "wilson95": [81.5, 91.0], "positions_per_event": 17, "diff_vs_template": -11.46, "boot95": [-16.67, -6.77]}
|
| 46 |
+
{"route": "k2_s43", "set": "sealed_4_new", "n": 192, "acc": 0.8229166666666666, "wilson95": [76.3, 87.0], "positions_per_event": 17, "diff_vs_template": -11.98, "boot95": [-18.23, -6.25]}
|
| 47 |
+
{"route": "k2_mean3", "set": "sealed_id", "acc": 0.9883333333333333, "vs": "text_template", "diff": -0.67, "boot95": [-2.0, 0.83]}
|
| 48 |
+
{"route": "k2_mean3", "set": "sealed_id", "acc": 0.9883333333333333, "vs": "text_kv", "diff": 92.33, "boot95": [88.67, 95.67]}
|
| 49 |
+
{"route": "k2_mean3", "set": "sealed_id", "acc": 0.9883333333333333, "vs": "text_abbrev", "diff": 95.33, "boot95": [92.5, 97.83]}
|
| 50 |
+
{"route": "k2_mean3", "set": "sealed_id", "acc": 0.9883333333333333, "vs": "text_csv", "diff": 95.83, "boot95": [93.17, 98.17]}
|
| 51 |
+
{"route": "k2_mean3", "set": "sealed_id", "acc": 0.9883333333333333, "vs": "text_values_csv", "diff": 95.83, "boot95": [93.17, 98.17]}
|
| 52 |
+
{"route": "k2_mean3", "set": "sealed_id", "acc": 0.9883333333333333, "vs": "text_values", "diff": 98.33, "boot95": [96.83, 99.5]}
|
| 53 |
+
{"route": "k2_mean3", "set": "sealed_ood", "acc": 0.99, "vs": "text_template", "diff": -0.25, "boot95": [-1.25, 0.92]}
|
| 54 |
+
{"route": "k2_mean3", "set": "sealed_ood", "acc": 0.99, "vs": "text_kv", "diff": 96.5, "boot95": [94.75, 98.0]}
|
| 55 |
+
{"route": "k2_mean3", "set": "sealed_ood", "acc": 0.99, "vs": "text_abbrev", "diff": 99.0, "boot95": [98.33, 99.58]}
|
| 56 |
+
{"route": "k2_mean3", "set": "sealed_ood", "acc": 0.99, "vs": "text_csv", "diff": 98.5, "boot95": [97.5, 99.33]}
|
| 57 |
+
{"route": "k2_mean3", "set": "sealed_ood", "acc": 0.99, "vs": "text_values_csv", "diff": 98.75, "boot95": [97.83, 99.5]}
|
| 58 |
+
{"route": "k2_mean3", "set": "sealed_ood", "acc": 0.99, "vs": "text_values", "diff": 98.75, "boot95": [97.83, 99.5]}
|
| 59 |
+
{"route": "k2_mean3", "set": "sealed_1_new", "acc": 0.9340277777777778, "vs": "text_template", "diff": -2.95, "boot95": [-6.08, 0.35]}
|
| 60 |
+
{"route": "k2_mean3", "set": "sealed_1_new", "acc": 0.9340277777777778, "vs": "text_kv", "diff": 92.36, "boot95": [89.41, 94.97]}
|
| 61 |
+
{"route": "k2_mean3", "set": "sealed_1_new", "acc": 0.9340277777777778, "vs": "text_abbrev", "diff": 93.4, "boot95": [90.97, 95.49]}
|
| 62 |
+
{"route": "k2_mean3", "set": "sealed_1_new", "acc": 0.9340277777777778, "vs": "text_csv", "diff": 92.88, "boot95": [90.28, 95.14]}
|
| 63 |
+
{"route": "k2_mean3", "set": "sealed_1_new", "acc": 0.9340277777777778, "vs": "text_values_csv", "diff": 93.4, "boot95": [90.97, 95.49]}
|
| 64 |
+
{"route": "k2_mean3", "set": "sealed_1_new", "acc": 0.9340277777777778, "vs": "text_values", "diff": 93.4, "boot95": [90.97, 95.49]}
|
| 65 |
+
{"route": "k2_mean3", "set": "sealed_2_new", "acc": 0.9184027777777778, "vs": "text_template", "diff": -6.6, "boot95": [-9.72, -3.47]}
|
| 66 |
+
{"route": "k2_mean3", "set": "sealed_2_new", "acc": 0.9184027777777778, "vs": "text_kv", "diff": 86.63, "boot95": [82.47, 90.62]}
|
| 67 |
+
{"route": "k2_mean3", "set": "sealed_2_new", "acc": 0.9184027777777778, "vs": "text_abbrev", "diff": 91.84, "boot95": [89.06, 94.44]}
|
| 68 |
+
{"route": "k2_mean3", "set": "sealed_2_new", "acc": 0.9184027777777778, "vs": "text_csv", "diff": 91.32, "boot95": [88.37, 94.1]}
|
| 69 |
+
{"route": "k2_mean3", "set": "sealed_2_new", "acc": 0.9184027777777778, "vs": "text_values_csv", "diff": 91.84, "boot95": [89.06, 94.44]}
|
| 70 |
+
{"route": "k2_mean3", "set": "sealed_2_new", "acc": 0.9184027777777778, "vs": "text_values", "diff": 91.84, "boot95": [89.06, 94.44]}
|
| 71 |
+
{"route": "k2_mean3", "set": "sealed_4_new", "acc": 0.8420138888888888, "vs": "text_template", "diff": -10.07, "boot95": [-15.1, -5.38]}
|
| 72 |
+
{"route": "k2_mean3", "set": "sealed_4_new", "acc": 0.8420138888888888, "vs": "text_kv", "diff": 75.87, "boot95": [70.14, 81.25]}
|
| 73 |
+
{"route": "k2_mean3", "set": "sealed_4_new", "acc": 0.8420138888888888, "vs": "text_abbrev", "diff": 84.2, "boot95": [79.51, 88.37]}
|
| 74 |
+
{"route": "k2_mean3", "set": "sealed_4_new", "acc": 0.8420138888888888, "vs": "text_csv", "diff": 84.2, "boot95": [79.51, 88.37]}
|
| 75 |
+
{"route": "k2_mean3", "set": "sealed_4_new", "acc": 0.8420138888888888, "vs": "text_values_csv", "diff": 84.2, "boot95": [79.51, 88.37]}
|
| 76 |
+
{"route": "k2_mean3", "set": "sealed_4_new", "acc": 0.8420138888888888, "vs": "text_values", "diff": 84.2, "boot95": [79.51, 88.37]}
|
| 77 |
+
{"route": "probes_k2_s17", "ridge_aug:sealed_ood": 1.0, "mlp_aug:sealed_ood": 1.0, "mlp_ctrl_shuffled:sealed_ood": 0.0, "mlp_cls_fit:sealed_ood": 0.995, "knn1:sealed_ood": 0.0, "knn5:sealed_ood": 0.1325, "ridge_aug:sealed_4_new": 0.5833333333333334, "mlp_aug:sealed_4_new": 0.546875, "mlp_ctrl_shuffled:sealed_4_new": 0.0, "mlp_cls_fit:sealed_4_new": 0.0, "knn1:sealed_4_new": 0.0, "knn5:sealed_4_new": 0.0, "ridge_aug:sealed_1_new": 0.9322916666666666, "mlp_aug:sealed_1_new": 1.0, "mlp_ctrl_shuffled:sealed_1_new": 0.0, "mlp_cls_fit:sealed_1_new": 0.0, "knn1:sealed_1_new": 0.0, "knn5:sealed_1_new": 0.0}
|
| 78 |
+
{"route": "probes_k2_s29", "ridge_aug:sealed_ood": 1.0, "mlp_aug:sealed_ood": 1.0, "mlp_ctrl_shuffled:sealed_ood": 0.0, "mlp_cls_fit:sealed_ood": 0.9975, "knn1:sealed_ood": 0.0, "knn5:sealed_ood": 0.14, "ridge_aug:sealed_4_new": 0.6822916666666666, "mlp_aug:sealed_4_new": 0.6458333333333334, "mlp_ctrl_shuffled:sealed_4_new": 0.0, "mlp_cls_fit:sealed_4_new": 0.0, "knn1:sealed_4_new": 0.0, "knn5:sealed_4_new": 0.0, "ridge_aug:sealed_1_new": 0.9270833333333334, "mlp_aug:sealed_1_new": 1.0, "mlp_ctrl_shuffled:sealed_1_new": 0.0, "mlp_cls_fit:sealed_1_new": 0.0, "knn1:sealed_1_new": 0.0, "knn5:sealed_1_new": 0.0}
|
| 79 |
+
{"route": "probes_k2_s43", "ridge_aug:sealed_ood": 1.0, "mlp_aug:sealed_ood": 1.0, "mlp_ctrl_shuffled:sealed_ood": 0.0, "mlp_cls_fit:sealed_ood": 1.0, "knn1:sealed_ood": 0.0, "knn5:sealed_ood": 0.175, "ridge_aug:sealed_4_new": 0.671875, "mlp_aug:sealed_4_new": 0.5, "mlp_ctrl_shuffled:sealed_4_new": 0.0, "mlp_cls_fit:sealed_4_new": 0.0, "knn1:sealed_4_new": 0.0, "knn5:sealed_4_new": 0.0, "ridge_aug:sealed_1_new": 0.96875, "mlp_aug:sealed_1_new": 1.0, "mlp_ctrl_shuffled:sealed_1_new": 0.0, "mlp_cls_fit:sealed_1_new": 0.0, "knn1:sealed_1_new": 0.0, "knn5:sealed_1_new": 0.0}
|
| 80 |
+
{"probe_mean3": "ridge_aug", "set": "sealed_ood", "acc": 1.0, "llm_k2_acc": 0.99, "diff_vs_llm": 1.0, "boot95": [0.42, 1.67]}
|
| 81 |
+
{"probe_mean3": "mlp_aug", "set": "sealed_ood", "acc": 1.0, "llm_k2_acc": 0.99, "diff_vs_llm": 1.0, "boot95": [0.42, 1.67]}
|
| 82 |
+
{"probe_mean3": "mlp_ctrl_shuffled", "set": "sealed_ood", "acc": 0.0, "llm_k2_acc": 0.99, "diff_vs_llm": -99.0, "boot95": [-99.58, -98.33]}
|
| 83 |
+
{"probe_mean3": "mlp_cls_fit", "set": "sealed_ood", "acc": 0.9975, "llm_k2_acc": 0.99, "diff_vs_llm": 0.75, "boot95": [0.08, 1.5]}
|
| 84 |
+
{"probe_mean3": "knn1", "set": "sealed_ood", "acc": 0.0, "llm_k2_acc": 0.99, "diff_vs_llm": -99.0, "boot95": [-99.58, -98.33]}
|
| 85 |
+
{"probe_mean3": "knn5", "set": "sealed_ood", "acc": 0.14916666666666667, "llm_k2_acc": 0.99, "diff_vs_llm": -84.08, "boot95": [-87.0, -81.08]}
|
| 86 |
+
{"probe_mean3": "ridge_aug", "set": "sealed_4_new", "acc": 0.6458333333333334, "llm_k2_acc": 0.8420138888888888, "diff_vs_llm": -19.62, "boot95": [-26.74, -12.5]}
|
| 87 |
+
{"probe_mean3": "mlp_aug", "set": "sealed_4_new", "acc": 0.564236111111111, "llm_k2_acc": 0.8420138888888888, "diff_vs_llm": -27.78, "boot95": [-35.24, -20.14]}
|
| 88 |
+
{"probe_mean3": "mlp_ctrl_shuffled", "set": "sealed_4_new", "acc": 0.0, "llm_k2_acc": 0.8420138888888888, "diff_vs_llm": -84.2, "boot95": [-88.54, -79.69]}
|
| 89 |
+
{"probe_mean3": "mlp_cls_fit", "set": "sealed_4_new", "acc": 0.0, "llm_k2_acc": 0.8420138888888888, "diff_vs_llm": -84.2, "boot95": [-88.54, -79.69]}
|
| 90 |
+
{"probe_mean3": "knn1", "set": "sealed_4_new", "acc": 0.0, "llm_k2_acc": 0.8420138888888888, "diff_vs_llm": -84.2, "boot95": [-88.54, -79.69]}
|
| 91 |
+
{"probe_mean3": "knn5", "set": "sealed_4_new", "acc": 0.0, "llm_k2_acc": 0.8420138888888888, "diff_vs_llm": -84.2, "boot95": [-88.54, -79.69]}
|
| 92 |
+
{"probe_mean3": "ridge_aug", "set": "sealed_1_new", "acc": 0.9427083333333334, "llm_k2_acc": 0.9340277777777778, "diff_vs_llm": 0.87, "boot95": [-2.95, 4.51]}
|
| 93 |
+
{"probe_mean3": "mlp_aug", "set": "sealed_1_new", "acc": 1.0, "llm_k2_acc": 0.9340277777777778, "diff_vs_llm": 6.6, "boot95": [4.51, 8.85]}
|
| 94 |
+
{"probe_mean3": "mlp_ctrl_shuffled", "set": "sealed_1_new", "acc": 0.0, "llm_k2_acc": 0.9340277777777778, "diff_vs_llm": -93.4, "boot95": [-95.49, -91.15]}
|
| 95 |
+
{"probe_mean3": "mlp_cls_fit", "set": "sealed_1_new", "acc": 0.0, "llm_k2_acc": 0.9340277777777778, "diff_vs_llm": -93.4, "boot95": [-95.49, -91.15]}
|
| 96 |
+
{"probe_mean3": "knn1", "set": "sealed_1_new", "acc": 0.0, "llm_k2_acc": 0.9340277777777778, "diff_vs_llm": -93.4, "boot95": [-95.49, -91.15]}
|
| 97 |
+
{"probe_mean3": "knn5", "set": "sealed_1_new", "acc": 0.0, "llm_k2_acc": 0.9340277777777778, "diff_vs_llm": -93.4, "boot95": [-95.49, -91.15]}
|
| 98 |
+
{"control": "random_receiver", "set": "sealed_ood", "text_template": 0.0, "k2_s17": 0.0, "k2_s29": 0.0, "k2_s43": 0.0}
|
| 99 |
+
{"control": "random_receiver", "set": "sealed_4_new", "text_template": 0.0, "k2_s17": 0.0, "k2_s29": 0.0, "k2_s43": 0.0}
|
020/sealed020.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"n_eligible_universe": 17494, "new_words": [["diagnostics", "manipulative", "favourable", "intelligence", "phonological", "persecuted", "contingency", "quantifying"], ["unaffected", "prerequisites", "dehydration", "intonation", "discerning", "kilometres", "watersheds", "foreseeable"], ["companions", "pediatrics", "epithelium", "complained", "multiplied", "sunglasses", "redistribute", "disclosure"], ["similarities", "applicability", "abnormality", "encountering", "predominantly", "provincial", "modalities", "universality"]], "sealed_1_new": [["start", "discerning", "open", "none"], ["update", "job", "sunglasses", "hour"], ["stop", "prerequisites", "active", "year"], ["start", "meeting", "complained", "month"], ["stop", "unaffected", "active", "minute"], ["read", "pump", "paid", "applicability"], ["execute", "order", "sunglasses", "day"], ["delete", "machine", "disclosure", "minute"], ["read", "inventory", "epithelium", "day"], ["favourable", "invoice", "failed", "month"], ["stop", "unaffected", "failed", "minute"], ["write", "pump", "multiplied", "today"], ["manipulative", "inventory", "high", "today"], ["quantifying", "meeting", "high", "hour"], ["query", "file", "companions", "week"], ["create", "pump", "pending", "abnormality"], ["quantifying", "customer", "idle", "year"], ["contingency", "motor", "pending", "tomorrow"], ["stop", "watersheds", "active", "now"], ["report", "unaffected", "running", "day"], ["write", "unaffected", "ready", "now"], ["persecuted", "pump", "paid", "none"], ["update", "watersheds", "blocked", "week"], ["notify", "prerequisites", "running", "hour"], ["create", "valve", "complete", "predominantly"], ["quantifying", "file", "running", "hour"], ["query", "file", "normal", "abnormality"], ["manipulative", "valve", "blocked", "tomorrow"], ["phonological", "valve", "low", "now"], ["contingency", "order", "ready", "month"], ["update", "order", "multiplied", "tomorrow"], ["persecuted", "invoice", "none", "today"], ["write", "pump", "none", "universality"], ["persecuted", "order", "high", "today"], ["notify", "kilometres", "ready", "tomorrow"], ["write", "intonation", "low", "day"], ["check", "customer", "multiplied", "day"], ["start", "meeting", "pediatrics", "hour"], ["report", "motor", "complained", "minute"], ["check", "machine", "complained", "today"], ["check", "file", "idle", "universality"], ["check", "file", "multiplied", "year"], ["write", "watersheds", "pending", "week"], ["start", "kilometres", "low", "today"], ["query", "machine", "companions", "today"], ["write", "valve", "sunglasses", "month"], ["query", "motor", "ready", "abnormality"], ["write", "valve", "pediatrics", "minute"], ["diagnostics", "inventory", "pending", "year"], ["start", "invoice", "paid", "universality"], ["write", "prerequisites", "closed", "day"], ["execute", "valve", "disclosure", "month"], ["read", "file", "multiplied", "tomorrow"], ["write", "dehydration", "low", "day"], ["persecuted", "valve", "idle", "tomorrow"], ["query", "file", "paid", "modalities"], ["stop", "email", "epithelium", "none"], ["check", "foreseeable", "ready", "hour"], ["write", "watersheds", "high", "hour"], ["report", "kilometres", "normal", "minute"], ["quantifying", "order", "running", "tomorrow"], ["notify", "motor", "none", "universality"], ["read", "machine", "high", "abnormality"], ["notify", "order", "companions", "week"], ["report", "discerning", "failed", "none"], ["intelligence", "order", "none", "year"], ["contingency", "customer", "complete", "year"], ["diagnostics", "order", "high", "hour"], ["update", "watersheds", "running", "day"], ["check", "motor", "open", "modalities"], ["execute", "customer", "epithelium", "now"], ["contingency", "machine", "active", "minute"], ["execute", "meeting", "closed", "provincial"], ["read", "kilometres", "complete", "year"], ["diagnostics", "job", "running", "now"], ["start", "meeting", "running", "modalities"], ["quantifying", "file", "overload", "year"], ["stop", "machine", "epithelium", "tomorrow"], ["delete", "machine", "idle", "similarities"], ["contingency", "meeting", "active", "week"], ["update", "order", "high", "encountering"], ["favourable", "machine", "overload", "month"], ["check", "file", "low", "similarities"], ["check", "pump", "epithelium", "day"], ["intelligence", "inventory", "idle", "year"], ["check", "motor", "open", "modalities"], ["check", "file", "pediatrics", "week"], ["read", "kilometres", "idle", "today"], ["write", "discerning", "overload", "today"], ["delete", "machine", "running", "predominantly"], ["persecuted", "email", "pending", "month"], ["favourable", "invoice", "blocked", "none"], ["update", "watersheds", "paid", "month"], ["notify", "email", "blocked", "universality"], ["phonological", "meeting", "normal", "year"], ["write", "intonation", "normal", "day"], ["delete", "customer", "complained", "month"], ["report", "pump", "pediatrics", "hour"], ["manipulative", "file", "high", "hour"], ["report", "customer", "high", "similarities"], ["write", "pump", "complained", "month"], ["intelligence", "email", "open", "tomorrow"], ["diagnostics", "inventory", "running", "month"], ["notify", "customer", "running", "universality"], ["notify", "valve", "redistribute", "hour"], ["execute", "dehydration", "active", "year"], ["query", "motor", "redistribute", "none"], ["quantifying", "pump", "closed", "day"], ["write", "meeting", "idle", "encountering"], ["query", "foreseeable", "normal", "none"], ["write", "motor", "complained", "none"], ["execute", "watersheds", "active", "year"], ["manipulative", "order", "pending", "minute"], ["report", "motor", "open", "predominantly"], ["intelligence", "pump", "open", "tomorrow"], ["update", "customer", "failed", "applicability"], ["persecuted", "meeting", "complete", "day"], ["check", "valve", "failed", "encountering"], ["notify", "customer", "complete", "predominantly"], ["delete", "pump", "sunglasses", "minute"], ["phonological", "invoice", "failed", "hour"], ["check", "unaffected", "normal", "hour"], ["report", "invoice", "high", "provincial"], ["persecuted", "motor", "high", "minute"], ["execute", "intonation", "running", "minute"], ["notify", "dehydration", "idle", "tomorrow"], ["persecuted", "order", "none", "now"], ["execute", "kilometres", "closed", "year"], ["manipulative", "motor", "normal", "year"], ["execute", "valve", "redistribute", "none"], ["favourable", "motor", "closed", "minute"], ["notify", "job", "ready", "encountering"], ["persecuted", "customer", "failed", "minute"], ["report", "intonation", "overload", "month"], ["write", "valve", "closed", "similarities"], ["query", "machine", "complained", "minute"], ["execute", "inventory", "epithelium", "hour"], ["execute", "pump", "low", "provincial"], ["read", "job", "epithelium", "year"], ["read", "customer", "redistribute", "minute"], ["read", "foreseeable", "complete", "day"], ["stop", "inventory", "complained", "now"], ["query", "pump", "failed", "provincial"], ["report", "customer", "pending", "provincial"], ["query", "meeting", "sunglasses", "year"], ["update", "foreseeable", "normal", "minute"], ["check", "valve", "failed", "provincial"], ["notify", "inventory", "pediatrics", "day"], ["update", "inventory", "none", "modalities"], ["persecuted", "order", "ready", "hour"], ["read", "unaffected", "ready", "month"], ["query", "unaffected", "idle", "day"], ["stop", "inventory", "redistribute", "week"], ["start", "email", "sunglasses", "day"], ["diagnostics", "job", "ready", "month"], ["favourable", "customer", "closed", "none"], ["check", "intonation", "running", "minute"], ["create", "dehydration", "closed", "day"], ["start", "watersheds", "pending", "week"], ["update", "order", "idle", "predominantly"], ["delete", "machine", "redistribute", "tomorrow"], ["report", "customer", "disclosure", "today"], ["read", "foreseeable", "closed", "day"], ["check", "intonation", "running", "week"], ["create", "job", "sunglasses", "month"], ["intelligence", "machine", "blocked", "minute"], ["start", "customer", "paid", "encountering"], ["notify", "kilometres", "ready", "tomorrow"], ["contingency", "meeting", "blocked", "tomorrow"], ["query", "invoice", "failed", "abnormality"], ["persecuted", "job", "ready", "none"], ["manipulative", "invoice", "paid", "minute"], ["read", "prerequisites", "paid", "month"], ["contingency", "customer", "overload", "week"], ["persecuted", "invoice", "blocked", "minute"], ["execute", "valve", "pediatrics", "month"], ["delete", "pump", "none", "applicability"], ["report", "invoice", "open", "predominantly"], ["contingency", "invoice", "active", "minute"], ["delete", "machine", "sunglasses", "today"], ["persecuted", "file", "open", "hour"], ["start", "invoice", "redistribute", "week"], ["persecuted", "order", "open", "tomorrow"], ["check", "file", "multiplied", "day"], ["stop", "intonation", "low", "day"], ["manipulative", "invoice", "idle", "minute"], ["start", "order", "disclosure", "month"], ["report", "customer", "low", "universality"], ["stop", "dehydration", "idle", "tomorrow"], ["execute", "discerning", "idle", "tomorrow"], ["execute", "unaffected", "running", "year"], ["quantifying", "valve", "high", "today"]], "sealed_2_new": [["start", "discerning", "open", "universality"], ["update", "kilometres", "failed", "applicability"], ["phonological", "customer", "pediatrics", "year"], ["intelligence", "meeting", "low", "modalities"], ["stop", "intonation", "active", "encountering"], ["read", "foreseeable", "paid", "predominantly"], ["execute", "order", "sunglasses", "predominantly"], ["delete", "machine", "complained", "abnormality"], ["read", "inventory", "companions", "encountering"], ["read", "dehydration", "failed", "abnormality"], ["stop", "intonation", "sunglasses", "minute"], ["write", "pump", "multiplied", "provincial"], ["phonological", "discerning", "high", "today"], ["write", "meeting", "redistribute", "encountering"], ["query", "file", "disclosure", "encountering"], ["manipulative", "unaffected", "pending", "week"], ["persecuted", "customer", "idle", "modalities"], ["quantifying", "motor", "complained", "tomorrow"], ["intelligence", "inventory", "companions", "now"], ["report", "customer", "complained", "encountering"], ["contingency", "email", "ready", "applicability"], ["quantifying", "intonation", "paid", "none"], ["update", "watersheds", "epithelium", "week"], ["notify", "job", "epithelium", "similarities"], ["create", "valve", "pediatrics", "modalities"], ["diagnostics", "kilometres", "running", "hour"], ["phonological", "file", "companions", "today"], ["write", "valve", "epithelium", "predominantly"], ["diagnostics", "valve", "low", "predominantly"], ["phonological", "order", "ready", "predominantly"], ["update", "order", "disclosure", "modalities"], ["start", "unaffected", "companions", "today"], ["write", "prerequisites", "none", "universality"], ["persecuted", "order", "high", "abnormality"], ["favourable", "job", "pediatrics", "tomorrow"], ["write", "meeting", "multiplied", "abnormality"], ["check", "customer", "redistribute", "universality"], ["quantifying", "intonation", "blocked", "hour"], ["intelligence", "motor", "overload", "predominantly"], ["check", "machine", "disclosure", "applicability"], ["diagnostics", "file", "idle", "predominantly"], ["phonological", "kilometres", "high", "year"], ["write", "watersheds", "pending", "provincial"], ["contingency", "unaffected", "low", "today"], ["persecuted", "machine", "closed", "universality"], ["persecuted", "valve", "paid", "applicability"], ["persecuted", "motor", "ready", "universality"], ["diagnostics", "valve", "idle", "applicability"], ["stop", "inventory", "redistribute", "provincial"], ["persecuted", "foreseeable", "paid", "none"], ["quantifying", "email", "closed", "modalities"], ["execute", "prerequisites", "running", "similarities"], ["quantifying", "file", "complained", "tomorrow"], ["favourable", "valve", "redistribute", "day"], ["contingency", "foreseeable", "idle", "tomorrow"], ["quantifying", "file", "epithelium", "year"], ["diagnostics", "dehydration", "open", "none"], ["check", "dehydration", "ready", "modalities"], ["manipulative", "foreseeable", "high", "hour"], ["quantifying", "motor", "multiplied", "minute"], ["execute", "intonation", "complained", "tomorrow"], ["quantifying", "dehydration", "none", "month"], ["read", "machine", "sunglasses", "predominantly"], ["favourable", "order", "redistribute", "week"], ["report", "kilometres", "sunglasses", "none"], ["persecuted", "order", "none", "similarities"], ["execute", "prerequisites", "complete", "modalities"], ["intelligence", "order", "complained", "hour"], ["favourable", "pump", "disclosure", "day"], ["check", "foreseeable", "epithelium", "week"], ["execute", "intonation", "disclosure", "now"], ["create", "prerequisites", "active", "provincial"], ["execute", "meeting", "companions", "applicability"], ["favourable", "intonation", "complete", "year"], ["diagnostics", "job", "companions", "now"], ["persecuted", "meeting", "multiplied", "month"], ["intelligence", "file", "companions", "year"], ["diagnostics", "intonation", "open", "tomorrow"], ["delete", "intonation", "idle", "predominantly"], ["create", "meeting", "multiplied", "similarities"], ["update", "unaffected", "high", "applicability"], ["manipulative", "machine", "overload", "similarities"], ["manipulative", "file", "low", "abnormality"], ["intelligence", "pump", "pediatrics", "day"], ["update", "intonation", "companions", "year"], ["check", "kilometres", "pediatrics", "day"], ["check", "foreseeable", "companions", "week"], ["read", "pump", "disclosure", "modalities"], ["write", "email", "complained", "predominantly"], ["manipulative", "discerning", "running", "now"], ["persecuted", "email", "pending", "modalities"], ["phonological", "invoice", "blocked", "encountering"], ["update", "intonation", "redistribute", "month"], ["notify", "kilometres", "blocked", "predominantly"], ["persecuted", "discerning", "normal", "year"], ["persecuted", "kilometres", "normal", "day"], ["delete", "dehydration", "ready", "provincial"], ["manipulative", "watersheds", "none", "hour"], ["create", "watersheds", "high", "modalities"], ["report", "watersheds", "pediatrics", "day"], ["write", "prerequisites", "paid", "encountering"], ["contingency", "email", "open", "applicability"], ["update", "dehydration", "companions", "month"], ["manipulative", "discerning", "running", "day"], ["diagnostics", "valve", "epithelium", "hour"], ["execute", "unaffected", "companions", "year"], ["favourable", "motor", "open", "similarities"], ["start", "pump", "pediatrics", "applicability"], ["write", "discerning", "idle", "provincial"], ["query", "kilometres", "sunglasses", "none"], ["write", "intonation", "blocked", "modalities"], ["execute", "foreseeable", "active", "encountering"], ["update", "order", "multiplied", "applicability"], ["report", "foreseeable", "open", "similarities"], ["quantifying", "pump", "epithelium", "tomorrow"], ["update", "customer", "complained", "predominantly"], ["stop", "unaffected", "complete", "predominantly"], ["check", "kilometres", "sunglasses", "today"], ["quantifying", "customer", "redistribute", "day"], ["diagnostics", "pump", "multiplied", "minute"], ["delete", "watersheds", "pediatrics", "hour"], ["quantifying", "file", "normal", "predominantly"], ["report", "watersheds", "high", "modalities"], ["check", "foreseeable", "high", "provincial"], ["quantifying", "valve", "running", "universality"], ["quantifying", "unaffected", "idle", "tomorrow"], ["persecuted", "prerequisites", "none", "now"], ["execute", "order", "companions", "provincial"], ["check", "motor", "redistribute", "abnormality"], ["intelligence", "valve", "active", "encountering"], ["report", "watersheds", "complained", "minute"], ["favourable", "job", "pediatrics", "month"], ["intelligence", "dehydration", "failed", "minute"], ["report", "watersheds", "overload", "encountering"], ["diagnostics", "valve", "closed", "abnormality"], ["contingency", "prerequisites", "idle", "minute"], ["phonological", "inventory", "paid", "universality"], ["execute", "pump", "pediatrics", "encountering"], ["contingency", "job", "companions", "year"], ["favourable", "watersheds", "active", "minute"], ["diagnostics", "prerequisites", "complete", "day"], ["stop", "discerning", "overload", "abnormality"], ["query", "unaffected", "pediatrics", "hour"], ["contingency", "customer", "pending", "modalities"], ["manipulative", "meeting", "complete", "predominantly"], ["update", "foreseeable", "normal", "predominantly"], ["check", "intonation", "failed", "applicability"], ["notify", "discerning", "multiplied", "day"], ["manipulative", "kilometres", "none", "hour"], ["read", "dehydration", "ready", "modalities"], ["phonological", "file", "ready", "applicability"], ["query", "file", "disclosure", "applicability"], ["stop", "inventory", "multiplied", "universality"], ["favourable", "email", "pediatrics", "day"], ["manipulative", "job", "ready", "provincial"], ["delete", "intonation", "closed", "encountering"], ["check", "unaffected", "running", "applicability"], ["persecuted", "email", "epithelium", "day"], ["start", "intonation", "pending", "encountering"], ["favourable", "order", "pediatrics", "year"], ["phonological", "dehydration", "high", "tomorrow"], ["intelligence", "customer", "paid", "abnormality"], ["read", "prerequisites", "disclosure", "day"], ["check", "kilometres", "multiplied", "week"], ["intelligence", "job", "ready", "applicability"], ["contingency", "machine", "epithelium", "minute"], ["start", "kilometres", "epithelium", "hour"], ["notify", "unaffected", "ready", "provincial"], ["delete", "kilometres", "blocked", "encountering"], ["persecuted", "invoice", "failed", "universality"], ["phonological", "job", "ready", "universality"], ["quantifying", "watersheds", "paid", "minute"], ["read", "email", "epithelium", "provincial"], ["report", "customer", "companions", "provincial"], ["quantifying", "prerequisites", "blocked", "minute"], ["diagnostics", "valve", "disclosure", "month"], ["intelligence", "pump", "none", "modalities"], ["report", "unaffected", "pediatrics", "tomorrow"], ["quantifying", "foreseeable", "active", "minute"], ["phonological", "machine", "multiplied", "today"], ["diagnostics", "file", "open", "applicability"], ["start", "foreseeable", "sunglasses", "week"], ["write", "intonation", "open", "predominantly"], ["persecuted", "file", "normal", "encountering"], ["stop", "invoice", "complained", "predominantly"], ["quantifying", "invoice", "idle", "abnormality"], ["phonological", "prerequisites", "paid", "month"], ["favourable", "customer", "disclosure", "year"], ["stop", "prerequisites", "idle", "provincial"], ["execute", "pump", "disclosure", "similarities"], ["persecuted", "foreseeable", "running", "year"], ["manipulative", "valve", "high", "similarities"]], "sealed_4_new": [["diagnostics", "watersheds", "redistribute", "predominantly"], ["phonological", "unaffected", "companions", "applicability"], ["manipulative", "kilometres", "redistribute", "similarities"], ["quantifying", "foreseeable", "multiplied", "modalities"], ["favourable", "discerning", "sunglasses", "encountering"], ["intelligence", "prerequisites", "complained", "applicability"], ["favourable", "intonation", "epithelium", "predominantly"], ["quantifying", "kilometres", "complained", "abnormality"], ["intelligence", "discerning", "companions", "modalities"], ["persecuted", "watersheds", "complained", "abnormality"], ["manipulative", "watersheds", "pediatrics", "universality"], ["phonological", "unaffected", "sunglasses", "encountering"], ["phonological", "kilometres", "multiplied", "provincial"], ["diagnostics", "intonation", "complained", "similarities"], ["persecuted", "foreseeable", "epithelium", "applicability"], ["phonological", "discerning", "sunglasses", "universality"], ["quantifying", "foreseeable", "pediatrics", "universality"], ["manipulative", "prerequisites", "redistribute", "modalities"], ["contingency", "prerequisites", "pediatrics", "universality"], ["contingency", "dehydration", "redistribute", "universality"], ["persecuted", "discerning", "companions", "predominantly"], ["diagnostics", "foreseeable", "epithelium", "similarities"], ["persecuted", "watersheds", "redistribute", "abnormality"], ["persecuted", "intonation", "multiplied", "provincial"], ["phonological", "prerequisites", "redistribute", "predominantly"], ["favourable", "dehydration", "pediatrics", "provincial"], ["intelligence", "watersheds", "pediatrics", "abnormality"], ["diagnostics", "discerning", "complained", "similarities"], ["manipulative", "kilometres", "sunglasses", "applicability"], ["manipulative", "foreseeable", "disclosure", "predominantly"], ["quantifying", "dehydration", "pediatrics", "universality"], ["persecuted", "watersheds", "sunglasses", "provincial"], ["contingency", "watersheds", "pediatrics", "applicability"], ["phonological", "watersheds", "sunglasses", "predominantly"], ["favourable", "prerequisites", "epithelium", "abnormality"], ["manipulative", "dehydration", "multiplied", "provincial"], ["favourable", "prerequisites", "sunglasses", "provincial"], ["persecuted", "discerning", "multiplied", "predominantly"], ["phonological", "kilometres", "sunglasses", "similarities"], ["phonological", "dehydration", "multiplied", "encountering"], ["persecuted", "intonation", "pediatrics", "predominantly"], ["manipulative", "foreseeable", "multiplied", "applicability"], ["manipulative", "unaffected", "companions", "similarities"], ["phonological", "prerequisites", "complained", "encountering"], ["favourable", "dehydration", "complained", "encountering"], ["diagnostics", "foreseeable", "multiplied", "abnormality"], ["quantifying", "unaffected", "disclosure", "applicability"], ["diagnostics", "foreseeable", "sunglasses", "encountering"], ["manipulative", "dehydration", "disclosure", "modalities"], ["contingency", "prerequisites", "epithelium", "encountering"], ["intelligence", "watersheds", "pediatrics", "provincial"], ["persecuted", "dehydration", "complained", "universality"], ["contingency", "foreseeable", "epithelium", "similarities"], ["quantifying", "watersheds", "complained", "universality"], ["quantifying", "foreseeable", "multiplied", "predominantly"], ["persecuted", "foreseeable", "epithelium", "abnormality"], ["persecuted", "watersheds", "multiplied", "abnormality"], ["manipulative", "intonation", "redistribute", "abnormality"], ["persecuted", "prerequisites", "sunglasses", "predominantly"], ["phonological", "intonation", "sunglasses", "similarities"], ["diagnostics", "unaffected", "sunglasses", "universality"], ["persecuted", "unaffected", "epithelium", "applicability"], ["contingency", "kilometres", "redistribute", "modalities"], ["favourable", "prerequisites", "epithelium", "modalities"], ["persecuted", "prerequisites", "companions", "abnormality"], ["contingency", "dehydration", "companions", "abnormality"], ["manipulative", "discerning", "disclosure", "abnormality"], ["quantifying", "prerequisites", "redistribute", "applicability"], ["manipulative", "discerning", "redistribute", "predominantly"], ["persecuted", "discerning", "multiplied", "universality"], ["favourable", "dehydration", "complained", "encountering"], ["quantifying", "prerequisites", "disclosure", "universality"], ["diagnostics", "dehydration", "epithelium", "predominantly"], ["contingency", "unaffected", "multiplied", "abnormality"], ["persecuted", "unaffected", "sunglasses", "universality"], ["favourable", "kilometres", "sunglasses", "abnormality"], ["intelligence", "discerning", "multiplied", "abnormality"], ["quantifying", "unaffected", "pediatrics", "universality"], ["phonological", "foreseeable", "complained", "predominantly"], ["phonological", "unaffected", "multiplied", "applicability"], ["diagnostics", "watersheds", "companions", "predominantly"], ["persecuted", "discerning", "sunglasses", "encountering"], ["quantifying", "watersheds", "companions", "encountering"], ["quantifying", "watersheds", "epithelium", "predominantly"], ["manipulative", "discerning", "complained", "modalities"], ["intelligence", "foreseeable", "complained", "encountering"], ["contingency", "prerequisites", "redistribute", "predominantly"], ["intelligence", "kilometres", "multiplied", "applicability"], ["manipulative", "intonation", "companions", "abnormality"], ["persecuted", "prerequisites", "epithelium", "provincial"], ["phonological", "discerning", "complained", "provincial"], ["manipulative", "foreseeable", "multiplied", "abnormality"], ["persecuted", "kilometres", "complained", "abnormality"], ["contingency", "watersheds", "companions", "provincial"], ["quantifying", "prerequisites", "complained", "modalities"], ["intelligence", "watersheds", "complained", "modalities"], ["persecuted", "foreseeable", "multiplied", "applicability"], ["favourable", "foreseeable", "disclosure", "predominantly"], ["quantifying", "intonation", "sunglasses", "predominantly"], ["persecuted", "discerning", "epithelium", "encountering"], ["persecuted", "intonation", "multiplied", "modalities"], ["favourable", "unaffected", "companions", "predominantly"], ["diagnostics", "watersheds", "multiplied", "universality"], ["phonological", "kilometres", "multiplied", "similarities"], ["favourable", "dehydration", "pediatrics", "provincial"], ["diagnostics", "discerning", "redistribute", "applicability"], ["phonological", "dehydration", "redistribute", "applicability"], ["favourable", "prerequisites", "companions", "similarities"], ["quantifying", "intonation", "redistribute", "similarities"], ["intelligence", "intonation", "multiplied", "abnormality"], ["persecuted", "intonation", "epithelium", "predominantly"], ["contingency", "discerning", "multiplied", "provincial"], ["persecuted", "unaffected", "redistribute", "modalities"], ["persecuted", "discerning", "sunglasses", "abnormality"], ["phonological", "unaffected", "redistribute", "applicability"], ["contingency", "intonation", "sunglasses", "encountering"], ["quantifying", "unaffected", "redistribute", "predominantly"], ["persecuted", "intonation", "pediatrics", "modalities"], ["persecuted", "watersheds", "complained", "abnormality"], ["intelligence", "prerequisites", "multiplied", "encountering"], ["diagnostics", "dehydration", "companions", "universality"], ["favourable", "dehydration", "complained", "encountering"], ["persecuted", "intonation", "complained", "provincial"], ["quantifying", "intonation", "epithelium", "similarities"], ["manipulative", "dehydration", "sunglasses", "applicability"], ["intelligence", "dehydration", "redistribute", "abnormality"], ["diagnostics", "intonation", "multiplied", "encountering"], ["manipulative", "prerequisites", "multiplied", "modalities"], ["manipulative", "foreseeable", "pediatrics", "universality"], ["manipulative", "dehydration", "pediatrics", "modalities"], ["contingency", "intonation", "redistribute", "modalities"], ["manipulative", "prerequisites", "disclosure", "universality"], ["manipulative", "prerequisites", "sunglasses", "provincial"], ["favourable", "intonation", "sunglasses", "similarities"], ["manipulative", "unaffected", "companions", "universality"], ["quantifying", "discerning", "disclosure", "similarities"], ["contingency", "dehydration", "pediatrics", "similarities"], ["contingency", "intonation", "multiplied", "modalities"], ["manipulative", "discerning", "redistribute", "universality"], ["phonological", "kilometres", "redistribute", "encountering"], ["favourable", "kilometres", "companions", "modalities"], ["phonological", "dehydration", "pediatrics", "predominantly"], ["phonological", "unaffected", "disclosure", "applicability"], ["persecuted", "watersheds", "sunglasses", "abnormality"], ["diagnostics", "prerequisites", "complained", "provincial"], ["quantifying", "dehydration", "redistribute", "universality"], ["phonological", "intonation", "companions", "abnormality"], ["manipulative", "watersheds", "complained", "universality"], ["favourable", "intonation", "disclosure", "modalities"], ["contingency", "dehydration", "redistribute", "similarities"], ["phonological", "prerequisites", "multiplied", "applicability"], ["favourable", "discerning", "redistribute", "abnormality"], ["quantifying", "unaffected", "sunglasses", "applicability"], ["favourable", "unaffected", "companions", "predominantly"], ["intelligence", "unaffected", "disclosure", "universality"], ["manipulative", "watersheds", "pediatrics", "provincial"], ["persecuted", "foreseeable", "redistribute", "modalities"], ["favourable", "foreseeable", "complained", "universality"], ["persecuted", "prerequisites", "pediatrics", "applicability"], ["persecuted", "watersheds", "multiplied", "modalities"], ["diagnostics", "dehydration", "disclosure", "provincial"], ["phonological", "foreseeable", "pediatrics", "modalities"], ["manipulative", "intonation", "multiplied", "predominantly"], ["quantifying", "intonation", "pediatrics", "similarities"], ["quantifying", "unaffected", "epithelium", "encountering"], ["quantifying", "kilometres", "complained", "abnormality"], ["contingency", "prerequisites", "sunglasses", "modalities"], ["phonological", "kilometres", "companions", "modalities"], ["quantifying", "foreseeable", "epithelium", "similarities"], ["intelligence", "intonation", "companions", "applicability"], ["persecuted", "discerning", "epithelium", "similarities"], ["phonological", "dehydration", "epithelium", "abnormality"], ["diagnostics", "unaffected", "sunglasses", "applicability"], ["quantifying", "intonation", "epithelium", "modalities"], ["intelligence", "intonation", "redistribute", "similarities"], ["contingency", "unaffected", "sunglasses", "similarities"], ["diagnostics", "discerning", "disclosure", "applicability"], ["contingency", "discerning", "sunglasses", "provincial"], ["diagnostics", "discerning", "redistribute", "modalities"], ["persecuted", "unaffected", "pediatrics", "universality"], ["contingency", "discerning", "sunglasses", "predominantly"], ["manipulative", "prerequisites", "redistribute", "applicability"], ["contingency", "prerequisites", "sunglasses", "encountering"], ["quantifying", "kilometres", "pediatrics", "encountering"], ["persecuted", "prerequisites", "companions", "applicability"], ["persecuted", "dehydration", "disclosure", "predominantly"], ["persecuted", "discerning", "multiplied", "predominantly"], ["persecuted", "unaffected", "multiplied", "provincial"], ["manipulative", "dehydration", "epithelium", "modalities"], ["diagnostics", "foreseeable", "companions", "universality"], ["favourable", "foreseeable", "multiplied", "universality"], ["quantifying", "prerequisites", "redistribute", "similarities"]], "sealed_id": [["check", "machine", "pending", "today"], ["write", "machine", "pending", "now"], ["delete", "pump", "ready", "day"], ["update", "meeting", "blocked", "hour"], ["stop", "customer", "open", "now"], ["read", "order", "overload", "minute"], ["report", "file", "open", "hour"], ["write", "invoice", "failed", "year"], ["start", "motor", "paid", "week"], ["start", "job", "blocked", "day"], ["delete", "job", "low", "hour"], ["notify", "motor", "blocked", "month"], ["report", "email", "open", "hour"], ["delete", "pump", "active", "month"], ["check", "machine", "blocked", "hour"], ["query", "meeting", "blocked", "month"], ["execute", "job", "active", "today"], ["delete", "inventory", "active", "now"], ["notify", "meeting", "pending", "hour"], ["start", "valve", "normal", "month"], ["write", "order", "ready", "week"], ["execute", "meeting", "normal", "today"], ["notify", "valve", "normal", "now"], ["check", "email", "none", "week"], ["read", "file", "low", "today"], ["notify", "machine", "running", "today"], ["notify", "machine", "complete", "hour"], ["read", "inventory", "pending", "year"], ["notify", "order", "running", "month"], ["check", "email", "low", "hour"], ["create", "invoice", "active", "tomorrow"], ["execute", "machine", "closed", "hour"], ["write", "invoice", "open", "year"], ["create", "order", "closed", "none"], ["delete", "email", "idle", "today"], ["create", "motor", "failed", "year"], ["stop", "machine", "none", "day"], ["update", "valve", "high", "tomorrow"], ["check", "job", "running", "hour"], ["read", "machine", "open", "week"], ["stop", "pump", "active", "month"], ["check", "machine", "ready", "minute"], ["check", "valve", "blocked", "hour"], ["check", "order", "low", "week"], ["check", "meeting", "running", "none"], ["read", "valve", "pending", "today"], ["create", "machine", "ready", "today"], ["check", "machine", "closed", "today"], ["report", "machine", "overload", "hour"], ["delete", "meeting", "blocked", "now"], ["delete", "valve", "blocked", "year"], ["write", "customer", "ready", "tomorrow"], ["execute", "machine", "normal", "minute"], ["delete", "job", "complete", "today"], ["write", "meeting", "failed", "week"], ["update", "meeting", "idle", "minute"], ["write", "pump", "idle", "none"], ["delete", "order", "active", "tomorrow"], ["create", "pump", "paid", "year"], ["report", "order", "normal", "now"], ["notify", "motor", "ready", "year"], ["check", "inventory", "active", "week"], ["execute", "email", "blocked", "now"], ["stop", "machine", "blocked", "year"], ["start", "customer", "running", "now"], ["write", "machine", "pending", "tomorrow"], ["start", "machine", "open", "month"], ["create", "file", "active", "day"], ["execute", "job", "normal", "minute"], ["read", "valve", "running", "tomorrow"], ["execute", "motor", "low", "hour"], ["query", "job", "ready", "week"], ["delete", "file", "open", "minute"], ["delete", "order", "low", "week"], ["execute", "email", "idle", "year"], ["read", "machine", "active", "now"], ["update", "meeting", "pending", "today"], ["report", "job", "low", "minute"], ["stop", "customer", "failed", "minute"], ["create", "order", "high", "tomorrow"], ["stop", "customer", "closed", "now"], ["query", "inventory", "overload", "today"], ["check", "job", "blocked", "hour"], ["read", "machine", "complete", "today"], ["read", "meeting", "blocked", "now"], ["notify", "machine", "blocked", "today"], ["execute", "customer", "low", "hour"], ["update", "customer", "overload", "year"], ["delete", "pump", "failed", "now"], ["start", "order", "complete", "minute"], ["check", "order", "overload", "minute"], ["write", "machine", "blocked", "hour"], ["stop", "pump", "pending", "minute"], ["create", "valve", "idle", "day"], ["write", "motor", "failed", "day"], ["check", "meeting", "overload", "today"], ["create", "machine", "active", "now"], ["update", "pump", "ready", "none"], ["stop", "machine", "high", "now"], ["stop", "motor", "pending", "hour"], ["query", "order", "ready", "minute"], ["notify", "valve", "pending", "minute"], ["execute", "meeting", "idle", "now"], ["read", "order", "high", "tomorrow"], ["read", "customer", "normal", "hour"], ["check", "email", "complete", "minute"], ["report", "invoice", "idle", "hour"], ["execute", "customer", "blocked", "now"], ["check", "job", "complete", "none"], ["query", "order", "paid", "minute"], ["start", "motor", "running", "now"], ["check", "pump", "low", "week"], ["query", "inventory", "overload", "minute"], ["report", "job", "blocked", "day"], ["check", "order", "ready", "year"], ["notify", "order", "open", "hour"], ["stop", "motor", "normal", "today"], ["execute", "invoice", "failed", "tomorrow"], ["notify", "machine", "open", "minute"], ["write", "pump", "high", "month"], ["execute", "email", "none", "tomorrow"], ["query", "order", "normal", "none"], ["delete", "order", "ready", "minute"], ["query", "motor", "active", "hour"], ["check", "invoice", "ready", "week"], ["write", "job", "high", "none"], ["report", "email", "normal", "none"], ["start", "file", "running", "hour"], ["notify", "inventory", "failed", "year"], ["update", "meeting", "high", "year"], ["read", "order", "none", "today"], ["create", "email", "pending", "today"], ["notify", "inventory", "running", "hour"], ["check", "meeting", "closed", "today"], ["write", "motor", "high", "month"], ["read", "machine", "normal", "now"], ["write", "pump", "overload", "tomorrow"], ["start", "inventory", "idle", "hour"], ["execute", "meeting", "normal", "now"], ["notify", "meeting", "blocked", "minute"], ["query", "motor", "overload", "tomorrow"], ["check", "email", "none", "minute"], ["update", "pump", "high", "year"], ["stop", "pump", "high", "tomorrow"], ["delete", "meeting", "active", "week"], ["report", "meeting", "normal", "minute"], ["start", "pump", "low", "hour"], ["start", "job", "ready", "now"], ["execute", "email", "normal", "month"], ["delete", "inventory", "normal", "hour"], ["notify", "meeting", "ready", "today"], ["query", "invoice", "low", "today"], ["notify", "invoice", "paid", "tomorrow"], ["update", "machine", "active", "week"], ["stop", "machine", "overload", "day"], ["report", "file", "closed", "month"], ["write", "invoice", "pending", "today"], ["notify", "motor", "idle", "now"], ["create", "motor", "active", "tomorrow"], ["write", "inventory", "open", "today"], ["update", "email", "normal", "today"], ["stop", "pump", "paid", "none"], ["report", "pump", "ready", "day"], ["stop", "job", "overload", "year"], ["stop", "valve", "blocked", "month"], ["delete", "motor", "normal", "week"], ["delete", "email", "high", "minute"], ["check", "machine", "blocked", "week"], ["check", "customer", "paid", "none"], ["query", "meeting", "high", "tomorrow"], ["stop", "pump", "overload", "week"], ["delete", "email", "paid", "today"], ["start", "job", "low", "minute"], ["stop", "invoice", "complete", "hour"], ["report", "file", "normal", "tomorrow"], ["notify", "motor", "high", "day"], ["start", "inventory", "ready", "day"], ["stop", "machine", "closed", "year"], ["update", "invoice", "complete", "hour"], ["create", "pump", "closed", "tomorrow"], ["create", "inventory", "active", "none"], ["execute", "pump", "none", "now"], ["execute", "invoice", "ready", "day"], ["read", "file", "running", "minute"], ["write", "job", "complete", "month"], ["start", "valve", "paid", "week"], ["update", "machine", "ready", "now"], ["check", "customer", "idle", "none"], ["stop", "motor", "idle", "now"], ["create", "email", "closed", "none"], ["notify", "meeting", "pending", "tomorrow"], ["notify", "customer", "complete", "month"], ["create", "email", "blocked", "week"], ["stop", "machine", "blocked", "hour"], ["delete", "email", "normal", "none"], ["create", "valve", "overload", "none"], ["delete", "inventory", "blocked", "week"], ["execute", "meeting", "overload", "hour"], ["stop", "customer", "high", "tomorrow"], ["check", "machine", "overload", "tomorrow"]], "sealed_ood": [["start", "invoice", "open", "none"], ["update", "job", "failed", "hour"], ["stop", "customer", "active", "year"], ["start", "meeting", "low", "month"], ["stop", "inventory", "active", "minute"], ["read", "pump", "paid", "tomorrow"], ["execute", "order", "paid", "day"], ["delete", "machine", "overload", "minute"], ["read", "inventory", "normal", "day"], ["read", "invoice", "failed", "month"], ["stop", "inventory", "failed", "minute"], ["write", "pump", "high", "today"], ["execute", "inventory", "high", "today"], ["write", "meeting", "high", "hour"], ["query", "file", "closed", "week"], ["create", "pump", "pending", "week"], ["delete", "customer", "idle", "year"], ["check", "motor", "pending", "tomorrow"], ["stop", "inventory", "active", "now"], ["report", "customer", "running", "day"], ["write", "email", "ready", "now"], ["read", "pump", "paid", "none"], ["update", "inventory", "blocked", "week"], ["notify", "job", "running", "hour"], ["create", "valve", "complete", "year"], ["check", "file", "running", "hour"], ["query", "file", "normal", "today"], ["write", "valve", "blocked", "tomorrow"], ["write", "valve", "low", "now"], ["query", "order", "ready", "month"], ["update", "order", "paid", "tomorrow"], ["start", "invoice", "none", "today"], ["write", "pump", "none", "month"], ["execute", "order", "high", "today"], ["notify", "job", "ready", "tomorrow"], ["write", "meeting", "low", "day"], ["check", "customer", "normal", "day"], ["start", "meeting", "blocked", "hour"], ["report", "motor", "overload", "minute"], ["check", "machine", "high", "today"], ["check", "file", "idle", "tomorrow"], ["check", "file", "high", "year"], ["write", "inventory", "pending", "week"], ["start", "meeting", "low", "today"], ["query", "machine", "closed", "today"], ["write", "valve", "paid", "month"], ["query", "motor", "ready", "month"], ["write", "valve", "idle", "minute"], ["stop", "inventory", "pending", "year"], ["start", "invoice", "paid", "none"], ["write", "email", "closed", "day"], ["execute", "valve", "running", "month"], ["read", "file", "blocked", "tomorrow"], ["write", "valve", "low", "day"], ["delete", "valve", "idle", "tomorrow"], ["query", "file", "paid", "year"], ["stop", "email", "open", "none"], ["check", "customer", "ready", "hour"], ["write", "customer", "high", "hour"], ["report", "motor", "normal", "minute"], ["execute", "order", "running", "tomorrow"], ["notify", "motor", "none", "month"], ["read", "machine", "high", "hour"], ["notify", "order", "closed", "week"], ["report", "customer", "failed", "none"], ["update", "order", "none", "year"], ["execute", "customer", "complete", "year"], ["stop", "order", "high", "hour"], ["update", "pump", "running", "day"], ["check", "motor", "open", "week"], ["execute", "customer", "overload", "now"], ["create", "machine", "active", "minute"], ["execute", "meeting", "closed", "day"], ["read", "email", "complete", "year"], ["create", "job", "running", "now"], ["start", "meeting", "running", "month"], ["query", "file", "overload", "year"], ["stop", "machine", "open", "tomorrow"], ["delete", "machine", "idle", "now"], ["create", "meeting", "active", "week"], ["update", "order", "high", "hour"], ["start", "machine", "overload", "month"], ["check", "file", "low", "year"], ["check", "pump", "complete", "day"], ["update", "inventory", "idle", "year"], ["check", "motor", "open", "day"], ["check", "file", "idle", "week"], ["read", "pump", "idle", "today"], ["write", "email", "overload", "today"], ["delete", "machine", "running", "now"], ["stop", "email", "pending", "month"], ["notify", "invoice", "blocked", "none"], ["update", "valve", "paid", "month"], ["notify", "email", "blocked", "tomorrow"], ["update", "meeting", "normal", "year"], ["write", "inventory", "normal", "day"], ["delete", "customer", "ready", "month"], ["report", "pump", "none", "hour"], ["create", "file", "high", "hour"], ["report", "customer", "high", "day"], ["write", "pump", "paid", "month"], ["start", "email", "open", "tomorrow"], ["update", "inventory", "running", "month"], ["notify", "customer", "running", "day"], ["notify", "valve", "high", "hour"], ["execute", "invoice", "active", "year"], ["query", "motor", "open", "none"], ["start", "pump", "closed", "day"], ["write", "meeting", "idle", "tomorrow"], ["query", "file", "normal", "none"], ["write", "motor", "blocked", "none"], ["execute", "email", "active", "year"], ["update", "order", "pending", "minute"], ["report", "motor", "open", "month"], ["query", "pump", "open", "tomorrow"], ["update", "customer", "failed", "today"], ["stop", "meeting", "complete", "day"], ["check", "valve", "failed", "today"], ["notify", "customer", "complete", "day"], ["delete", "pump", "active", "minute"], ["delete", "invoice", "failed", "hour"], ["check", "file", "normal", "hour"], ["report", "invoice", "high", "hour"], ["check", "motor", "high", "minute"], ["execute", "valve", "running", "minute"], ["notify", "motor", "idle", "tomorrow"], ["execute", "order", "none", "now"], ["execute", "order", "closed", "year"], ["check", "motor", "normal", "year"], ["execute", "valve", "active", "none"], ["report", "motor", "closed", "minute"], ["notify", "job", "ready", "month"], ["report", "customer", "failed", "minute"], ["report", "order", "overload", "month"], ["write", "valve", "closed", "tomorrow"], ["query", "machine", "idle", "minute"], ["execute", "inventory", "paid", "hour"], ["execute", "pump", "low", "year"], ["read", "job", "running", "year"], ["read", "customer", "active", "minute"], ["read", "job", "complete", "day"], ["stop", "inventory", "overload", "now"], ["query", "pump", "failed", "hour"], ["report", "customer", "pending", "now"], ["query", "meeting", "complete", "year"], ["update", "inventory", "normal", "minute"], ["check", "valve", "failed", "hour"], ["notify", "inventory", "closed", "day"], ["update", "inventory", "none", "hour"], ["read", "order", "ready", "hour"], ["read", "file", "ready", "month"], ["query", "file", "idle", "day"], ["stop", "inventory", "paid", "week"], ["start", "email", "low", "day"], ["report", "job", "ready", "month"], ["delete", "customer", "closed", "none"], ["check", "file", "running", "minute"], ["create", "email", "closed", "day"], ["start", "pump", "pending", "week"], ["update", "order", "idle", "year"], ["delete", "machine", "high", "tomorrow"], ["report", "customer", "paid", "today"], ["read", "email", "closed", "day"], ["check", "motor", "running", "week"], ["create", "job", "ready", "month"], ["delete", "machine", "blocked", "minute"], ["start", "customer", "paid", "hour"], ["notify", "pump", "ready", "tomorrow"], ["delete", "meeting", "blocked", "tomorrow"], ["query", "invoice", "failed", "today"], ["create", "job", "ready", "none"], ["read", "invoice", "paid", "minute"], ["read", "email", "paid", "month"], ["report", "customer", "overload", "week"], ["start", "invoice", "blocked", "minute"], ["execute", "valve", "paid", "month"], ["delete", "pump", "none", "month"], ["report", "invoice", "open", "tomorrow"], ["read", "invoice", "active", "minute"], ["delete", "machine", "blocked", "today"], ["check", "file", "open", "hour"], ["start", "invoice", "overload", "week"], ["write", "order", "open", "tomorrow"], ["check", "file", "normal", "day"], ["stop", "invoice", "low", "day"], ["start", "invoice", "idle", "minute"], ["start", "order", "paid", "month"], ["report", "customer", "low", "year"], ["stop", "motor", "idle", "tomorrow"], ["execute", "pump", "idle", "tomorrow"], ["execute", "order", "running", "year"], ["read", "valve", "high", "today"], ["write", "email", "high", "day"], ["notify", "pump", "pending", "week"], ["create", "motor", "normal", "year"], ["check", "file", "normal", "week"], ["update", "job", "low", "year"], ["read", "job", "high", "today"], ["write", "valve", "pending", "month"], ["write", "email", "paid", "month"], ["start", "meeting", "none", "year"], ["notify", "job", "none", "hour"], ["update", "valve", "blocked", "none"], ["execute", "order", "closed", "hour"], ["read", "pump", "blocked", "year"], ["check", "meeting", "blocked", "none"], ["read", "invoice", "open", "day"], ["delete", "machine", "overload", "hour"], ["start", "invoice", "low", "today"], ["query", "pump", "open", "none"], ["create", "job", "normal", "tomorrow"], ["delete", "inventory", "high", "today"], ["start", "invoice", "open", "hour"], ["notify", "pump", "low", "week"], ["stop", "inventory", "normal", "month"], ["stop", "inventory", "active", "none"], ["delete", "machine", "ready", "tomorrow"], ["execute", "customer", "low", "year"], ["delete", "machine", "active", "hour"], ["query", "machine", "normal", "day"], ["stop", "email", "running", "tomorrow"], ["create", "job", "paid", "tomorrow"], ["check", "meeting", "idle", "tomorrow"], ["notify", "pump", "running", "today"], ["write", "valve", "paid", "week"], ["report", "customer", "failed", "today"], ["report", "inventory", "normal", "year"], ["check", "valve", "none", "hour"], ["stop", "inventory", "pending", "today"], ["report", "motor", "active", "year"], ["stop", "email", "failed", "week"], ["stop", "inventory", "closed", "now"], ["write", "motor", "active", "year"], ["check", "job", "open", "tomorrow"], ["start", "valve", "high", "hour"], ["write", "email", "closed", "year"], ["create", "invoice", "failed", "today"], ["report", "customer", "complete", "year"], ["report", "machine", "low", "year"], ["write", "email", "normal", "year"], ["notify", "job", "none", "today"], ["check", "file", "active", "year"], ["create", "meeting", "closed", "year"], ["update", "inventory", "ready", "week"], ["delete", "machine", "complete", "tomorrow"], ["write", "file", "high", "today"], ["check", "file", "normal", "year"], ["start", "pump", "blocked", "none"], ["delete", "job", "normal", "year"], ["execute", "valve", "overload", "today"], ["execute", "order", "paid", "none"], ["read", "inventory", "blocked", "tomorrow"], ["execute", "valve", "normal", "hour"], ["execute", "customer", "closed", "day"], ["stop", "inventory", "closed", "month"], ["create", "order", "normal", "day"], ["check", "file", "overload", "month"], ["query", "file", "low", "hour"], ["delete", "machine", "paid", "year"], ["write", "meeting", "ready", "month"], ["stop", "inventory", "overload", "hour"], ["read", "valve", "overload", "now"], ["stop", "inventory", "normal", "minute"], ["report", "customer", "ready", "week"], ["write", "valve", "none", "month"], ["query", "machine", "ready", "now"], ["read", "file", "running", "day"], ["create", "meeting", "ready", "day"], ["delete", "machine", "idle", "day"], ["notify", "motor", "open", "tomorrow"], ["check", "order", "blocked", "tomorrow"], ["delete", "invoice", "none", "hour"], ["notify", "job", "pending", "tomorrow"], ["check", "motor", "blocked", "today"], ["update", "job", "idle", "tomorrow"], ["create", "valve", "failed", "hour"], ["update", "pump", "open", "none"], ["write", "customer", "closed", "day"], ["update", "meeting", "active", "year"], ["query", "pump", "low", "day"], ["execute", "valve", "failed", "week"], ["query", "machine", "running", "today"], ["start", "customer", "active", "year"], ["delete", "order", "overload", "month"], ["notify", "job", "none", "year"], ["notify", "order", "pending", "week"], ["stop", "email", "complete", "day"], ["read", "inventory", "overload", "now"], ["read", "invoice", "failed", "year"], ["delete", "customer", "pending", "today"], ["stop", "order", "overload", "month"], ["start", "motor", "overload", "now"], ["report", "job", "running", "year"], ["update", "inventory", "failed", "hour"], ["create", "meeting", "closed", "now"], ["update", "order", "running", "day"], ["write", "file", "open", "none"], ["execute", "pump", "complete", "day"], ["read", "invoice", "paid", "none"], ["read", "valve", "open", "none"], ["report", "customer", "low", "today"], ["check", "pump", "paid", "month"], ["write", "email", "none", "today"], ["update", "file", "ready", "month"], ["write", "valve", "complete", "none"], ["read", "pump", "running", "minute"], ["read", "invoice", "paid", "year"], ["read", "job", "running", "day"], ["update", "email", "failed", "hour"], ["write", "order", "ready", "hour"], ["read", "order", "pending", "week"], ["start", "pump", "active", "year"], ["check", "order", "normal", "day"], ["read", "motor", "ready", "month"], ["report", "order", "open", "none"], ["report", "customer", "closed", "hour"], ["stop", "email", "ready", "today"], ["delete", "job", "blocked", "tomorrow"], ["create", "valve", "low", "day"], ["delete", "machine", "ready", "minute"], ["write", "email", "paid", "hour"], ["start", "meeting", "overload", "today"], ["update", "job", "closed", "week"], ["create", "valve", "open", "none"], ["execute", "machine", "active", "minute"], ["update", "valve", "none", "month"], ["report", "motor", "active", "day"], ["delete", "machine", "open", "tomorrow"], ["stop", "inventory", "high", "none"], ["check", "inventory", "ready", "hour"], ["write", "meeting", "pending", "day"], ["check", "valve", "high", "today"], ["report", "motor", "paid", "hour"], ["report", "customer", "pending", "week"], ["query", "machine", "overload", "hour"], ["read", "email", "running", "year"], ["report", "motor", "active", "none"], ["query", "file", "normal", "week"], ["create", "meeting", "none", "tomorrow"], ["start", "email", "none", "hour"], ["check", "file", "closed", "now"], ["delete", "file", "idle", "tomorrow"], ["check", "machine", "idle", "tomorrow"], ["report", "customer", "high", "today"], ["notify", "pump", "overload", "tomorrow"], ["start", "pump", "idle", "tomorrow"], ["check", "file", "active", "tomorrow"], ["check", "meeting", "high", "hour"], ["execute", "email", "ready", "month"], ["check", "motor", "running", "today"], ["check", "valve", "blocked", "none"], ["query", "valve", "running", "year"], ["write", "email", "active", "month"], ["report", "pump", "overload", "month"], ["write", "valve", "high", "week"], ["check", "motor", "running", "month"], ["report", "motor", "high", "hour"], ["read", "pump", "overload", "year"], ["create", "motor", "low", "day"], ["start", "customer", "idle", "tomorrow"], ["read", "invoice", "paid", "hour"], ["delete", "customer", "failed", "week"], ["read", "valve", "none", "hour"], ["create", "file", "open", "tomorrow"], ["read", "invoice", "closed", "now"], ["notify", "invoice", "complete", "year"], ["delete", "customer", "paid", "week"], ["stop", "machine", "paid", "month"], ["check", "inventory", "failed", "today"], ["create", "job", "active", "month"], ["report", "motor", "idle", "today"], ["read", "customer", "complete", "day"], ["check", "file", "blocked", "now"], ["notify", "machine", "high", "today"], ["read", "invoice", "none", "minute"], ["stop", "email", "closed", "none"], ["notify", "job", "running", "tomorrow"], ["write", "valve", "failed", "none"], ["notify", "pump", "overload", "none"], ["read", "motor", "high", "today"], ["check", "invoice", "idle", "tomorrow"], ["update", "customer", "normal", "year"], ["query", "machine", "high", "now"], ["delete", "machine", "ready", "day"], ["create", "job", "pending", "today"], ["delete", "job", "failed", "hour"], ["create", "order", "paid", "hour"], ["write", "invoice", "overload", "now"], ["write", "valve", "active", "year"], ["report", "inventory", "low", "year"], ["stop", "motor", "none", "month"], ["stop", "email", "low", "minute"], ["check", "file", "pending", "day"], ["check", "file", "open", "none"], ["start", "file", "low", "year"], ["report", "motor", "failed", "today"], ["query", "motor", "failed", "hour"], ["start", "file", "open", "tomorrow"], ["read", "email", "blocked", "tomorrow"], ["check", "motor", "running", "none"]], "seed": 20261016}
|
021/c021.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CUERDAS-021 common: sealed set, varied replacement pool, training loop, evaluation.
|
| 2 |
+
import os, sys, json, hashlib, random, re, time
|
| 3 |
+
import torch, torch.nn as nn, torch.nn.functional as F
|
| 4 |
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 5 |
+
import run018 as R
|
| 6 |
+
INP = os.environ['INP']; OD = os.environ['OD']
|
| 7 |
+
raw = open(f'{INP}/sealed021.json').read(); H = hashlib.sha256(raw.encode()).hexdigest()
|
| 8 |
+
assert H == 'a86a6a5ee375ae67a3e79f969c9553b0e5548ed3b0e80e5315862e6552ae72bc', H
|
| 9 |
+
S = json.loads(raw); NEW = S['new_values']; EXCL = set(S['exclude_token_ids'])
|
| 10 |
+
SETS = ['sealed_id', 'sealed_ood', 'sealed_1_new', 'sealed_2_new', 'sealed_4_new']
|
| 11 |
+
SMOKE = bool(os.environ.get('SMOKE'))
|
| 12 |
+
if SMOKE: # dry run on dev messages only; sealed sets are never touched
|
| 13 |
+
for s_ in SETS: S[s_] = R.FRESH['ood'][:6]
|
| 14 |
+
STEPS = 4 if SMOKE else 3000; EVERY = 2 if SMOKE else 250
|
| 15 |
+
t1 = R.t1; tok = R.tok; Emb = R.Emb; D = R.D; FIELDS = R.FIELDS; V = R.V
|
| 16 |
+
CAND = [torch.tensor(sorted({t1(w)[0] for w in V[j] + NEW[j]})) for j in range(4)]
|
| 17 |
+
assert all(len(CAND[j]) == len(V[j]) + 8 for j in range(4))
|
| 18 |
+
def gidx(cand, j, w): return int((cand[j] == t1(w)[0]).nonzero()[0])
|
| 19 |
+
# ---- varied pool (training) and held-out validation pool; all disjoint from sealed tokens and from 014/dev/020 values ----
|
| 20 |
+
voc = tok.get_vocab(); BAN = EXCL | R.BANNED
|
| 21 |
+
def ok_tok(t): return voc[t] not in BAN
|
| 22 |
+
WORDS = sorted(t[1:] for t in voc if re.fullmatch(r'Ġ[a-z]{3,14}', t) and ok_tok(t))
|
| 23 |
+
NAMES = sorted(t[1:] for t in voc if re.fullmatch(r'Ġ[A-Z][a-z]{2,10}', t) and ok_tok(t))
|
| 24 |
+
SUBS = sorted(t for t in voc if re.fullmatch(r'[a-z]{2,5}', t))
|
| 25 |
+
g0 = random.Random(2101); g0.shuffle(WORDS); g0.shuffle(NAMES)
|
| 26 |
+
VW, VN = WORDS[-300:], NAMES[-300:]; WORDS, NAMES = WORDS[:-300], NAMES[:-300]
|
| 27 |
+
def sample_value(g, W_=None, N_=None):
|
| 28 |
+
W_ = W_ or WORDS; N_ = N_ or NAMES; k = g.randrange(5)
|
| 29 |
+
if k == 0: return g.choice(W_)
|
| 30 |
+
if k == 1: return g.choice(N_)
|
| 31 |
+
if k == 2: return str(g.randrange(10000))
|
| 32 |
+
if k == 3: return ' '.join(g.choice(W_) for _ in range(g.choice((2, 3))))
|
| 33 |
+
return g.choice(W_) + g.choice(SUBS)
|
| 34 |
+
def sample_msgs(g, bs=12):
|
| 35 |
+
out = []
|
| 36 |
+
for _ in range(bs):
|
| 37 |
+
m = list(g.choice(R.FIT))
|
| 38 |
+
for j in range(4):
|
| 39 |
+
if g.random() < .4: m[j] = sample_value(g)
|
| 40 |
+
out.append(m)
|
| 41 |
+
return out
|
| 42 |
+
# restricted transfer validation: 50 held-out values per field (no numbers: first token not unique), options = field vocab + those
|
| 43 |
+
gv = random.Random(2102); CANDV = []; VPOOL = []
|
| 44 |
+
for j in range(4):
|
| 45 |
+
used = {t1(w)[0] for w in V[j]}; ws = []
|
| 46 |
+
while len(ws) < 50:
|
| 47 |
+
k = gv.randrange(3); w = gv.choice(VW) if k == 0 else gv.choice(VN) if k == 1 else gv.choice(VW) + ' ' + gv.choice(VW)
|
| 48 |
+
if t1(w)[0] not in used: used.add(t1(w)[0]); ws.append(w)
|
| 49 |
+
VPOOL.append(ws); CANDV.append(torch.tensor(sorted(used)))
|
| 50 |
+
VAL_TR = [[gv.choice(VPOOL[j]) for j in range(4)] for _ in range(96)]
|
| 51 |
+
VALC = R.VAL[:96]
|
| 52 |
+
if SMOKE: VALC, VAL_TR = VALC[:6], VAL_TR[:6]
|
| 53 |
+
# ---- generic readout: rows = prefix (embeddings) + body tokens ----
|
| 54 |
+
def run_h(embs):
|
| 55 |
+
L = max(e.shape[0] for e in embs); B = len(embs); x = torch.zeros(B, L, D); m = torch.zeros(B, L, dtype=torch.long)
|
| 56 |
+
for i, e in enumerate(embs): x[i, :e.shape[0]] = e; m[i, :e.shape[0]] = 1
|
| 57 |
+
h = R.lm.model(inputs_embeds=x, attention_mask=m).last_hidden_state[torch.arange(B), m.sum(1) - 1]
|
| 58 |
+
return h, R.lm.lm_head(h)
|
| 59 |
+
def k2_rows(cod, msgs, tpl):
|
| 60 |
+
Sl = cod(torch.stack([torch.stack([R.lex(w) for w in m]) for m in msgs]))
|
| 61 |
+
return [torch.cat([Sl[i], Emb[R.sufids(tpl, j)]]) for i in range(len(msgs)) for j in range(4)]
|
| 62 |
+
def text_rows(msgs, tpl, fmt, prefix=None):
|
| 63 |
+
rows = []
|
| 64 |
+
for m in msgs:
|
| 65 |
+
for j in range(4):
|
| 66 |
+
e = Emb[tok.encode(fmt.format(**dict(zip(FIELDS, m))) + tpl.format(field=FIELDS[j]), add_special_tokens=False)]
|
| 67 |
+
rows.append(e if prefix is None else torch.cat([prefix, e]))
|
| 68 |
+
return rows
|
| 69 |
+
@torch.no_grad()
|
| 70 |
+
def per_ex(rowfn, msgs, cand, bs=24):
|
| 71 |
+
out = []
|
| 72 |
+
for b in range(0, len(msgs), bs):
|
| 73 |
+
mb = msgs[b:b+bs]; _, lg = run_h(rowfn(mb)); good = [True] * len(mb)
|
| 74 |
+
for r_ in range(len(mb) * 4):
|
| 75 |
+
i, j = divmod(r_, 4); good[i] &= int(lg[r_][cand[j]].argmax()) == gidx(cand, j, mb[i][j])
|
| 76 |
+
out += good
|
| 77 |
+
return [float(x) for x in out]
|
| 78 |
+
def val_score(rowfn_main):
|
| 79 |
+
return sum(per_ex(rowfn_main, VALC, R.CAND)) / len(VALC) + sum(per_ex(rowfn_main, VAL_TR, CANDV)) / len(VAL_TR)
|
| 80 |
+
def train(params, rowfn, teach, seed, tag, log, steps=None, lr=5e-4, every=None, set_mode=lambda t: None):
|
| 81 |
+
"""Same loss/schedule/selection as 018 distill2: CE + KL(teacher) + 0.5(1-cos) ; OneCycle; select on validation only."""
|
| 82 |
+
steps = steps or STEPS; every = every or EVERY
|
| 83 |
+
g = random.Random(seed + 2100); opt = torch.optim.AdamW(params, lr=lr, weight_decay=1e-3)
|
| 84 |
+
sch = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=lr, total_steps=steps, pct_start=.05)
|
| 85 |
+
def snap(): return [p.detach().clone() for p in params]
|
| 86 |
+
set_mode(False); v = val_score(lambda mb: rowfn(mb, R.SUF_MAIN)); set_mode(True); best = (v, snap(), 0); log({'run': tag, 'step': 0, 'val': v})
|
| 87 |
+
t0 = time.time()
|
| 88 |
+
for s_ in range(1, steps + 1):
|
| 89 |
+
msgs = sample_msgs(g); tpl = g.choice(R.SUF_TRAIN)
|
| 90 |
+
with torch.no_grad(): ht, lt = R.readout_h(teach, msgs, tpl)
|
| 91 |
+
hs, ls = run_h(rowfn(msgs, tpl)); tg = torch.tensor([t1(m[j])[0] for m in msgs for j in range(4)])
|
| 92 |
+
loss = F.cross_entropy(ls, tg) + F.kl_div(F.log_softmax(ls, -1), F.log_softmax(lt, -1), log_target=True, reduction='batchmean') + .5 * (1 - F.cosine_similarity(hs, ht, -1).mean())
|
| 93 |
+
opt.zero_grad(); loss.backward(); torch.nn.utils.clip_grad_norm_(params, 1.0); opt.step(); sch.step()
|
| 94 |
+
if s_ % every == 0:
|
| 95 |
+
set_mode(False); v = val_score(lambda mb: rowfn(mb, R.SUF_MAIN)); set_mode(True)
|
| 96 |
+
log({'run': tag, 'step': s_, 'val': v, 'loss': float(loss), 's': round(time.time() - t0)})
|
| 97 |
+
if v > best[0]: best = (v, snap(), s_)
|
| 98 |
+
with torch.no_grad():
|
| 99 |
+
for p, b in zip(params, best[1]): p.copy_(b)
|
| 100 |
+
set_mode(False); return best[2], best[0]
|
| 101 |
+
def load_teacher(sd):
|
| 102 |
+
t = R.Binder(sd); t.load_state_dict(torch.load(f'{INP}/binder4_1000_s{sd}.pt', map_location=R.DEV)); return t.eval().requires_grad_(False)
|
| 103 |
+
def opener(name):
|
| 104 |
+
f = open(f'{OD}/{name}', 'w')
|
| 105 |
+
def log(d): f.write(json.dumps(d) + '\n'); f.flush(); print(json.dumps(d), flush=True)
|
| 106 |
+
return log
|
021/cuerdas021-a.log
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[{"stream_name":"stdout","time":6.186180912,"data":" ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 44.1/44.1 kB 1.4 MB/s eta 0:00:00\n"}
|
| 2 |
+
,{"stream_name":"stdout","time":7.168235528,"data":" ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 10.0/10.0 MB 81.7 MB/s eta 0:00:00\n"}
|
| 3 |
+
,{"stream_name":"stdout","time":7.200287347,"data":" ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 566.4/566.4 kB 29.8 MB/s eta 0:00:00\n"}
|
| 4 |
+
,{"stream_name":"stdout","time":7.269625736,"data":" ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.0/3.0 MB 53.7 MB/s eta 0:00:00\n"}
|
| 5 |
+
,{"stream_name":"stderr","time":21.344989509,"data":"ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n"}
|
| 6 |
+
,{"stream_name":"stderr","time":21.345019411,"data":"litellm 1.85.7 requires tokenizers\u003c1.0,\u003e=0.21.0, but you have tokenizers 0.20.3 which is incompatible.\n"}
|
| 7 |
+
,{"stream_name":"stderr","time":21.345025011,"data":"gradio 6.26.0 requires huggingface-hub\u003c2.0,\u003e=1.16.0, but you have huggingface-hub 0.36.2 which is incompatible.\n"}
|
| 8 |
+
,{"stream_name":"stderr","time":21.345030991,"data":"diffusers 0.40.0 requires huggingface-hub\u003c2.0,\u003e=1.23.0, but you have huggingface-hub 0.36.2 which is incompatible.\n"}
|
| 9 |
+
,{"stream_name":"stdout","time":75.159877172,"data":"{\"sealed_sha256\": \"a86a6a5ee375ae67a3e79f969c9553b0e5548ed3b0e80e5315862e6552ae72bc\", \"torch\": \"2.11.0+cu128\", \"pool_sizes\": {\"words\": 20654, \"names\": 8188, \"subs\": 7189}}\n"}
|
| 10 |
+
,{"stream_name":"stdout","time":88.355573409,"data":"{\"sys\": \"text_template\", \"sealed_id\": 1.0, \"sealed_ood\": 0.985, \"sealed_1_new\": 0.96875, \"sealed_2_new\": 0.9739583333333334, \"sealed_4_new\": 0.984375}\n"}
|
| 11 |
+
,{"stream_name":"stdout","time":95.592428269,"data":"{\"sys\": \"k2_018_s17\", \"sealed_id\": 1.0, \"sealed_ood\": 0.99, \"sealed_1_new\": 0.875, \"sealed_2_new\": 0.625, \"sealed_4_new\": 0.2864583333333333}\n"}
|
| 12 |
+
,{"stream_name":"stdout","time":102.609933367,"data":"{\"sys\": \"k2_018_s29\", \"sealed_id\": 0.995, \"sealed_ood\": 0.99, \"sealed_1_new\": 0.875, \"sealed_2_new\": 0.578125, \"sealed_4_new\": 0.265625}\n"}
|
| 13 |
+
,{"stream_name":"stdout","time":109.535114583,"data":"{\"sys\": \"k2_018_s43\", \"sealed_id\": 0.975, \"sealed_ood\": 0.9775, \"sealed_1_new\": 0.7552083333333334, \"sealed_2_new\": 0.4583333333333333, \"sealed_4_new\": 0.25}\n"}
|
| 14 |
+
,{"stream_name":"stdout","time":110.769495618,"data":"{\"run\": \"k2_021_s17\", \"step\": 0, \"val\": 1.1145833333333333}\n"}
|
| 15 |
+
,{"stream_name":"stderr","time":161.364732013,"data":"/usr/local/lib/python3.13/dist-packages/torch/utils/_device.py:116: UserWarning: Converting a tensor with requires_grad=True to a scalar may lead to unexpected behavior.\n"}
|
| 16 |
+
,{"stream_name":"stderr","time":161.364784997,"data":"Consider using tensor.detach() first. (Triggered internally at /pytorch/torch/csrc/autograd/generated/python_variable_methods.cpp:836.)\n"}
|
| 17 |
+
,{"stream_name":"stderr","time":161.364795195,"data":" return func(*args, **kwargs)\n"}
|
| 18 |
+
,{"stream_name":"stdout","time":161.364818194,"data":"{\"run\": \"k2_021_s17\", \"step\": 250, \"val\": 1.0833333333333333, \"loss\": 4.719815254211426, \"s\": 51}\n"}
|
| 19 |
+
,{"stream_name":"stdout","time":212.200652872,"data":"{\"run\": \"k2_021_s17\", \"step\": 500, \"val\": 1.0833333333333333, \"loss\": 5.05856990814209, \"s\": 101}\n"}
|
| 20 |
+
,{"stream_name":"stdout","time":264.450103516,"data":"{\"run\": \"k2_021_s17\", \"step\": 750, \"val\": 1.0625, \"loss\": 4.387829780578613, \"s\": 154}\n"}
|
| 21 |
+
,{"stream_name":"stdout","time":316.827434258,"data":"{\"run\": \"k2_021_s17\", \"step\": 1000, \"val\": 1.09375, \"loss\": 4.11569881439209, \"s\": 206}\n"}
|
| 22 |
+
,{"stream_name":"stdout","time":369.502860696,"data":"{\"run\": \"k2_021_s17\", \"step\": 1250, \"val\": 1.1354166666666667, \"loss\": 4.992859363555908, \"s\": 259}\n"}
|
| 23 |
+
,{"stream_name":"stdout","time":422.265248199,"data":"{\"run\": \"k2_021_s17\", \"step\": 1500, \"val\": 1.1458333333333335, \"loss\": 4.392324447631836, \"s\": 311}\n"}
|
| 24 |
+
,{"stream_name":"stdout","time":474.904848973,"data":"{\"run\": \"k2_021_s17\", \"step\": 1750, \"val\": 1.1354166666666667, \"loss\": 4.7229413986206055, \"s\": 364}\n"}
|
| 25 |
+
,{"stream_name":"stdout","time":527.486876182,"data":"{\"run\": \"k2_021_s17\", \"step\": 2000, \"val\": 1.1458333333333333, \"loss\": 4.124819278717041, \"s\": 417}\n"}
|
| 26 |
+
,{"stream_name":"stdout","time":579.923370702,"data":"{\"run\": \"k2_021_s17\", \"step\": 2250, \"val\": 1.1666666666666667, \"loss\": 3.794137477874756, \"s\": 469}\n"}
|
| 27 |
+
,{"stream_name":"stdout","time":631.86940922,"data":"{\"run\": \"k2_021_s17\", \"step\": 2500, \"val\": 1.1666666666666667, \"loss\": 2.527881622314453, \"s\": 521}\n"}
|
| 28 |
+
,{"stream_name":"stdout","time":684.082385,"data":"{\"run\": \"k2_021_s17\", \"step\": 2750, \"val\": 1.1770833333333333, \"loss\": 3.1677093505859375, \"s\": 573}\n"}
|
| 29 |
+
,{"stream_name":"stdout","time":736.534519603,"data":"{\"run\": \"k2_021_s17\", \"step\": 3000, \"val\": 1.1666666666666667, \"loss\": 1.943210482597351, \"s\": 626}\n"}
|
| 30 |
+
,{"stream_name":"stdout","time":736.561427792,"data":"{\"selected\": \"k2_021_s17\", \"step\": 2750, \"val\": 1.1770833333333333}\n"}
|
| 31 |
+
,{"stream_name":"stdout","time":744.219044901,"data":"{\"sys\": \"k2_021_s17\", \"sealed_id\": 1.0, \"sealed_ood\": 0.9975, \"sealed_1_new\": 0.90625, \"sealed_2_new\": 0.6302083333333334, \"sealed_4_new\": 0.3125}\n"}
|
| 32 |
+
,{"stream_name":"stdout","time":745.734205364,"data":"{\"run\": \"k2_021_s29\", \"step\": 0, \"val\": 1.0729166666666667}\n"}
|
| 33 |
+
,{"stream_name":"stdout","time":797.886349926,"data":"{\"run\": \"k2_021_s29\", \"step\": 250, \"val\": 1.0520833333333335, \"loss\": 4.769656181335449, \"s\": 52}\n"}
|
| 34 |
+
,{"stream_name":"stdout","time":850.649595978,"data":"{\"run\": \"k2_021_s29\", \"step\": 500, \"val\": 1.0833333333333333, \"loss\": 4.504295825958252, \"s\": 105}\n"}
|
| 35 |
+
,{"stream_name":"stdout","time":902.865580802,"data":"{\"run\": \"k2_021_s29\", \"step\": 750, \"val\": 1.15625, \"loss\": 3.0690648555755615, \"s\": 157}\n"}
|
| 36 |
+
,{"stream_name":"stdout","time":955.496481741,"data":"{\"run\": \"k2_021_s29\", \"step\": 1000, \"val\": 1.125, \"loss\": 2.2323856353759766, \"s\": 210}\n"}
|
| 37 |
+
,{"stream_name":"stdout","time":1008.341980537,"data":"{\"run\": \"k2_021_s29\", \"step\": 1250, \"val\": 1.1666666666666667, \"loss\": 4.31823205947876, \"s\": 263}\n"}
|
| 38 |
+
,{"stream_name":"stdout","time":1060.535361573,"data":"{\"run\": \"k2_021_s29\", \"step\": 1500, \"val\": 1.1770833333333333, \"loss\": 4.3195061683654785, \"s\": 315}\n"}
|
| 39 |
+
,{"stream_name":"stdout","time":1112.771534115,"data":"{\"run\": \"k2_021_s29\", \"step\": 1750, \"val\": 1.1354166666666667, \"loss\": 3.4602582454681396, \"s\": 367}\n"}
|
| 40 |
+
,{"stream_name":"stdout","time":1164.997576792,"data":"{\"run\": \"k2_021_s29\", \"step\": 2000, \"val\": 1.1041666666666667, \"loss\": 3.7912955284118652, \"s\": 419}\n"}
|
| 41 |
+
,{"stream_name":"stdout","time":1217.644480861,"data":"{\"run\": \"k2_021_s29\", \"step\": 2250, \"val\": 1.09375, \"loss\": 3.81158709526062, \"s\": 472}\n"}
|
| 42 |
+
,{"stream_name":"stdout","time":1270.042174474,"data":"{\"run\": \"k2_021_s29\", \"step\": 2500, \"val\": 1.1354166666666667, \"loss\": 2.8178346157073975, \"s\": 524}\n"}
|
| 43 |
+
,{"stream_name":"stdout","time":1322.556426989,"data":"{\"run\": \"k2_021_s29\", \"step\": 2750, \"val\": 1.1145833333333333, \"loss\": 3.228869676589966, \"s\": 577}\n"}
|
| 44 |
+
,{"stream_name":"stdout","time":1374.978668944,"data":"{\"run\": \"k2_021_s29\", \"step\": 3000, \"val\": 1.1145833333333333, \"loss\": 3.688642978668213, \"s\": 629}\n"}
|
| 45 |
+
,{"stream_name":"stdout","time":1374.998128133,"data":"{\"selected\": \"k2_021_s29\", \"step\": 1500, \"val\": 1.1770833333333333}\n"}
|
| 46 |
+
,{"stream_name":"stdout","time":1382.613613675,"data":"{\"sys\": \"k2_021_s29\", \"sealed_id\": 0.99, \"sealed_ood\": 0.985, \"sealed_1_new\": 0.8854166666666666, \"sealed_2_new\": 0.5677083333333334, \"sealed_4_new\": 0.3020833333333333}\n"}
|
| 47 |
+
,{"stream_name":"stdout","time":1384.182599023,"data":"{\"run\": \"k2_021_s43\", \"step\": 0, \"val\": 1.1041666666666667}\n"}
|
| 48 |
+
,{"stream_name":"stdout","time":1436.533073168,"data":"{\"run\": \"k2_021_s43\", \"step\": 250, \"val\": 1.0625, \"loss\": 3.4185612201690674, \"s\": 52}\n"}
|
| 49 |
+
,{"stream_name":"stdout","time":1488.680435255,"data":"{\"run\": \"k2_021_s43\", \"step\": 500, \"val\": 1.0, \"loss\": 4.755449295043945, \"s\": 104}\n"}
|
| 50 |
+
,{"stream_name":"stdout","time":1541.653197742,"data":"{\"run\": \"k2_021_s43\", \"step\": 750, \"val\": 1.0520833333333335, \"loss\": 4.684804916381836, \"s\": 157}\n"}
|
| 51 |
+
,{"stream_name":"stdout","time":1594.18542137,"data":"{\"run\": \"k2_021_s43\", \"step\": 1000, \"val\": 1.09375, \"loss\": 4.735645771026611, \"s\": 210}\n"}
|
| 52 |
+
,{"stream_name":"stdout","time":1647.036378086,"data":"{\"run\": \"k2_021_s43\", \"step\": 1250, \"val\": 1.1458333333333333, \"loss\": 3.7620046138763428, \"s\": 263}\n"}
|
| 53 |
+
,{"stream_name":"stdout","time":1699.512461238,"data":"{\"run\": \"k2_021_s43\", \"step\": 1500, \"val\": 1.1354166666666667, \"loss\": 4.758123397827148, \"s\": 315}\n"}
|
| 54 |
+
,{"stream_name":"stdout","time":1751.688814083,"data":"{\"run\": \"k2_021_s43\", \"step\": 1750, \"val\": 1.1145833333333333, \"loss\": 4.265966892242432, \"s\": 368}\n"}
|
| 55 |
+
,{"stream_name":"stdout","time":1803.454996524,"data":"{\"run\": \"k2_021_s43\", \"step\": 2000, \"val\": 1.0625, \"loss\": 3.849872589111328, \"s\": 419}\n"}
|
| 56 |
+
,{"stream_name":"stdout","time":1855.058249654,"data":"{\"run\": \"k2_021_s43\", \"step\": 2250, \"val\": 1.1041666666666665, \"loss\": 4.972720146179199, \"s\": 471}\n"}
|
| 57 |
+
,{"stream_name":"stdout","time":1907.043534368,"data":"{\"run\": \"k2_021_s43\", \"step\": 2500, \"val\": 1.125, \"loss\": 2.9099667072296143, \"s\": 523}\n"}
|
| 58 |
+
,{"stream_name":"stdout","time":1958.882514238,"data":"{\"run\": \"k2_021_s43\", \"step\": 2750, \"val\": 1.125, \"loss\": 3.000356674194336, \"s\": 575}\n"}
|
| 59 |
+
,{"stream_name":"stdout","time":2010.517405519,"data":"{\"run\": \"k2_021_s43\", \"step\": 3000, \"val\": 1.125, \"loss\": 2.915379762649536, \"s\": 626}\n"}
|
| 60 |
+
,{"stream_name":"stdout","time":2010.535140228,"data":"{\"selected\": \"k2_021_s43\", \"step\": 1250, \"val\": 1.1458333333333333}\n"}
|
| 61 |
+
,{"stream_name":"stdout","time":2018.04167291,"data":"{\"sys\": \"k2_021_s43\", \"sealed_id\": 0.995, \"sealed_ood\": 0.9775, \"sealed_1_new\": 0.6614583333333334, \"sealed_2_new\": 0.3802083333333333, \"sealed_4_new\": 0.265625}\n"}
|
| 62 |
+
,{"stream_name":"stdout","time":2075.919319203,"data":"{\"probe\": \"ridge_aug\", \"seed\": 17, \"set\": \"sealed_ood\", \"acc\": 1.0}\n"}
|
| 63 |
+
,{"stream_name":"stdout","time":2076.041626453,"data":"{\"probe\": \"ridge_aug\", \"seed\": 17, \"set\": \"sealed_1_new\", \"acc\": 0.890625}\n"}
|
| 64 |
+
,{"stream_name":"stdout","time":2076.170779267,"data":"{\"probe\": \"ridge_aug\", \"seed\": 17, \"set\": \"sealed_2_new\", \"acc\": 0.7135416666666666}\n"}
|
| 65 |
+
,{"stream_name":"stdout","time":2076.301331179,"data":"{\"probe\": \"ridge_aug\", \"seed\": 17, \"set\": \"sealed_4_new\", \"acc\": 0.4322916666666667}\n"}
|
| 66 |
+
,{"stream_name":"stdout","time":2076.563257368,"data":"{\"probe\": \"mlp_aug\", \"seed\": 17, \"set\": \"sealed_ood\", \"acc\": 1.0}\n"}
|
| 67 |
+
,{"stream_name":"stdout","time":2076.684802789,"data":"{\"probe\": \"mlp_aug\", \"seed\": 17, \"set\": \"sealed_1_new\", \"acc\": 1.0}\n"}
|
| 68 |
+
,{"stream_name":"stdout","time":2076.808395106,"data":"{\"probe\": \"mlp_aug\", \"seed\": 17, \"set\": \"sealed_2_new\", \"acc\": 0.984375}\n"}
|
| 69 |
+
,{"stream_name":"stdout","time":2076.931173743,"data":"{\"probe\": \"mlp_aug\", \"seed\": 17, \"set\": \"sealed_4_new\", \"acc\": 0.24479166666666666}\n"}
|
| 70 |
+
,{"stream_name":"stdout","time":2077.182778765,"data":"{\"probe\": \"mlp_ctrl_shuffled\", \"seed\": 17, \"set\": \"sealed_ood\", \"acc\": 0.0}\n"}
|
| 71 |
+
,{"stream_name":"stdout","time":2077.302429164,"data":"{\"probe\": \"mlp_ctrl_shuffled\", \"seed\": 17, \"set\": \"sealed_1_new\", \"acc\": 0.0}\n"}
|
| 72 |
+
,{"stream_name":"stdout","time":2077.423131583,"data":"{\"probe\": \"mlp_ctrl_shuffled\", \"seed\": 17, \"set\": \"sealed_2_new\", \"acc\": 0.0}\n"}
|
| 73 |
+
,{"stream_name":"stdout","time":2077.545791347,"data":"{\"probe\": \"mlp_ctrl_shuffled\", \"seed\": 17, \"set\": \"sealed_4_new\", \"acc\": 0.0}\n"}
|
| 74 |
+
,{"stream_name":"stdout","time":2121.917288868,"data":"{\"probe\": \"ridge_aug\", \"seed\": 29, \"set\": \"sealed_ood\", \"acc\": 1.0}\n"}
|
| 75 |
+
,{"stream_name":"stdout","time":2122.039396132,"data":"{\"probe\": \"ridge_aug\", \"seed\": 29, \"set\": \"sealed_1_new\", \"acc\": 0.9114583333333334}\n"}
|
| 76 |
+
,{"stream_name":"stdout","time":2122.164308907,"data":"{\"probe\": \"ridge_aug\", \"seed\": 29, \"set\": \"sealed_2_new\", \"acc\": 0.7604166666666666}\n"}
|
| 77 |
+
,{"stream_name":"stdout","time":2122.28846083,"data":"{\"probe\": \"ridge_aug\", \"seed\": 29, \"set\": \"sealed_4_new\", \"acc\": 0.421875}\n"}
|
| 78 |
+
,{"stream_name":"stdout","time":2122.539580224,"data":"{\"probe\": \"mlp_aug\", \"seed\": 29, \"set\": \"sealed_ood\", \"acc\": 1.0}\n"}
|
| 79 |
+
,{"stream_name":"stdout","time":2122.66336736,"data":"{\"probe\": \"mlp_aug\", \"seed\": 29, \"set\": \"sealed_1_new\", \"acc\": 1.0}\n"}
|
| 80 |
+
,{"stream_name":"stdout","time":2122.788177567,"data":"{\"probe\": \"mlp_aug\", \"seed\": 29, \"set\": \"sealed_2_new\", \"acc\": 0.984375}\n"}
|
| 81 |
+
,{"stream_name":"stdout","time":2122.913684039,"data":"{\"probe\": \"mlp_aug\", \"seed\": 29, \"set\": \"sealed_4_new\", \"acc\": 0.359375}\n"}
|
| 82 |
+
,{"stream_name":"stdout","time":2123.169120479,"data":"{\"probe\": \"mlp_ctrl_shuffled\", \"seed\": 29, \"set\": \"sealed_ood\", \"acc\": 0.0}\n"}
|
| 83 |
+
,{"stream_name":"stdout","time":2123.29559726,"data":"{\"probe\": \"mlp_ctrl_shuffled\", \"seed\": 29, \"set\": \"sealed_1_new\", \"acc\": 0.0}\n"}
|
| 84 |
+
,{"stream_name":"stdout","time":2123.421828004,"data":"{\"probe\": \"mlp_ctrl_shuffled\", \"seed\": 29, \"set\": \"sealed_2_new\", \"acc\": 0.0}\n"}
|
| 85 |
+
,{"stream_name":"stdout","time":2123.548216944,"data":"{\"probe\": \"mlp_ctrl_shuffled\", \"seed\": 29, \"set\": \"sealed_4_new\", \"acc\": 0.0}\n"}
|
| 86 |
+
,{"stream_name":"stdout","time":2168.125527831,"data":"{\"probe\": \"ridge_aug\", \"seed\": 43, \"set\": \"sealed_ood\", \"acc\": 1.0}\n"}
|
| 87 |
+
,{"stream_name":"stdout","time":2168.262863917,"data":"{\"probe\": \"ridge_aug\", \"seed\": 43, \"set\": \"sealed_1_new\", \"acc\": 0.8802083333333334}\n"}
|
| 88 |
+
,{"stream_name":"stdout","time":2168.391112545,"data":"{\"probe\": \"ridge_aug\", \"seed\": 43, \"set\": \"sealed_2_new\", \"acc\": 0.7083333333333334}\n"}
|
| 89 |
+
,{"stream_name":"stdout","time":2168.517144571,"data":"{\"probe\": \"ridge_aug\", \"seed\": 43, \"set\": \"sealed_4_new\", \"acc\": 0.4791666666666667}\n"}
|
| 90 |
+
,{"stream_name":"stdout","time":2168.76732195,"data":"{\"probe\": \"mlp_aug\", \"seed\": 43, \"set\": \"sealed_ood\", \"acc\": 1.0}\n"}
|
| 91 |
+
,{"stream_name":"stdout","time":2168.89285081,"data":"{\"probe\": \"mlp_aug\", \"seed\": 43, \"set\": \"sealed_1_new\", \"acc\": 1.0}\n"}
|
| 92 |
+
,{"stream_name":"stdout","time":2169.022178723,"data":"{\"probe\": \"mlp_aug\", \"seed\": 43, \"set\": \"sealed_2_new\", \"acc\": 0.9635416666666666}\n"}
|
| 93 |
+
,{"stream_name":"stdout","time":2169.154125255,"data":"{\"probe\": \"mlp_aug\", \"seed\": 43, \"set\": \"sealed_4_new\", \"acc\": 0.3229166666666667}\n"}
|
| 94 |
+
,{"stream_name":"stdout","time":2169.415876585,"data":"{\"probe\": \"mlp_ctrl_shuffled\", \"seed\": 43, \"set\": \"sealed_ood\", \"acc\": 0.0}\n"}
|
| 95 |
+
,{"stream_name":"stdout","time":2169.541812178,"data":"{\"probe\": \"mlp_ctrl_shuffled\", \"seed\": 43, \"set\": \"sealed_1_new\", \"acc\": 0.0}\n"}
|
| 96 |
+
,{"stream_name":"stdout","time":2169.667708314,"data":"{\"probe\": \"mlp_ctrl_shuffled\", \"seed\": 43, \"set\": \"sealed_2_new\", \"acc\": 0.0}\n"}
|
| 97 |
+
,{"stream_name":"stdout","time":2169.796014845,"data":"{\"probe\": \"mlp_ctrl_shuffled\", \"seed\": 43, \"set\": \"sealed_4_new\", \"acc\": 0.0}\n"}
|
| 98 |
+
,{"stream_name":"stdout","time":2183.039922099,"data":"{\"control\": \"random_receiver\", \"set\": \"sealed_ood\", \"text_template\": 0.0, \"k2_021_s17\": 0.0, \"k2_021_s29\": 0.0, \"k2_021_s43\": 0.0}\n"}
|
| 99 |
+
,{"stream_name":"stdout","time":2189.400291153,"data":"{\"control\": \"random_receiver\", \"set\": \"sealed_4_new\", \"text_template\": 0.0, \"k2_021_s17\": 0.0, \"k2_021_s29\": 0.0, \"k2_021_s43\": 0.0}\n"}
|
| 100 |
+
,{"stream_name":"stderr","time":2198.480349048,"data":"/usr/local/lib/python3.13/dist-packages/mistune.py:435: SyntaxWarning: invalid escape sequence '\\|'\n"}
|
| 101 |
+
,{"stream_name":"stderr","time":2198.480380816,"data":" cells[i][c] = re.sub('\\\\\\\\\\|', '|', cell)\n"}
|
| 102 |
+
,{"stream_name":"stderr","time":2198.73720129,"data":"/usr/local/lib/python3.13/dist-packages/nbconvert/filters/filter_links.py:36: SyntaxWarning: invalid escape sequence '\\_'\n"}
|
| 103 |
+
,{"stream_name":"stderr","time":2198.737254349,"data":" text = re.sub(r'_', '\\_', text) # Escape underscores in display text\n"}
|
| 104 |
+
,{"stream_name":"stderr","time":2199.560446684,"data":"[NbConvertApp] Converting notebook __script__.ipynb to html\n"}
|
| 105 |
+
,{"stream_name":"stderr","time":2200.058837332,"data":"[NbConvertApp] Writing 278885 bytes to __results__.html\n"}
|
| 106 |
+
]
|
021/cuerdas021-b.log
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[{"stream_name":"stdout","time":4.361350775,"data":" ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 44.1/44.1 kB 1.4 MB/s eta 0:00:00\n"}
|
| 2 |
+
,{"stream_name":"stdout","time":5.34820775,"data":" ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 10.0/10.0 MB 87.4 MB/s eta 0:00:00\n"}
|
| 3 |
+
,{"stream_name":"stdout","time":5.383756769,"data":" ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 566.4/566.4 kB 24.8 MB/s eta 0:00:00\n"}
|
| 4 |
+
,{"stream_name":"stdout","time":5.438858748,"data":" ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.0/3.0 MB 71.6 MB/s eta 0:00:00\n"}
|
| 5 |
+
,{"stream_name":"stderr","time":18.988138587,"data":"ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n"}
|
| 6 |
+
,{"stream_name":"stderr","time":18.988160772,"data":"litellm 1.85.7 requires tokenizers\u003c1.0,\u003e=0.21.0, but you have tokenizers 0.20.3 which is incompatible.\n"}
|
| 7 |
+
,{"stream_name":"stderr","time":18.988165608,"data":"gradio 6.26.0 requires huggingface-hub\u003c2.0,\u003e=1.16.0, but you have huggingface-hub 0.36.2 which is incompatible.\n"}
|
| 8 |
+
,{"stream_name":"stderr","time":18.988169421,"data":"diffusers 0.40.0 requires huggingface-hub\u003c2.0,\u003e=1.23.0, but you have huggingface-hub 0.36.2 which is incompatible.\n"}
|
| 9 |
+
,{"stream_name":"stdout","time":71.795176862,"data":"{\"positions_per_event\": \"values\", \"sealed_id\": 19.0, \"sealed_ood\": 19.0, \"sealed_1_new\": 19.4, \"sealed_2_new\": 19.95, \"sealed_4_new\": 20.85}\n"}
|
| 10 |
+
,{"stream_name":"stdout","time":71.846933674,"data":"{\"positions_per_event\": \"values_csv\", \"sealed_id\": 22.0, \"sealed_ood\": 22.0, \"sealed_1_new\": 22.4, \"sealed_2_new\": 22.95, \"sealed_4_new\": 23.85}\n"}
|
| 11 |
+
,{"stream_name":"stdout","time":71.929283084,"data":"{\"positions_per_event\": \"kv\", \"sealed_id\": 31.0, \"sealed_ood\": 31.0, \"sealed_1_new\": 31.21, \"sealed_2_new\": 31.41, \"sealed_4_new\": 32.01}\n"}
|
| 12 |
+
,{"stream_name":"stdout","time":72.048858006,"data":"{\"positions_per_event\": \"template\", \"sealed_id\": 35.0, \"sealed_ood\": 35.0, \"sealed_1_new\": 35.21, \"sealed_2_new\": 35.41, \"sealed_4_new\": 36.01}\n"}
|
| 13 |
+
,{"stream_name":"stdout","time":117.831483784,"data":"{\"sys\": \"fewshot_values\", \"shared_prefix_positions\": 99, \"sealed_id\": 0.345, \"sealed_ood\": 0.325, \"sealed_1_new\": 0.03125, \"sealed_2_new\": 0.0, \"sealed_4_new\": 0.0}\n"}
|
| 14 |
+
,{"stream_name":"stdout","time":174.966069479,"data":"{\"sys\": \"fewshot_values_csv\", \"shared_prefix_positions\": 105, \"sealed_id\": 0.89, \"sealed_ood\": 0.9075, \"sealed_1_new\": 0.5, \"sealed_2_new\": 0.24479166666666666, \"sealed_4_new\": 0.046875}\n"}
|
| 15 |
+
,{"stream_name":"stdout","time":245.00421725,"data":"{\"sys\": \"fewshot_kv\", \"shared_prefix_positions\": 123, \"sealed_id\": 0.945, \"sealed_ood\": 0.9175, \"sealed_1_new\": 0.84375, \"sealed_2_new\": 0.8645833333333334, \"sealed_4_new\": 0.8958333333333334}\n"}
|
| 16 |
+
,{"stream_name":"stdout","time":321.724758965,"data":"{\"sys\": \"fewshot_template\", \"shared_prefix_positions\": 131, \"sealed_id\": 0.9, \"sealed_ood\": 0.865, \"sealed_1_new\": 0.6666666666666666, \"sealed_2_new\": 0.5833333333333334, \"sealed_4_new\": 0.3854166666666667}\n"}
|
| 17 |
+
,{"stream_name":"stdout","time":323.808552115,"data":"{\"run\": \"sp_values_s17\", \"step\": 0, \"val\": 0.0}\n"}
|
| 18 |
+
,{"stream_name":"stderr","time":411.215770849,"data":"/usr/local/lib/python3.13/dist-packages/torch/utils/_device.py:116: UserWarning: Converting a tensor with requires_grad=True to a scalar may lead to unexpected behavior.\n"}
|
| 19 |
+
,{"stream_name":"stderr","time":411.215814739,"data":"Consider using tensor.detach() first. (Triggered internally at /pytorch/torch/csrc/autograd/generated/python_variable_methods.cpp:836.)\n"}
|
| 20 |
+
,{"stream_name":"stderr","time":411.215821065,"data":" return func(*args, **kwargs)\n"}
|
| 21 |
+
,{"stream_name":"stdout","time":411.21584086,"data":"{\"run\": \"sp_values_s17\", \"step\": 250, \"val\": 1.6145833333333333, \"loss\": 3.044680595397949, \"s\": 87}\n"}
|
| 22 |
+
,{"stream_name":"stdout","time":497.745659182,"data":"{\"run\": \"sp_values_s17\", \"step\": 500, \"val\": 1.6666666666666667, \"loss\": 3.143272638320923, \"s\": 174}\n"}
|
| 23 |
+
,{"stream_name":"stdout","time":584.606442804,"data":"{\"run\": \"sp_values_s17\", \"step\": 750, \"val\": 1.6770833333333335, \"loss\": 2.6209170818328857, \"s\": 261}\n"}
|
| 24 |
+
,{"stream_name":"stdout","time":671.475163664,"data":"{\"run\": \"sp_values_s17\", \"step\": 1000, \"val\": 1.6041666666666665, \"loss\": 3.025043487548828, \"s\": 348}\n"}
|
| 25 |
+
,{"stream_name":"stdout","time":758.830169521,"data":"{\"run\": \"sp_values_s17\", \"step\": 1250, \"val\": 1.7083333333333335, \"loss\": 2.137320041656494, \"s\": 435}\n"}
|
| 26 |
+
,{"stream_name":"stdout","time":846.891577041,"data":"{\"run\": \"sp_values_s17\", \"step\": 1500, \"val\": 1.71875, \"loss\": 1.954566240310669, \"s\": 523}\n"}
|
| 27 |
+
,{"stream_name":"stdout","time":934.268076228,"data":"{\"run\": \"sp_values_s17\", \"step\": 1750, \"val\": 1.6354166666666665, \"loss\": 3.326815128326416, \"s\": 610}\n"}
|
| 28 |
+
,{"stream_name":"stdout","time":1022.820649516,"data":"{\"run\": \"sp_values_s17\", \"step\": 2000, \"val\": 1.7291666666666665, \"loss\": 2.481618642807007, \"s\": 699}\n"}
|
| 29 |
+
,{"stream_name":"stdout","time":1110.19483707,"data":"{\"run\": \"sp_values_s17\", \"step\": 2250, \"val\": 1.7083333333333335, \"loss\": 2.0035409927368164, \"s\": 786}\n"}
|
| 30 |
+
,{"stream_name":"stdout","time":1196.253999266,"data":"{\"run\": \"sp_values_s17\", \"step\": 2500, \"val\": 1.7395833333333335, \"loss\": 1.278429627418518, \"s\": 872}\n"}
|
| 31 |
+
,{"stream_name":"stdout","time":1283.403222855,"data":"{\"run\": \"sp_values_s17\", \"step\": 2750, \"val\": 1.7708333333333335, \"loss\": 1.8047574758529663, \"s\": 960}\n"}
|
| 32 |
+
,{"stream_name":"stdout","time":1370.785137672,"data":"{\"run\": \"sp_values_s17\", \"step\": 3000, \"val\": 1.7395833333333335, \"loss\": 1.2187349796295166, \"s\": 1047}\n"}
|
| 33 |
+
,{"stream_name":"stdout","time":1382.603182089,"data":"{\"sys\": \"sp_values_s17\", \"params\": 3842368, \"selected_step\": 2750, \"val\": 1.7708333333333335, \"soft_prompt_positions\": 8, \"sealed_id\": 1.0, \"sealed_ood\": 1.0, \"sealed_1_new\": 1.0, \"sealed_2_new\": 0.984375, \"sealed_4_new\": 0.96875}\n"}
|
| 34 |
+
,{"stream_name":"stdout","time":1384.615533132,"data":"{\"run\": \"sp_values_s29\", \"step\": 0, \"val\": 0.0}\n"}
|
| 35 |
+
,{"stream_name":"stdout","time":1471.512106137,"data":"{\"run\": \"sp_values_s29\", \"step\": 250, \"val\": 1.375, \"loss\": 4.450610160827637, \"s\": 87}\n"}
|
| 36 |
+
,{"stream_name":"stdout","time":1559.097106882,"data":"{\"run\": \"sp_values_s29\", \"step\": 500, \"val\": 1.6145833333333335, \"loss\": 2.346332550048828, \"s\": 174}\n"}
|
| 37 |
+
,{"stream_name":"stdout","time":1645.830903071,"data":"{\"run\": \"sp_values_s29\", \"step\": 750, \"val\": 1.6145833333333335, \"loss\": 2.190868854522705, \"s\": 261}\n"}
|
| 38 |
+
,{"stream_name":"stdout","time":1732.973225386,"data":"{\"run\": \"sp_values_s29\", \"step\": 1000, \"val\": 1.6666666666666665, \"loss\": 1.3257291316986084, \"s\": 348}\n"}
|
| 39 |
+
,{"stream_name":"stdout","time":1820.431503621,"data":"{\"run\": \"sp_values_s29\", \"step\": 1250, \"val\": 1.6458333333333335, \"loss\": 2.51035737991333, \"s\": 436}\n"}
|
| 40 |
+
,{"stream_name":"stdout","time":1907.325803849,"data":"{\"run\": \"sp_values_s29\", \"step\": 1500, \"val\": 1.6875, \"loss\": 2.5555827617645264, \"s\": 523}\n"}
|
| 41 |
+
,{"stream_name":"stdout","time":1993.856924421,"data":"{\"run\": \"sp_values_s29\", \"step\": 1750, \"val\": 1.6041666666666665, \"loss\": 1.5838260650634766, \"s\": 609}\n"}
|
| 42 |
+
,{"stream_name":"stdout","time":2081.193454877,"data":"{\"run\": \"sp_values_s29\", \"step\": 2000, \"val\": 1.6770833333333335, \"loss\": 2.4831151962280273, \"s\": 697}\n"}
|
| 43 |
+
,{"stream_name":"stdout","time":2168.354451021,"data":"{\"run\": \"sp_values_s29\", \"step\": 2250, \"val\": 1.7083333333333335, \"loss\": 1.9796416759490967, \"s\": 784}\n"}
|
| 44 |
+
,{"stream_name":"stdout","time":2255.405604058,"data":"{\"run\": \"sp_values_s29\", \"step\": 2500, \"val\": 1.6666666666666665, \"loss\": 1.4970152378082275, \"s\": 871}\n"}
|
| 45 |
+
,{"stream_name":"stdout","time":2342.536059791,"data":"{\"run\": \"sp_values_s29\", \"step\": 2750, \"val\": 1.6875, \"loss\": 2.13800311088562, \"s\": 958}\n"}
|
| 46 |
+
,{"stream_name":"stdout","time":2428.773263448,"data":"{\"run\": \"sp_values_s29\", \"step\": 3000, \"val\": 1.6875, \"loss\": 2.1700429916381836, \"s\": 1044}\n"}
|
| 47 |
+
,{"stream_name":"stdout","time":2440.483019732,"data":"{\"sys\": \"sp_values_s29\", \"params\": 3842368, \"selected_step\": 2250, \"val\": 1.7083333333333335, \"soft_prompt_positions\": 8, \"sealed_id\": 1.0, \"sealed_ood\": 1.0, \"sealed_1_new\": 1.0, \"sealed_2_new\": 0.9895833333333334, \"sealed_4_new\": 0.9635416666666666}\n"}
|
| 48 |
+
,{"stream_name":"stdout","time":2442.496346525,"data":"{\"run\": \"sp_values_s43\", \"step\": 0, \"val\": 0.0}\n"}
|
| 49 |
+
,{"stream_name":"stdout","time":2529.924771935,"data":"{\"run\": \"sp_values_s43\", \"step\": 250, \"val\": 1.4583333333333333, \"loss\": 2.9020471572875977, \"s\": 87}\n"}
|
| 50 |
+
,{"stream_name":"stdout","time":2615.888976615,"data":"{\"run\": \"sp_values_s43\", \"step\": 500, \"val\": 1.46875, \"loss\": 2.5136404037475586, \"s\": 173}\n"}
|
| 51 |
+
,{"stream_name":"stdout","time":2704.372018282,"data":"{\"run\": \"sp_values_s43\", \"step\": 750, \"val\": 1.625, \"loss\": 2.120654821395874, \"s\": 262}\n"}
|
| 52 |
+
,{"stream_name":"stdout","time":2791.388173071,"data":"{\"run\": \"sp_values_s43\", \"step\": 1000, \"val\": 1.6770833333333335, \"loss\": 3.3165745735168457, \"s\": 349}\n"}
|
| 53 |
+
,{"stream_name":"stdout","time":2879.397411429,"data":"{\"run\": \"sp_values_s43\", \"step\": 1250, \"val\": 1.78125, \"loss\": 2.017547607421875, \"s\": 437}\n"}
|
| 54 |
+
,{"stream_name":"stdout","time":2966.957135373,"data":"{\"run\": \"sp_values_s43\", \"step\": 1500, \"val\": 1.78125, \"loss\": 2.347735643386841, \"s\": 524}\n"}
|
| 55 |
+
,{"stream_name":"stdout","time":3054.617083351,"data":"{\"run\": \"sp_values_s43\", \"step\": 1750, \"val\": 1.7708333333333335, \"loss\": 2.394887924194336, \"s\": 612}\n"}
|
| 56 |
+
,{"stream_name":"stdout","time":3141.533202514,"data":"{\"run\": \"sp_values_s43\", \"step\": 2000, \"val\": 1.625, \"loss\": 1.9778445959091187, \"s\": 699}\n"}
|
| 57 |
+
,{"stream_name":"stdout","time":3227.879030741,"data":"{\"run\": \"sp_values_s43\", \"step\": 2250, \"val\": 1.78125, \"loss\": 2.360875129699707, \"s\": 785}\n"}
|
| 58 |
+
,{"stream_name":"stdout","time":3314.498209102,"data":"{\"run\": \"sp_values_s43\", \"step\": 2500, \"val\": 1.7916666666666665, \"loss\": 1.7670954465866089, \"s\": 872}\n"}
|
| 59 |
+
,{"stream_name":"stdout","time":3401.342048246,"data":"{\"run\": \"sp_values_s43\", \"step\": 2750, \"val\": 1.75, \"loss\": 2.02799654006958, \"s\": 959}\n"}
|
| 60 |
+
,{"stream_name":"stdout","time":3488.451204805,"data":"{\"run\": \"sp_values_s43\", \"step\": 3000, \"val\": 1.7395833333333335, \"loss\": 1.7585619688034058, \"s\": 1046}\n"}
|
| 61 |
+
,{"stream_name":"stdout","time":3500.244749137,"data":"{\"sys\": \"sp_values_s43\", \"params\": 3842368, \"selected_step\": 2500, \"val\": 1.7916666666666665, \"soft_prompt_positions\": 8, \"sealed_id\": 1.0, \"sealed_ood\": 1.0, \"sealed_1_new\": 1.0, \"sealed_2_new\": 0.984375, \"sealed_4_new\": 0.9322916666666666}\n"}
|
| 62 |
+
,{"stream_name":"stdout","time":3502.496340876,"data":"{\"run\": \"sp_values_csv_s17\", \"step\": 0, \"val\": 0.0}\n"}
|
| 63 |
+
,{"stream_name":"stdout","time":3597.190372836,"data":"{\"run\": \"sp_values_csv_s17\", \"step\": 250, \"val\": 1.90625, \"loss\": 2.3863179683685303, \"s\": 95}\n"}
|
| 64 |
+
,{"stream_name":"stdout","time":3691.237500232,"data":"{\"run\": \"sp_values_csv_s17\", \"step\": 500, \"val\": 1.9583333333333335, \"loss\": 2.344421863555908, \"s\": 189}\n"}
|
| 65 |
+
,{"stream_name":"stdout","time":3785.572447256,"data":"{\"run\": \"sp_values_csv_s17\", \"step\": 750, \"val\": 1.9791666666666665, \"loss\": 1.9774900674819946, \"s\": 283}\n"}
|
| 66 |
+
,{"stream_name":"stdout","time":3879.655793829,"data":"{\"run\": \"sp_values_csv_s17\", \"step\": 1000, \"val\": 1.9583333333333335, \"loss\": 1.8494818210601807, \"s\": 377}\n"}
|
| 67 |
+
,{"stream_name":"stdout","time":3974.548123947,"data":"{\"run\": \"sp_values_csv_s17\", \"step\": 1250, \"val\": 1.9479166666666665, \"loss\": 1.641819715499878, \"s\": 472}\n"}
|
| 68 |
+
,{"stream_name":"stdout","time":4069.833349564,"data":"{\"run\": \"sp_values_csv_s17\", \"step\": 1500, \"val\": 1.9895833333333335, \"loss\": 1.3760128021240234, \"s\": 567}\n"}
|
| 69 |
+
,{"stream_name":"stdout","time":4164.647401236,"data":"{\"run\": \"sp_values_csv_s17\", \"step\": 1750, \"val\": 1.9895833333333335, \"loss\": 1.9606835842132568, \"s\": 662}\n"}
|
| 70 |
+
,{"stream_name":"stdout","time":4260.292046457,"data":"{\"run\": \"sp_values_csv_s17\", \"step\": 2000, \"val\": 2.0, \"loss\": 1.9302983283996582, \"s\": 758}\n"}
|
| 71 |
+
,{"stream_name":"stdout","time":4354.666832377,"data":"{\"run\": \"sp_values_csv_s17\", \"step\": 2250, \"val\": 1.9895833333333335, \"loss\": 1.5998188257217407, \"s\": 852}\n"}
|
| 72 |
+
,{"stream_name":"stdout","time":4448.229734015,"data":"{\"run\": \"sp_values_csv_s17\", \"step\": 2500, \"val\": 1.9895833333333335, \"loss\": 1.07892906665802, \"s\": 946}\n"}
|
| 73 |
+
,{"stream_name":"stdout","time":4542.45081406,"data":"{\"run\": \"sp_values_csv_s17\", \"step\": 2750, \"val\": 2.0, \"loss\": 1.3307654857635498, \"s\": 1040}\n"}
|
| 74 |
+
,{"stream_name":"stdout","time":4636.918178215,"data":"{\"run\": \"sp_values_csv_s17\", \"step\": 3000, \"val\": 2.0, \"loss\": 0.9787193536758423, \"s\": 1134}\n"}
|
| 75 |
+
,{"stream_name":"stdout","time":4650.271880555,"data":"{\"sys\": \"sp_values_csv_s17\", \"params\": 3842368, \"selected_step\": 2000, \"val\": 2.0, \"soft_prompt_positions\": 8, \"sealed_id\": 1.0, \"sealed_ood\": 1.0, \"sealed_1_new\": 1.0, \"sealed_2_new\": 1.0, \"sealed_4_new\": 1.0}\n"}
|
| 76 |
+
,{"stream_name":"stdout","time":4652.516545442,"data":"{\"run\": \"sp_values_csv_s29\", \"step\": 0, \"val\": 0.0}\n"}
|
| 77 |
+
,{"stream_name":"stdout","time":4746.943424249,"data":"{\"run\": \"sp_values_csv_s29\", \"step\": 250, \"val\": 1.9583333333333335, \"loss\": 2.902418851852417, \"s\": 94}\n"}
|
| 78 |
+
,{"stream_name":"stdout","time":4842.027314419,"data":"{\"run\": \"sp_values_csv_s29\", \"step\": 500, \"val\": 1.9375, \"loss\": 1.829451084136963, \"s\": 190}\n"}
|
| 79 |
+
,{"stream_name":"stdout","time":4935.870307238,"data":"{\"run\": \"sp_values_csv_s29\", \"step\": 750, \"val\": 1.9375, \"loss\": 1.5734684467315674, \"s\": 283}\n"}
|
| 80 |
+
,{"stream_name":"stdout","time":5030.65106833,"data":"{\"run\": \"sp_values_csv_s29\", \"step\": 1000, \"val\": 1.96875, \"loss\": 0.8627220988273621, \"s\": 378}\n"}
|
| 81 |
+
,{"stream_name":"stdout","time":5125.899735903,"data":"{\"run\": \"sp_values_csv_s29\", \"step\": 1250, \"val\": 1.9791666666666665, \"loss\": 1.8343243598937988, \"s\": 473}\n"}
|
| 82 |
+
,{"stream_name":"stdout","time":5219.968519164,"data":"{\"run\": \"sp_values_csv_s29\", \"step\": 1500, \"val\": 1.9895833333333335, \"loss\": 1.6268024444580078, \"s\": 567}\n"}
|
| 83 |
+
,{"stream_name":"stdout","time":5313.952497176,"data":"{\"run\": \"sp_values_csv_s29\", \"step\": 1750, \"val\": 1.96875, \"loss\": 1.224697232246399, \"s\": 661}\n"}
|
| 84 |
+
,{"stream_name":"stdout","time":5408.727257067,"data":"{\"run\": \"sp_values_csv_s29\", \"step\": 2000, \"val\": 1.9583333333333335, \"loss\": 1.8627893924713135, \"s\": 756}\n"}
|
| 85 |
+
,{"stream_name":"stdout","time":5503.152377398,"data":"{\"run\": \"sp_values_csv_s29\", \"step\": 2250, \"val\": 1.9583333333333335, \"loss\": 1.6173484325408936, \"s\": 851}\n"}
|
| 86 |
+
,{"stream_name":"stdout","time":5597.532104906,"data":"{\"run\": \"sp_values_csv_s29\", \"step\": 2500, \"val\": 1.9791666666666665, \"loss\": 1.2948013544082642, \"s\": 945}\n"}
|
| 87 |
+
,{"stream_name":"stdout","time":5692.405593825,"data":"{\"run\": \"sp_values_csv_s29\", \"step\": 2750, \"val\": 1.96875, \"loss\": 1.518986463546753, \"s\": 1040}\n"}
|
| 88 |
+
,{"stream_name":"stdout","time":5786.105113037,"data":"{\"run\": \"sp_values_csv_s29\", \"step\": 3000, \"val\": 1.96875, \"loss\": 1.6026052236557007, \"s\": 1134}\n"}
|
| 89 |
+
,{"stream_name":"stdout","time":5799.436388588,"data":"{\"sys\": \"sp_values_csv_s29\", \"params\": 3842368, \"selected_step\": 1500, \"val\": 1.9895833333333335, \"soft_prompt_positions\": 8, \"sealed_id\": 1.0, \"sealed_ood\": 1.0, \"sealed_1_new\": 1.0, \"sealed_2_new\": 1.0, \"sealed_4_new\": 1.0}\n"}
|
| 90 |
+
,{"stream_name":"stdout","time":5801.656527668,"data":"{\"run\": \"sp_values_csv_s43\", \"step\": 0, \"val\": 0.0}\n"}
|
| 91 |
+
,{"stream_name":"stdout","time":5896.668236069,"data":"{\"run\": \"sp_values_csv_s43\", \"step\": 250, \"val\": 1.8854166666666665, \"loss\": 2.186886787414551, \"s\": 95}\n"}
|
| 92 |
+
,{"stream_name":"stdout","time":5990.220831139,"data":"{\"run\": \"sp_values_csv_s43\", \"step\": 500, \"val\": 1.96875, \"loss\": 1.741999864578247, \"s\": 189}\n"}
|
| 93 |
+
,{"stream_name":"stdout","time":6086.120661244,"data":"{\"run\": \"sp_values_csv_s43\", \"step\": 750, \"val\": 1.8645833333333335, \"loss\": 1.5374534130096436, \"s\": 284}\n"}
|
| 94 |
+
,{"stream_name":"stdout","time":6181.057805559,"data":"{\"run\": \"sp_values_csv_s43\", \"step\": 1000, \"val\": 1.9583333333333335, \"loss\": 2.0583114624023438, \"s\": 379}\n"}
|
| 95 |
+
,{"stream_name":"stdout","time":6276.707419186,"data":"{\"run\": \"sp_values_csv_s43\", \"step\": 1250, \"val\": 1.9166666666666665, \"loss\": 1.2399407625198364, \"s\": 475}\n"}
|
| 96 |
+
,{"stream_name":"stdout","time":6371.443170589,"data":"{\"run\": \"sp_values_csv_s43\", \"step\": 1500, \"val\": 1.96875, \"loss\": 1.6992583274841309, \"s\": 570}\n"}
|
| 97 |
+
,{"stream_name":"stdout","time":6466.554435181,"data":"{\"run\": \"sp_values_csv_s43\", \"step\": 1750, \"val\": 1.9895833333333335, \"loss\": 1.7322700023651123, \"s\": 665}\n"}
|
| 98 |
+
,{"stream_name":"stdout","time":6561.062734654,"data":"{\"run\": \"sp_values_csv_s43\", \"step\": 2000, \"val\": 1.9583333333333335, \"loss\": 1.5983898639678955, \"s\": 759}\n"}
|
| 99 |
+
,{"stream_name":"stdout","time":6654.941557139,"data":"{\"run\": \"sp_values_csv_s43\", \"step\": 2250, \"val\": 1.96875, \"loss\": 1.6161710023880005, \"s\": 853}\n"}
|
| 100 |
+
,{"stream_name":"stdout","time":6749.082979317,"data":"{\"run\": \"sp_values_csv_s43\", \"step\": 2500, \"val\": 1.96875, \"loss\": 1.3451036214828491, \"s\": 947}\n"}
|
| 101 |
+
,{"stream_name":"stdout","time":6843.554073075,"data":"{\"run\": \"sp_values_csv_s43\", \"step\": 2750, \"val\": 1.9791666666666665, \"loss\": 1.2667261362075806, \"s\": 1042}\n"}
|
| 102 |
+
,{"stream_name":"stdout","time":6938.307644552,"data":"{\"run\": \"sp_values_csv_s43\", \"step\": 3000, \"val\": 1.96875, \"loss\": 1.4136821031570435, \"s\": 1137}\n"}
|
| 103 |
+
,{"stream_name":"stdout","time":6951.710262143,"data":"{\"sys\": \"sp_values_csv_s43\", \"params\": 3842368, \"selected_step\": 1750, \"val\": 1.9895833333333335, \"soft_prompt_positions\": 8, \"sealed_id\": 1.0, \"sealed_ood\": 1.0, \"sealed_1_new\": 1.0, \"sealed_2_new\": 1.0, \"sealed_4_new\": 1.0}\n"}
|
| 104 |
+
,{"stream_name":"stdout","time":6954.609131963,"data":"{\"run\": \"sp_kv_s17\", \"step\": 0, \"val\": 0.0625}\n"}
|
| 105 |
+
,{"stream_name":"stdout","time":7069.711213562,"data":"{\"run\": \"sp_kv_s17\", \"step\": 250, \"val\": 2.0, \"loss\": 1.9125851392745972, \"s\": 115}\n"}
|
| 106 |
+
,{"stream_name":"stdout","time":7185.042959089,"data":"{\"run\": \"sp_kv_s17\", \"step\": 500, \"val\": 2.0, \"loss\": 2.0021212100982666, \"s\": 230}\n"}
|
| 107 |
+
,{"stream_name":"stdout","time":7300.369875705,"data":"{\"run\": \"sp_kv_s17\", \"step\": 750, \"val\": 2.0, \"loss\": 1.5451703071594238, \"s\": 346}\n"}
|
| 108 |
+
,{"stream_name":"stdout","time":7415.495030592,"data":"{\"run\": \"sp_kv_s17\", \"step\": 1000, \"val\": 2.0, \"loss\": 1.4742144346237183, \"s\": 461}\n"}
|
| 109 |
+
,{"stream_name":"stdout","time":7531.547118465,"data":"{\"run\": \"sp_kv_s17\", \"step\": 1250, \"val\": 2.0, \"loss\": 1.4599279165267944, \"s\": 577}\n"}
|
| 110 |
+
,{"stream_name":"stdout","time":7647.554036009,"data":"{\"run\": \"sp_kv_s17\", \"step\": 1500, \"val\": 2.0, \"loss\": 1.2338306903839111, \"s\": 693}\n"}
|
| 111 |
+
,{"stream_name":"stdout","time":7763.436058644,"data":"{\"run\": \"sp_kv_s17\", \"step\": 1750, \"val\": 2.0, \"loss\": 1.8669570684432983, \"s\": 809}\n"}
|
| 112 |
+
,{"stream_name":"stdout","time":7880.156176707,"data":"{\"run\": \"sp_kv_s17\", \"step\": 2000, \"val\": 2.0, \"loss\": 1.7696266174316406, \"s\": 926}\n"}
|
| 113 |
+
,{"stream_name":"stdout","time":7995.855988061,"data":"{\"run\": \"sp_kv_s17\", \"step\": 2250, \"val\": 2.0, \"loss\": 1.350313425064087, \"s\": 1041}\n"}
|
| 114 |
+
,{"stream_name":"stdout","time":8110.64570648,"data":"{\"run\": \"sp_kv_s17\", \"step\": 2500, \"val\": 2.0, \"loss\": 0.9131432175636292, \"s\": 1156}\n"}
|
| 115 |
+
,{"stream_name":"stdout","time":8225.736727802,"data":"{\"run\": \"sp_kv_s17\", \"step\": 2750, \"val\": 2.0, \"loss\": 1.2375553846359253, \"s\": 1271}\n"}
|
| 116 |
+
,{"stream_name":"stdout","time":8341.095640136,"data":"{\"run\": \"sp_kv_s17\", \"step\": 3000, \"val\": 2.0, \"loss\": 0.8844823837280273, \"s\": 1386}\n"}
|
| 117 |
+
,{"stream_name":"stdout","time":8358.2884852,"data":"{\"sys\": \"sp_kv_s17\", \"params\": 3842368, \"selected_step\": 250, \"val\": 2.0, \"soft_prompt_positions\": 8, \"sealed_id\": 1.0, \"sealed_ood\": 1.0, \"sealed_1_new\": 1.0, \"sealed_2_new\": 1.0, \"sealed_4_new\": 1.0}\n"}
|
| 118 |
+
,{"stream_name":"stdout","time":8361.209239923,"data":"{\"run\": \"sp_kv_s29\", \"step\": 0, \"val\": 0.07291666666666667}\n"}
|
| 119 |
+
,{"stream_name":"stdout","time":8476.470163654,"data":"{\"run\": \"sp_kv_s29\", \"step\": 250, \"val\": 1.9895833333333335, \"loss\": 2.1503524780273438, \"s\": 115}\n"}
|
| 120 |
+
,{"stream_name":"stdout","time":8592.654786484,"data":"{\"run\": \"sp_kv_s29\", \"step\": 500, \"val\": 2.0, \"loss\": 1.5459184646606445, \"s\": 231}\n"}
|
| 121 |
+
,{"stream_name":"stdout","time":8707.826241838,"data":"{\"run\": \"sp_kv_s29\", \"step\": 750, \"val\": 1.9895833333333335, \"loss\": 1.2727928161621094, \"s\": 347}\n"}
|
| 122 |
+
,{"stream_name":"stdout","time":8823.59979388,"data":"{\"run\": \"sp_kv_s29\", \"step\": 1000, \"val\": 2.0, \"loss\": 0.745368242263794, \"s\": 462}\n"}
|
| 123 |
+
,{"stream_name":"stdout","time":8939.752280122,"data":"{\"run\": \"sp_kv_s29\", \"step\": 1250, \"val\": 2.0, \"loss\": 1.6049551963806152, \"s\": 579}\n"}
|
| 124 |
+
,{"stream_name":"stdout","time":9054.866158633,"data":"{\"run\": \"sp_kv_s29\", \"step\": 1500, \"val\": 2.0, \"loss\": 1.5594961643218994, \"s\": 694}\n"}
|
| 125 |
+
,{"stream_name":"stdout","time":9169.872659237,"data":"{\"run\": \"sp_kv_s29\", \"step\": 1750, \"val\": 2.0, \"loss\": 1.1384079456329346, \"s\": 809}\n"}
|
| 126 |
+
,{"stream_name":"stdout","time":9285.538980744,"data":"{\"run\": \"sp_kv_s29\", \"step\": 2000, \"val\": 1.96875, \"loss\": 1.4778153896331787, \"s\": 924}\n"}
|
| 127 |
+
,{"stream_name":"stdout","time":9401.162170161,"data":"{\"run\": \"sp_kv_s29\", \"step\": 2250, \"val\": 2.0, \"loss\": 1.5377285480499268, \"s\": 1040}\n"}
|
| 128 |
+
,{"stream_name":"stdout","time":9516.750259155,"data":"{\"run\": \"sp_kv_s29\", \"step\": 2500, \"val\": 2.0, \"loss\": 1.1550800800323486, \"s\": 1156}\n"}
|
| 129 |
+
,{"stream_name":"stdout","time":9632.356820272,"data":"{\"run\": \"sp_kv_s29\", \"step\": 2750, \"val\": 2.0, \"loss\": 1.3703510761260986, \"s\": 1271}\n"}
|
| 130 |
+
,{"stream_name":"stdout","time":9747.15441662,"data":"{\"run\": \"sp_kv_s29\", \"step\": 3000, \"val\": 2.0, \"loss\": 1.3216395378112793, \"s\": 1386}\n"}
|
| 131 |
+
,{"stream_name":"stdout","time":9764.37948733,"data":"{\"sys\": \"sp_kv_s29\", \"params\": 3842368, \"selected_step\": 500, \"val\": 2.0, \"soft_prompt_positions\": 8, \"sealed_id\": 1.0, \"sealed_ood\": 1.0, \"sealed_1_new\": 1.0, \"sealed_2_new\": 1.0, \"sealed_4_new\": 1.0}\n"}
|
| 132 |
+
,{"stream_name":"stdout","time":9767.312728899,"data":"{\"run\": \"sp_kv_s43\", \"step\": 0, \"val\": 0.07291666666666667}\n"}
|
| 133 |
+
,{"stream_name":"stdout","time":9883.208554309,"data":"{\"run\": \"sp_kv_s43\", \"step\": 250, \"val\": 2.0, \"loss\": 1.6502081155776978, \"s\": 116}\n"}
|
| 134 |
+
,{"stream_name":"stdout","time":9997.853492914,"data":"{\"run\": \"sp_kv_s43\", \"step\": 500, \"val\": 2.0, \"loss\": 1.304010272026062, \"s\": 231}\n"}
|
| 135 |
+
,{"stream_name":"stdout","time":10114.418303539,"data":"{\"run\": \"sp_kv_s43\", \"step\": 750, \"val\": 2.0, \"loss\": 1.260686993598938, \"s\": 347}\n"}
|
| 136 |
+
,{"stream_name":"stdout","time":10230.311754109,"data":"{\"run\": \"sp_kv_s43\", \"step\": 1000, \"val\": 2.0, \"loss\": 1.5032280683517456, \"s\": 463}\n"}
|
| 137 |
+
,{"stream_name":"stdout","time":10346.890779959,"data":"{\"run\": \"sp_kv_s43\", \"step\": 1250, \"val\": 2.0, \"loss\": 1.139223337173462, \"s\": 580}\n"}
|
| 138 |
+
,{"stream_name":"stdout","time":10462.6621338,"data":"{\"run\": \"sp_kv_s43\", \"step\": 1500, \"val\": 2.0, \"loss\": 1.4122791290283203, \"s\": 695}\n"}
|
| 139 |
+
,{"stream_name":"stdout","time":10578.37733939,"data":"{\"run\": \"sp_kv_s43\", \"step\": 1750, \"val\": 2.0, \"loss\": 1.3497402667999268, \"s\": 811}\n"}
|
| 140 |
+
,{"stream_name":"stdout","time":10693.879305726,"data":"{\"run\": \"sp_kv_s43\", \"step\": 2000, \"val\": 2.0, \"loss\": 1.427781581878662, \"s\": 927}\n"}
|
| 141 |
+
,{"stream_name":"stdout","time":10808.648862386,"data":"{\"run\": \"sp_kv_s43\", \"step\": 2250, \"val\": 2.0, \"loss\": 1.3728115558624268, \"s\": 1041}\n"}
|
| 142 |
+
,{"stream_name":"stdout","time":10924.288212135,"data":"{\"run\": \"sp_kv_s43\", \"step\": 2500, \"val\": 2.0, \"loss\": 1.1903347969055176, \"s\": 1157}\n"}
|
| 143 |
+
,{"stream_name":"stdout","time":11039.960710764,"data":"{\"run\": \"sp_kv_s43\", \"step\": 2750, \"val\": 2.0, \"loss\": 1.029153823852539, \"s\": 1273}\n"}
|
| 144 |
+
,{"stream_name":"stdout","time":11155.485832086,"data":"{\"run\": \"sp_kv_s43\", \"step\": 3000, \"val\": 2.0, \"loss\": 1.0028929710388184, \"s\": 1388}\n"}
|
| 145 |
+
,{"stream_name":"stdout","time":11172.736693556,"data":"{\"sys\": \"sp_kv_s43\", \"params\": 3842368, \"selected_step\": 250, \"val\": 2.0, \"soft_prompt_positions\": 8, \"sealed_id\": 1.0, \"sealed_ood\": 1.0, \"sealed_1_new\": 1.0, \"sealed_2_new\": 1.0, \"sealed_4_new\": 1.0}\n"}
|
| 146 |
+
,{"stream_name":"stderr","time":11181.503505782,"data":"/usr/local/lib/python3.13/dist-packages/mistune.py:435: SyntaxWarning: invalid escape sequence '\\|'\n"}
|
| 147 |
+
,{"stream_name":"stderr","time":11181.503534751,"data":" cells[i][c] = re.sub('\\\\\\\\\\|', '|', cell)\n"}
|
| 148 |
+
,{"stream_name":"stderr","time":11181.765160135,"data":"/usr/local/lib/python3.13/dist-packages/nbconvert/filters/filter_links.py:36: SyntaxWarning: invalid escape sequence '\\_'\n"}
|
| 149 |
+
,{"stream_name":"stderr","time":11181.765180439,"data":" text = re.sub(r'_', '\\_', text) # Escape underscores in display text\n"}
|
| 150 |
+
,{"stream_name":"stderr","time":11182.577499966,"data":"[NbConvertApp] Converting notebook __script__.ipynb to html\n"}
|
| 151 |
+
,{"stream_name":"stderr","time":11183.090133157,"data":"[NbConvertApp] Writing 278885 bytes to __results__.html\n"}
|
| 152 |
+
]
|
021/evalA.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CUERDAS-021 A: K=2 retrained with varied pool; sealed eval once; probes; controls.
|
| 2 |
+
from c021 import *
|
| 3 |
+
from transformers import AutoModelForCausalLM, AutoConfig
|
| 4 |
+
log = opener('resultsA.jsonl'); PRED = open(f'{OD}/predictionsA.jsonl', 'w'); SEEDS = [17] if SMOKE else [17, 29, 43]
|
| 5 |
+
log({'sealed_sha256': H, 'torch': torch.__version__, 'pool_sizes': {'words': len(WORDS), 'names': len(NAMES), 'subs': len(SUBS)}})
|
| 6 |
+
def ev(sysname, rowfn):
|
| 7 |
+
res = {}
|
| 8 |
+
for s in SETS:
|
| 9 |
+
r = per_ex(rowfn, S[s], CAND); res[s] = r
|
| 10 |
+
for i, c in enumerate(r): PRED.write(json.dumps({'sys': sysname, 'set': s, 'i': i, 'correct': bool(c)}) + '\n')
|
| 11 |
+
PRED.flush(); log({'sys': sysname, **{s: sum(v) / len(v) for s, v in res.items()}}); return res
|
| 12 |
+
ev('text_template', lambda mb: text_rows(mb, R.SUF_MAIN, R.TPL))
|
| 13 |
+
for sd in SEEDS:
|
| 14 |
+
c = R.PackK(2, sd); c.load_state_dict(torch.load(f'{INP}/k2long_s{sd}.pt', map_location=R.DEV)); c.eval()
|
| 15 |
+
ev(f'k2_018_s{sd}', lambda mb, c=c: k2_rows(c, mb, R.SUF_MAIN))
|
| 16 |
+
cods = {}
|
| 17 |
+
for sd in SEEDS:
|
| 18 |
+
teach = load_teacher(sd); k2 = R.PackK(2, sd); k2.load_state_dict(torch.load(f'{INP}/k2_s{sd}.pt', map_location=R.DEV))
|
| 19 |
+
step, v = train(list(k2.parameters()), lambda mb, tpl: k2_rows(k2, mb, tpl), teach, sd, f'k2_021_s{sd}', log, set_mode=lambda t: k2.train(t))
|
| 20 |
+
k2.eval().requires_grad_(False); torch.save(k2.state_dict(), f'{OD}/k2v_s{sd}.pt'); cods[sd] = k2
|
| 21 |
+
log({'selected': f'k2_021_s{sd}', 'step': step, 'val': v})
|
| 22 |
+
ev(f'k2_021_s{sd}', lambda mb, k2=k2: k2_rows(k2, mb, R.SUF_MAIN))
|
| 23 |
+
# ---- probes on retrained slots ----
|
| 24 |
+
VALS = [V[j] + NEW[j] for j in range(4)]; CEV = [F.normalize(torch.stack([R.lex(w) for w in VALS[j]]), dim=-1) for j in range(4)]
|
| 25 |
+
g = random.Random(2103); AUG = sample_msgs(g, 300 if SMOKE else 20000)
|
| 26 |
+
with torch.no_grad(): YA = torch.stack([torch.stack([R.lex(w) for w in m]) for m in AUG])
|
| 27 |
+
def slots(cod, ms):
|
| 28 |
+
with torch.no_grad(): return torch.cat([cod(torch.stack([torch.stack([R.lex(w) for w in m]) for m in ms[i:i+512]])).flatten(1) for i in range(0, len(ms), 512)])
|
| 29 |
+
def dec(P, ms):
|
| 30 |
+
ok = torch.ones(len(ms), dtype=torch.bool, device='cpu')
|
| 31 |
+
for j in range(4): ok &= (F.normalize(P[:, j], dim=-1) @ CEV[j].T).argmax(1).cpu() == torch.tensor([VALS[j].index(m[j]) for m in ms], device='cpu')
|
| 32 |
+
return ok.float().tolist()
|
| 33 |
+
def train_mlp(X, Y, shuffle=False, steps=None, seed=0):
|
| 34 |
+
steps = steps or (5 if SMOKE else 4000); torch.manual_seed(seed); Y = Y[torch.randperm(len(Y))] if shuffle else Y; mu, sdv = X.mean(0), X.std(0) + 1e-6
|
| 35 |
+
net = nn.Sequential(nn.Linear(X.shape[1], 1024), nn.GELU(), nn.Linear(1024, 1024), nn.GELU(), nn.Linear(1024, 4 * D)); opt = torch.optim.Adam(net.parameters(), 1e-3)
|
| 36 |
+
for _ in range(steps):
|
| 37 |
+
b = torch.randint(0, len(X), (256,)); p = net((X[b] - mu) / sdv).view(-1, 4, D); loss = (1 - F.cosine_similarity(p, Y[b], -1)).mean(); opt.zero_grad(); loss.backward(); opt.step()
|
| 38 |
+
net.eval(); return lambda Z: net((Z - mu) / sdv).view(-1, 4, D).detach()
|
| 39 |
+
for sd in SEEDS:
|
| 40 |
+
cod = cods[sd]; XA = slots(cod, AUG)
|
| 41 |
+
A = torch.linalg.solve(XA.T @ XA + 10 * torch.eye(XA.shape[1]), XA.T @ YA.flatten(1))
|
| 42 |
+
fns = {'ridge_aug': lambda Z: (Z @ A).view(-1, 4, D), 'mlp_aug': train_mlp(XA, YA, seed=sd), 'mlp_ctrl_shuffled': train_mlp(XA, YA, shuffle=True, seed=sd)}
|
| 43 |
+
for name, fn in fns.items():
|
| 44 |
+
for s in ('sealed_ood', 'sealed_1_new', 'sealed_2_new', 'sealed_4_new'):
|
| 45 |
+
r = dec(fn(slots(cod, S[s])), S[s])
|
| 46 |
+
for i, c in enumerate(r): PRED.write(json.dumps({'sys': f'{name}_s{sd}', 'set': s, 'i': i, 'correct': bool(c)}) + '\n')
|
| 47 |
+
log({'probe': name, 'seed': sd, 'set': s, 'acc': sum(r) / len(r)})
|
| 48 |
+
# ---- random-weight receiver ----
|
| 49 |
+
torch.manual_seed(0); rnd = AutoModelForCausalLM.from_config(AutoConfig.from_pretrained(R.MP)).float().eval().requires_grad_(False).to(R.DEV)
|
| 50 |
+
rnd.model.embed_tokens.weight.data.copy_(R.Emb.data); R.lm = rnd
|
| 51 |
+
for s in ('sealed_ood', 'sealed_4_new'):
|
| 52 |
+
e = {'control': 'random_receiver', 'set': s, 'text_template': sum(per_ex(lambda mb: text_rows(mb, R.SUF_MAIN, R.TPL), S[s], CAND)) / len(S[s])}
|
| 53 |
+
for sd in SEEDS: e[f'k2_021_s{sd}'] = sum(per_ex(lambda mb, c=cods[sd]: k2_rows(c, mb, R.SUF_MAIN), S[s], CAND)) / len(S[s])
|
| 54 |
+
log(e)
|
| 55 |
+
PRED.close()
|
021/evalB.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CUERDAS-021 B: compact text with matched effort: few-shot and reparameterized soft prompt (~3.84M params, same steps/data/loss/selection).
|
| 2 |
+
from c021 import *
|
| 3 |
+
log = opener('resultsB.jsonl'); PRED = open(f'{OD}/predictionsB.jsonl', 'w'); SEEDS = [17] if SMOKE else [17, 29, 43]
|
| 4 |
+
FMT = {'values': "{action} {target} {status} {window}", 'values_csv': "{action}, {target}, {status}, {window}",
|
| 5 |
+
'kv': "action: {action}; target: {target}; status: {status}; window: {window}.", 'template': R.TPL}
|
| 6 |
+
def npos(fmt, ms): return sum(len(tok.encode(fmt.format(**dict(zip(FIELDS, m))), add_special_tokens=False)) for m in ms) / len(ms) + 15
|
| 7 |
+
def ev(sysname, rowfn, extra):
|
| 8 |
+
res = {}
|
| 9 |
+
for s in SETS:
|
| 10 |
+
r = per_ex(rowfn, S[s], CAND); res[s] = r
|
| 11 |
+
for i, c in enumerate(r): PRED.write(json.dumps({'sys': sysname, 'set': s, 'i': i, 'correct': bool(c)}) + '\n')
|
| 12 |
+
PRED.flush(); log({'sys': sysname, **extra, **{s: sum(v) / len(v) for s, v in res.items()}})
|
| 13 |
+
for fn, fmt in FMT.items():
|
| 14 |
+
log({'positions_per_event': fn, **{s: round(npos(fmt, S[s]), 2) for s in SETS}})
|
| 15 |
+
# few-shot: 2 fixed training demos with all 4 answers (shared, cacheable prefix)
|
| 16 |
+
demos = random.Random(0).sample(R.FIT, 2)
|
| 17 |
+
for fn, fmt in FMT.items():
|
| 18 |
+
pre = ''.join(fmt.format(**dict(zip(FIELDS, d))) + ''.join(R.SUF_MAIN.format(field=FIELDS[j]) + ' ' + d[j] + '.' for j in range(4)) + '\n\n' for d in demos)
|
| 19 |
+
P = len(tok.encode(pre, add_special_tokens=False))
|
| 20 |
+
ev(f'fewshot_{fn}', lambda mb, f=pre + fmt: text_rows(mb, R.SUF_MAIN, f), {'shared_prefix_positions': P})
|
| 21 |
+
class SoftPrompt(nn.Module): # prefix-tuning style reparameterization: P = base + MLP(base)
|
| 22 |
+
def __init__(s, seed, n=8, h=3328):
|
| 23 |
+
super().__init__(); torch.manual_seed(seed)
|
| 24 |
+
s.base = nn.Parameter(Emb[torch.randint(1000, 20000, (n,))].clone())
|
| 25 |
+
s.mlp = nn.Sequential(nn.Linear(D, h), nn.Tanh(), nn.Linear(h, D)); nn.init.zeros_(s.mlp[-1].weight); nn.init.zeros_(s.mlp[-1].bias)
|
| 26 |
+
def forward(s): return s.base + s.mlp(s.base)
|
| 27 |
+
for fn in ('values', 'values_csv', 'kv'):
|
| 28 |
+
fmt = FMT[fn]
|
| 29 |
+
for sd in SEEDS:
|
| 30 |
+
sp = SoftPrompt(sd); npar = sum(p.numel() for p in sp.parameters()); teach = load_teacher(sd)
|
| 31 |
+
step, v = train(list(sp.parameters()), lambda mb, tpl: text_rows(mb, tpl, fmt, prefix=sp()), teach, sd, f'sp_{fn}_s{sd}', log, set_mode=lambda t: sp.train(t))
|
| 32 |
+
sp.eval().requires_grad_(False); torch.save(sp.state_dict(), f'{OD}/sp_{fn}_s{sd}.pt')
|
| 33 |
+
with torch.no_grad(): pr = sp()
|
| 34 |
+
ev(f'sp_{fn}_s{sd}', lambda mb: text_rows(mb, R.SUF_MAIN, fmt, prefix=pr), {'params': npar, 'selected_step': step, 'val': v, 'soft_prompt_positions': 8})
|
| 35 |
+
PRED.close()
|
021/kaggle_kernel_A.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os,subprocess,sys,glob,shutil
|
| 2 |
+
subprocess.run([sys.executable,'-m','pip','install','-q','transformers==4.46.3'],check=True)
|
| 3 |
+
inp=os.path.dirname(glob.glob('/kaggle/input/**/sealed021.json',recursive=True)[0])
|
| 4 |
+
S='/kaggle/working/src'; os.makedirs(S+'/data',exist_ok=True); shutil.copy(f'{inp}/expanded.json',f'{S}/data/')
|
| 5 |
+
W='/kaggle/working/code'; os.makedirs(W,exist_ok=True)
|
| 6 |
+
for f in ('run018.py','c021.py','evalA.py'): shutil.copy(f'{inp}/{f}',W)
|
| 7 |
+
from transformers import AutoTokenizer,AutoModelForCausalLM
|
| 8 |
+
mp='/kaggle/working/m135'; R='12fd25f77366fa6b3b4b768ec3050bf629380bac'
|
| 9 |
+
AutoTokenizer.from_pretrained('HuggingFaceTB/SmolLM2-135M-Instruct',revision=R).save_pretrained(mp); AutoModelForCausalLM.from_pretrained('HuggingFaceTB/SmolLM2-135M-Instruct',revision=R).save_pretrained(mp)
|
| 10 |
+
subprocess.run([sys.executable,f'{W}/evalA.py'],env=dict(os.environ,SRC=S,MP=mp,OD='/kaggle/working',INP=inp),check=True)
|
| 11 |
+
shutil.rmtree(mp); shutil.rmtree(S); shutil.rmtree(W)
|
021/kaggle_kernel_B.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os,subprocess,sys,glob,shutil
|
| 2 |
+
subprocess.run([sys.executable,'-m','pip','install','-q','transformers==4.46.3'],check=True)
|
| 3 |
+
inp=os.path.dirname(glob.glob('/kaggle/input/**/sealed021.json',recursive=True)[0])
|
| 4 |
+
S='/kaggle/working/src'; os.makedirs(S+'/data',exist_ok=True); shutil.copy(f'{inp}/expanded.json',f'{S}/data/')
|
| 5 |
+
W='/kaggle/working/code'; os.makedirs(W,exist_ok=True)
|
| 6 |
+
for f in ('run018.py','c021.py','evalB.py'): shutil.copy(f'{inp}/{f}',W)
|
| 7 |
+
from transformers import AutoTokenizer,AutoModelForCausalLM
|
| 8 |
+
mp='/kaggle/working/m135'; R='12fd25f77366fa6b3b4b768ec3050bf629380bac'
|
| 9 |
+
AutoTokenizer.from_pretrained('HuggingFaceTB/SmolLM2-135M-Instruct',revision=R).save_pretrained(mp); AutoModelForCausalLM.from_pretrained('HuggingFaceTB/SmolLM2-135M-Instruct',revision=R).save_pretrained(mp)
|
| 10 |
+
subprocess.run([sys.executable,f'{W}/evalB.py'],env=dict(os.environ,SRC=S,MP=mp,OD='/kaggle/working',INP=inp),check=True)
|
| 11 |
+
shutil.rmtree(mp); shutil.rmtree(S); shutil.rmtree(W)
|
021/make_sealed021.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CUERDAS-021 sealed test generator (new seed). Unseen values: words, capitalized names, multi-token values.
|
| 2 |
+
import json, random, itertools, re, hashlib, sys
|
| 3 |
+
from transformers import AutoTokenizer
|
| 4 |
+
tok = AutoTokenizer.from_pretrained(sys.argv[1]); SEED = 20261017
|
| 5 |
+
meta = json.load(open(sys.argv[2])); V = meta['vocabulary']; rows = meta['rows']; SPL = meta['splits']
|
| 6 |
+
s020 = json.load(open(sys.argv[3]))
|
| 7 |
+
def enc(w): return tok.encode(' ' + w, add_special_tokens=False)
|
| 8 |
+
used = {tuple(r['values']) for r in rows}
|
| 9 |
+
fit = [tuple(rows[i]['values']) for i in SPL['fit']]; seen_at = {v[:2] for v in fit}; seen_sw = {v[2:] for v in fit}
|
| 10 |
+
isid = lambda v: v[:2] in seen_at and v[2:] in seen_sw
|
| 11 |
+
rng = random.Random(15015); uni = [v for v in itertools.product(*[range(len(x)) for x in V]) if v not in used]; rng.shuffle(uni)
|
| 12 |
+
dev = set([v for v in uni if isid(v)][:200] + [v for v in uni if not isid(v)][:400])
|
| 13 |
+
idx = lambda m: tuple(V[j].index(m[j]) for j in range(4))
|
| 14 |
+
s20 = {idx(m) for k in ('sealed_id', 'sealed_ood') for m in s020[k]}
|
| 15 |
+
elig = [v for v in itertools.product(*[range(len(x)) for x in V]) if v not in used and v not in dev and v not in s20 and not all(v[j] < (7, 7, 8, 5)[j] for j in range(4))]
|
| 16 |
+
r = random.Random(SEED); r.shuffle(elig)
|
| 17 |
+
sid = [v for v in elig if isid(v)][:200]; sood = [v for v in elig if not isid(v)][:400]
|
| 18 |
+
W = lambda v: [V[j][v[j]] for j in range(4)]
|
| 19 |
+
voc = tok.get_vocab()
|
| 20 |
+
words = sorted(t[1:] for t in voc if re.fullmatch(r'Ġ[a-z]{3,14}', t)); names = sorted(t[1:] for t in voc if re.fullmatch(r'Ġ[A-Z][a-z]{2,10}', t))
|
| 21 |
+
subs = sorted(t for t in voc if re.fullmatch(r'[a-z]{2,5}', t))
|
| 22 |
+
r.shuffle(words); r.shuffle(names); r.shuffle(subs)
|
| 23 |
+
old = set(sum(s020['new_words'], [])) | set('inspect repair cancel approve archive scan print send sensor battery printer server truck door screen cable broken stable late empty full warm missing urgent night evening weekend noon spring summer winter midnight'.split())
|
| 24 |
+
taken = {enc(w)[0] for vs in V for w in vs} | {enc(w)[0] for w in old}
|
| 25 |
+
NEW = [[] for _ in range(4)]; TYPE = {}
|
| 26 |
+
wi = iter(words); ni = iter(names)
|
| 27 |
+
def take(it):
|
| 28 |
+
while True:
|
| 29 |
+
w = next(it)
|
| 30 |
+
if len(enc(w)) == 1 and enc(w)[0] not in taken and w not in old: taken.add(enc(w)[0]); return w
|
| 31 |
+
for j in range(4):
|
| 32 |
+
for _ in range(3): w = take(wi); NEW[j].append(w); TYPE[w] = 'word'
|
| 33 |
+
for _ in range(3): w = take(ni); NEW[j].append(w); TYPE[w] = 'name'
|
| 34 |
+
a = take(wi); b = take(wi); p = a + ' ' + b; assert len(enc(p)) == 2; NEW[j].append(p); TYPE[p] = 'phrase'
|
| 35 |
+
while True:
|
| 36 |
+
a = take(wi); sb = subs.pop(); pw = a + sb
|
| 37 |
+
if enc(pw) == [enc(a)[0], tok.convert_tokens_to_ids(sb)]: NEW[j].append(pw); TYPE[pw] = 'multitoken_word'; break
|
| 38 |
+
excl = sorted({t for j in range(4) for w in NEW[j] for t in enc(w)})
|
| 39 |
+
def novel(n_new, n=192):
|
| 40 |
+
out = []
|
| 41 |
+
for k in range(n):
|
| 42 |
+
b = W(sood[k % 400])
|
| 43 |
+
for jj in r.sample(range(4), n_new): b[jj] = r.choice(NEW[jj])
|
| 44 |
+
out.append(b)
|
| 45 |
+
return out
|
| 46 |
+
S = {'seed': SEED, 'new_values': NEW, 'types': TYPE, 'exclude_token_ids': excl,
|
| 47 |
+
'sealed_id': [W(v) for v in sid], 'sealed_ood': [W(v) for v in sood], 'sealed_1_new': novel(1), 'sealed_2_new': novel(2), 'sealed_4_new': novel(4)}
|
| 48 |
+
s = json.dumps(S, sort_keys=True); open(sys.argv[4], 'w').write(s)
|
| 49 |
+
print('sha256', hashlib.sha256(s.encode()).hexdigest(), 'eligible', len(elig))
|
| 50 |
+
print(NEW)
|
021/predictionsA.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
021/predictionsB.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
021/preregistro021.md
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CUERDAS-021 — Pre-registro (escrito ANTES de cualquier evaluación sobre el conjunto sellado)
|
| 2 |
+
|
| 3 |
+
Fecha: 2026-10-16 (America/Mexico_City). Aprobado por Erik Sanchez. Solo recursos privados (Kaggle T4 gratis).
|
| 4 |
+
|
| 5 |
+
## Conjunto sellado nuevo
|
| 6 |
+
- `make_sealed021.py`, semilla nueva **20261017**. Universo elegible: combinaciones del vocabulario 014 que no están en filas 014, ni en dev-ID/dev-CH (semilla 15015), ni en el sellado 020, ni en el universo 013 → 16,894.
|
| 7 |
+
- `sealed_id` (200), `sealed_ood` (400, con al menos un par retenido), `sealed_1_new`/`2_new`/`4_new` (192 c/u).
|
| 8 |
+
- Valores nuevos: 8 por campo = 3 palabras minúsculas de 1 token, 3 nombres/palabras capitalizadas de 1 token, 1 frase de 2 palabras y 1 palabra de varios sub-tokens. Opciones por campo = vocabulario 014 + esos 8 (primer token único). Los números NO entran al test sellado: en este tokenizador todos empiezan con el token " ", y la lectura de primer token no los distingue; sí entran al entrenamiento.
|
| 9 |
+
- Archivo `sealed021.json`, **SHA-256 a86a6a5ee375ae67a3e79f969c9553b0e5548ed3b0e80e5315862e6552ae72bc**; el kernel lo verifica. Todos los tokens de los valores sellados se excluyen del pool de entrenamiento y de validación.
|
| 10 |
+
- Se evalúa **una sola vez**. Una corrida previa de prueba (SMOKE) usó solo mensajes de desarrollo, nunca el sellado.
|
| 11 |
+
|
| 12 |
+
## Exp. 1: K=2 reentrenado con pool variado
|
| 13 |
+
- Mismo protocolo que 018 (init desde K=2 de 017, maestro binder 015, CE+KL+0.5·coseno, 3,000 pasos, lr 5e-4, OneCycle, lote 12×4, 4 paráfrasis) pero el reemplazo (p=0.4 por campo) toma valores de 5 tipos: palabra minúscula (3–14 letras), nombre capitalizado, número 0–9999, frase de 2–3 palabras, pseudo-palabra multi-token.
|
| 14 |
+
- Semillas 17, 29, 43. Selección cada 250 pasos SOLO con validación: contrato (96) + transferencia restringida nueva (96, valores reservados de palabras/nombres/frases, disjuntos de entrenamiento y sellado).
|
| 15 |
+
- Reporte: exactitud por # de palabras nuevas (1/2/4) vs plantilla de texto, Wilson 95%, bootstrap pareado 10,000. Referencia: checkpoints 018 en el mismo sellado.
|
| 16 |
+
- Sondas sin LLM (ridge_aug λ=10, mlp_aug, mlp con objetivos barajados), entrenadas con 20,000 mensajes del pool de entrenamiento nuevo; decodificación por coseno contra el embedding léxico del valor completo. Receptor con pesos aleatorios.
|
| 17 |
+
|
| 18 |
+
## Exp. 2: texto compacto con esfuerzo equivalente
|
| 19 |
+
- Formatos: `values`, `values_csv`, `kv` (+ plantilla como referencia).
|
| 20 |
+
- Few-shot: 2 ejemplos fijos de FIT con las 4 respuestas (prefijo compartido, se reporta su largo aparte).
|
| 21 |
+
- Soft prompt reparametrizado (estilo prefix-tuning): 8 vectores = base + MLP(576→3328→576), **3,842,368 parámetros** (codificador K=2: 3,842,952). Mismos datos, pasos, pérdida (con el mismo maestro), optimizador y selección por validación que K=2. 3 semillas por formato.
|
| 22 |
+
- Posiciones por evento = tokens del registro + 15 (+8 del soft prompt, compartibles; se reportan ambos).
|
| 23 |
+
|
| 24 |
+
## Reglas de decisión
|
| 25 |
+
1. **Transferencia**: el reentrenamiento "mejora" si la media de 3 semillas en `sealed_4_new` supera a los checkpoints 018 con IC bootstrap > 0; "alcanza al texto" si queda a ≥ −2 pp de la plantilla.
|
| 26 |
+
2. **Ventaja vs texto compacto**: K=2 conserva ventaja si ningún formato compacto entrenado (soft prompt) o few-shot queda a ≥ −2 pp de K=2 en `sealed_ood` con menos de 17+8 posiciones totales; si alguno empata o gana, se dice que la ventaja de posiciones se reduce a la diferencia de posiciones medida.
|
| 27 |
+
3. **Lookup**: si una sonda sin LLM iguala al LLM en `sealed_4_new`, se reporta que la información es recuperable sin LLM.
|
| 28 |
+
Todo resultado se reporta.
|
021/resultados021.md
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CUERDAS-021 — Resultados (completo)
|
| 2 |
+
|
| 3 |
+
Fecha: 16 oct 2026 (hora CDMX). Pre-registro: `preregistro021.md` (SHA-256 1f2a27c7…). Conjunto sellado nuevo `sealed021.json` (semilla 20261017, SHA-256 a86a6a5e…72bc), verificado por el kernel y **evaluado una sola vez**. Kernel privado `cuerdas021-a` (Tesla T4). Datos: `gpu/resultsA.jsonl`, `gpu/predictionsA.jsonl`, checkpoints `gpu/k2v_s*.pt`.
|
| 4 |
+
|
| 5 |
+
Valores nuevos del sellado (8 por campo): 3 palabras, 3 nombres capitalizados, 1 frase de 2 palabras, 1 palabra multi-token. Exactitud exact-match (%), lectura de primer token; Wilson 95% y bootstrap pareado (10,000) de la media de 3 semillas.
|
| 6 |
+
|
| 7 |
+
## Parte A — K=2 reentrenado con pool variado (palabras, nombres, números, frases, multi-token)
|
| 8 |
+
|
| 9 |
+
Selección solo por validación (pasos elegidos: 2750 / 1500 / 1250).
|
| 10 |
+
|
| 11 |
+
| Conjunto (n) | Texto plantilla | K=2 018 (s17/s29/s43) | **K=2 021 (s17/s29/s43)** | Δ 021 vs texto | Δ 021 vs 018 |
|
| 12 |
+
|---|---|---|---|---|---|
|
| 13 |
+
| sealed_id (200) | 100 [98.1, 100] | 100 / 99.5 / 97.5 | 100 / 99.0 / 99.5 | −0.50 [−1.17, 0.00] | +0.50 |
|
| 14 |
+
| sealed_ood (400) | 98.5 [96.8, 99.3] | 99.0 / 99.0 / 97.8 | 99.75 / 98.5 / 97.75 (media 98.67) | **+0.17 [−1.25, +1.67]** | +0.08 [−0.92, +1.08] |
|
| 15 |
+
| 1 nueva (192) | 96.9 [93.4, 98.6] | 87.5 / 87.5 / 75.5 | 90.6 / 88.5 / 66.1 (81.8) | −15.10 [−19.97, −10.24] | −1.74 [−4.86, +1.39] |
|
| 16 |
+
| 2 nuevas (192) | 97.4 [94.0, 98.9] | 62.5 / 57.8 / 45.8 | 63.0 / 56.8 / 38.0 (52.6) | −44.79 [−50.87, −38.89] | −2.78 [−6.42, +0.87] |
|
| 17 |
+
| 4 nuevas (192) | 98.4 [95.5, 99.5] | 28.6 / 26.6 / 25.0 | 31.2 / 30.2 / 26.6 (29.3) | **−69.10 [−75.00, −63.02]** | +2.60 [−1.04, +6.25] |
|
| 18 |
+
|
| 19 |
+
Por tipo (1 nueva, s17): palabras 97%, nombres 93%, frases 79%, multi-token 69% (texto: 100/95/92/100%).
|
| 20 |
+
|
| 21 |
+
**Regla 1: NO se cumple.** El pool más grande y variado no mejora la transferencia (4 nuevas: +2.6 pp, IC cruza 0) y queda muy lejos del texto (−69 pp). Con nombres y valores multi-token, el codificador K=2 falla fuerte. La paridad en combinaciones nuevas de palabras conocidas se mantiene (98.67 vs 98.5%).
|
| 22 |
+
|
| 23 |
+
### Sondas sin LLM sobre las ranuras 021 (media 3 semillas)
|
| 24 |
+
|
| 25 |
+
| Decodificador | sealed_ood | 1 nueva | 2 nuevas | 4 nuevas |
|
| 26 |
+
|---|---|---|---|---|
|
| 27 |
+
| LLM congelado (K=2 021) | 98.67 | 81.8 | 52.6 | 29.3 |
|
| 28 |
+
| ridge_aug | 100 | 89.4 | 72.7 | **44.4** (+15.1 [+6.8, +23.4] vs LLM) |
|
| 29 |
+
| mlp_aug | 100 | **100** | **97.7** | 30.9 |
|
| 30 |
+
| mlp objetivos barajados (control) | 0 | 0 | 0 | 0 |
|
| 31 |
+
| Receptor pesos aleatorios (texto / K=2) | 0 / 0 | — | — | 0 / 0 |
|
| 32 |
+
|
| 33 |
+
**Regla 3: se cumple.** Sondas sin LLM igualan o superan al LLM leyendo las mismas ranuras (ridge +15 pp con 4 nuevas; MLP 97.7% vs 52.6% con 2 nuevas). La información de los valores nuevos **sí está en las ranuras**; el cuello de botella es que el LLM congelado no la lee (sobre todo nombres y multi-token). Las ranuras se comportan como un código casi léxico decodificable sin el LLM.
|
| 34 |
+
|
| 35 |
+
## Parte B — texto compacto con esfuerzo equivalente (kernel privado `cuerdas021-b`, ≈6 h de T4)
|
| 36 |
+
|
| 37 |
+
Soft prompt reparametrizado: 8 vectores, 3,842,368 parámetros (K=2: 3,842,952). Mismos datos, pasos, pérdida, maestro y selección por validación que K=2. Few-shot: 2 ejemplos fijos (prefijo compartido de 99–131 posiciones). Posiciones por evento = registro + 15 (el soft prompt suma 8 posiciones compartidas).
|
| 38 |
+
|
| 39 |
+
| Ruta | Posiciones/evento (+compartidas) | sealed_ood | 1 nueva | 2 nuevas | 4 nuevas |
|
| 40 |
+
|---|---|---|---|---|---|
|
| 41 |
+
| **K=2 021** (media 3 semillas) | **17** | 98.67 | 81.8 | 52.6 | 29.3 |
|
| 42 |
+
| Plantilla texto zero-shot | 35 | 98.5 | 96.9 | 97.4 | 98.4 |
|
| 43 |
+
| Soft prompt + `values` (s17/s29/s43) | 19.0–20.9 (+8) | 100/100/100 | 100 | 98.6 | 95.5 (96.9/96.4/93.2) |
|
| 44 |
+
| **Soft prompt + `values_csv`** | **22.0–23.9 (+8)** | **100** | **100** | **100** | **100** |
|
| 45 |
+
| Soft prompt + `kv` | 31.0–32.0 (+8) | 100 | 100 | 100 | 100 |
|
| 46 |
+
| Few-shot `kv` | 31 (+123) | 91.75 | 84.4 | 86.5 | 89.6 |
|
| 47 |
+
| Few-shot `values_csv` | 22 (+105) | 90.75 | 50.0 | 24.5 | 4.7 |
|
| 48 |
+
| Few-shot `values` | 19 (+99) | 32.5 | 3.1 | 0 | 0 |
|
| 49 |
+
| Few-shot plantilla | 35 (+131) | 86.5 | 66.7 | 58.3 | 38.5 |
|
| 50 |
+
|
| 51 |
+
Soft prompt `values_csv` − K=2 (bootstrap pareado, media de semillas): sealed_ood +1.33 [+0.58, +2.17]; 1 nueva +18.2 [+14.4, +22.4]; 2 nuevas +47.4 [+41.7, +53.1]; 4 nuevas **+70.7 [+64.9, +76.2]** pp.
|
| 52 |
+
|
| 53 |
+
**Regla 2: K=2 NO conserva ventaja.** Con el mismo presupuesto (parámetros, datos, pasos, maestro), un soft prompt delante de texto compacto con comas da **100% en todo** con 22 posiciones por evento (+8 compartidas), y con solo valores llega a 100% en conocidos y 93–97% con 4 nuevas a 19 posiciones. K=2 ahorra apenas 2–5 posiciones por evento (17 vs 19–22) y pierde hasta 70 pp con valores nuevos. Few-shot sin entrenar no alcanza.
|
| 54 |
+
|
| 55 |
+
## Veredicto honesto
|
| 56 |
+
- La ventaja de Cuerdas desaparece frente a un baseline justo. El "51% menos posiciones" era contra una plantilla verbosa; contra texto compacto entrenado el ahorro es de ~2–5 posiciones por evento (−11% a −23%), a cambio de perder mucha exactitud con valores nuevos.
|
| 57 |
+
- La paridad en combinaciones de palabras conocidas es real (dos tests sellados), pero el texto compacto entrenado la iguala o supera (100%).
|
| 58 |
+
- Las ranuras sí contienen la información (las sondas la leen), pero el receptor congelado no las lee bien; el texto compacto sí.
|
| 59 |
+
- Conclusión: como método, Cuerdas no tiene ventaja práctica en este escenario. Como reporte técnico, el resultado negativo, bien documentado y pre-registrado, es valioso.
|
021/resultsA.jsonl
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"sealed_sha256": "a86a6a5ee375ae67a3e79f969c9553b0e5548ed3b0e80e5315862e6552ae72bc", "torch": "2.11.0+cu128", "pool_sizes": {"words": 20654, "names": 8188, "subs": 7189}}
|
| 2 |
+
{"sys": "text_template", "sealed_id": 1.0, "sealed_ood": 0.985, "sealed_1_new": 0.96875, "sealed_2_new": 0.9739583333333334, "sealed_4_new": 0.984375}
|
| 3 |
+
{"sys": "k2_018_s17", "sealed_id": 1.0, "sealed_ood": 0.99, "sealed_1_new": 0.875, "sealed_2_new": 0.625, "sealed_4_new": 0.2864583333333333}
|
| 4 |
+
{"sys": "k2_018_s29", "sealed_id": 0.995, "sealed_ood": 0.99, "sealed_1_new": 0.875, "sealed_2_new": 0.578125, "sealed_4_new": 0.265625}
|
| 5 |
+
{"sys": "k2_018_s43", "sealed_id": 0.975, "sealed_ood": 0.9775, "sealed_1_new": 0.7552083333333334, "sealed_2_new": 0.4583333333333333, "sealed_4_new": 0.25}
|
| 6 |
+
{"run": "k2_021_s17", "step": 0, "val": 1.1145833333333333}
|
| 7 |
+
{"run": "k2_021_s17", "step": 250, "val": 1.0833333333333333, "loss": 4.719815254211426, "s": 51}
|
| 8 |
+
{"run": "k2_021_s17", "step": 500, "val": 1.0833333333333333, "loss": 5.05856990814209, "s": 101}
|
| 9 |
+
{"run": "k2_021_s17", "step": 750, "val": 1.0625, "loss": 4.387829780578613, "s": 154}
|
| 10 |
+
{"run": "k2_021_s17", "step": 1000, "val": 1.09375, "loss": 4.11569881439209, "s": 206}
|
| 11 |
+
{"run": "k2_021_s17", "step": 1250, "val": 1.1354166666666667, "loss": 4.992859363555908, "s": 259}
|
| 12 |
+
{"run": "k2_021_s17", "step": 1500, "val": 1.1458333333333335, "loss": 4.392324447631836, "s": 311}
|
| 13 |
+
{"run": "k2_021_s17", "step": 1750, "val": 1.1354166666666667, "loss": 4.7229413986206055, "s": 364}
|
| 14 |
+
{"run": "k2_021_s17", "step": 2000, "val": 1.1458333333333333, "loss": 4.124819278717041, "s": 417}
|
| 15 |
+
{"run": "k2_021_s17", "step": 2250, "val": 1.1666666666666667, "loss": 3.794137477874756, "s": 469}
|
| 16 |
+
{"run": "k2_021_s17", "step": 2500, "val": 1.1666666666666667, "loss": 2.527881622314453, "s": 521}
|
| 17 |
+
{"run": "k2_021_s17", "step": 2750, "val": 1.1770833333333333, "loss": 3.1677093505859375, "s": 573}
|
| 18 |
+
{"run": "k2_021_s17", "step": 3000, "val": 1.1666666666666667, "loss": 1.943210482597351, "s": 626}
|
| 19 |
+
{"selected": "k2_021_s17", "step": 2750, "val": 1.1770833333333333}
|
| 20 |
+
{"sys": "k2_021_s17", "sealed_id": 1.0, "sealed_ood": 0.9975, "sealed_1_new": 0.90625, "sealed_2_new": 0.6302083333333334, "sealed_4_new": 0.3125}
|
| 21 |
+
{"run": "k2_021_s29", "step": 0, "val": 1.0729166666666667}
|
| 22 |
+
{"run": "k2_021_s29", "step": 250, "val": 1.0520833333333335, "loss": 4.769656181335449, "s": 52}
|
| 23 |
+
{"run": "k2_021_s29", "step": 500, "val": 1.0833333333333333, "loss": 4.504295825958252, "s": 105}
|
| 24 |
+
{"run": "k2_021_s29", "step": 750, "val": 1.15625, "loss": 3.0690648555755615, "s": 157}
|
| 25 |
+
{"run": "k2_021_s29", "step": 1000, "val": 1.125, "loss": 2.2323856353759766, "s": 210}
|
| 26 |
+
{"run": "k2_021_s29", "step": 1250, "val": 1.1666666666666667, "loss": 4.31823205947876, "s": 263}
|
| 27 |
+
{"run": "k2_021_s29", "step": 1500, "val": 1.1770833333333333, "loss": 4.3195061683654785, "s": 315}
|
| 28 |
+
{"run": "k2_021_s29", "step": 1750, "val": 1.1354166666666667, "loss": 3.4602582454681396, "s": 367}
|
| 29 |
+
{"run": "k2_021_s29", "step": 2000, "val": 1.1041666666666667, "loss": 3.7912955284118652, "s": 419}
|
| 30 |
+
{"run": "k2_021_s29", "step": 2250, "val": 1.09375, "loss": 3.81158709526062, "s": 472}
|
| 31 |
+
{"run": "k2_021_s29", "step": 2500, "val": 1.1354166666666667, "loss": 2.8178346157073975, "s": 524}
|
| 32 |
+
{"run": "k2_021_s29", "step": 2750, "val": 1.1145833333333333, "loss": 3.228869676589966, "s": 577}
|
| 33 |
+
{"run": "k2_021_s29", "step": 3000, "val": 1.1145833333333333, "loss": 3.688642978668213, "s": 629}
|
| 34 |
+
{"selected": "k2_021_s29", "step": 1500, "val": 1.1770833333333333}
|
| 35 |
+
{"sys": "k2_021_s29", "sealed_id": 0.99, "sealed_ood": 0.985, "sealed_1_new": 0.8854166666666666, "sealed_2_new": 0.5677083333333334, "sealed_4_new": 0.3020833333333333}
|
| 36 |
+
{"run": "k2_021_s43", "step": 0, "val": 1.1041666666666667}
|
| 37 |
+
{"run": "k2_021_s43", "step": 250, "val": 1.0625, "loss": 3.4185612201690674, "s": 52}
|
| 38 |
+
{"run": "k2_021_s43", "step": 500, "val": 1.0, "loss": 4.755449295043945, "s": 104}
|
| 39 |
+
{"run": "k2_021_s43", "step": 750, "val": 1.0520833333333335, "loss": 4.684804916381836, "s": 157}
|
| 40 |
+
{"run": "k2_021_s43", "step": 1000, "val": 1.09375, "loss": 4.735645771026611, "s": 210}
|
| 41 |
+
{"run": "k2_021_s43", "step": 1250, "val": 1.1458333333333333, "loss": 3.7620046138763428, "s": 263}
|
| 42 |
+
{"run": "k2_021_s43", "step": 1500, "val": 1.1354166666666667, "loss": 4.758123397827148, "s": 315}
|
| 43 |
+
{"run": "k2_021_s43", "step": 1750, "val": 1.1145833333333333, "loss": 4.265966892242432, "s": 368}
|
| 44 |
+
{"run": "k2_021_s43", "step": 2000, "val": 1.0625, "loss": 3.849872589111328, "s": 419}
|
| 45 |
+
{"run": "k2_021_s43", "step": 2250, "val": 1.1041666666666665, "loss": 4.972720146179199, "s": 471}
|
| 46 |
+
{"run": "k2_021_s43", "step": 2500, "val": 1.125, "loss": 2.9099667072296143, "s": 523}
|
| 47 |
+
{"run": "k2_021_s43", "step": 2750, "val": 1.125, "loss": 3.000356674194336, "s": 575}
|
| 48 |
+
{"run": "k2_021_s43", "step": 3000, "val": 1.125, "loss": 2.915379762649536, "s": 626}
|
| 49 |
+
{"selected": "k2_021_s43", "step": 1250, "val": 1.1458333333333333}
|
| 50 |
+
{"sys": "k2_021_s43", "sealed_id": 0.995, "sealed_ood": 0.9775, "sealed_1_new": 0.6614583333333334, "sealed_2_new": 0.3802083333333333, "sealed_4_new": 0.265625}
|
| 51 |
+
{"probe": "ridge_aug", "seed": 17, "set": "sealed_ood", "acc": 1.0}
|
| 52 |
+
{"probe": "ridge_aug", "seed": 17, "set": "sealed_1_new", "acc": 0.890625}
|
| 53 |
+
{"probe": "ridge_aug", "seed": 17, "set": "sealed_2_new", "acc": 0.7135416666666666}
|
| 54 |
+
{"probe": "ridge_aug", "seed": 17, "set": "sealed_4_new", "acc": 0.4322916666666667}
|
| 55 |
+
{"probe": "mlp_aug", "seed": 17, "set": "sealed_ood", "acc": 1.0}
|
| 56 |
+
{"probe": "mlp_aug", "seed": 17, "set": "sealed_1_new", "acc": 1.0}
|
| 57 |
+
{"probe": "mlp_aug", "seed": 17, "set": "sealed_2_new", "acc": 0.984375}
|
| 58 |
+
{"probe": "mlp_aug", "seed": 17, "set": "sealed_4_new", "acc": 0.24479166666666666}
|
| 59 |
+
{"probe": "mlp_ctrl_shuffled", "seed": 17, "set": "sealed_ood", "acc": 0.0}
|
| 60 |
+
{"probe": "mlp_ctrl_shuffled", "seed": 17, "set": "sealed_1_new", "acc": 0.0}
|
| 61 |
+
{"probe": "mlp_ctrl_shuffled", "seed": 17, "set": "sealed_2_new", "acc": 0.0}
|
| 62 |
+
{"probe": "mlp_ctrl_shuffled", "seed": 17, "set": "sealed_4_new", "acc": 0.0}
|
| 63 |
+
{"probe": "ridge_aug", "seed": 29, "set": "sealed_ood", "acc": 1.0}
|
| 64 |
+
{"probe": "ridge_aug", "seed": 29, "set": "sealed_1_new", "acc": 0.9114583333333334}
|
| 65 |
+
{"probe": "ridge_aug", "seed": 29, "set": "sealed_2_new", "acc": 0.7604166666666666}
|
| 66 |
+
{"probe": "ridge_aug", "seed": 29, "set": "sealed_4_new", "acc": 0.421875}
|
| 67 |
+
{"probe": "mlp_aug", "seed": 29, "set": "sealed_ood", "acc": 1.0}
|
| 68 |
+
{"probe": "mlp_aug", "seed": 29, "set": "sealed_1_new", "acc": 1.0}
|
| 69 |
+
{"probe": "mlp_aug", "seed": 29, "set": "sealed_2_new", "acc": 0.984375}
|
| 70 |
+
{"probe": "mlp_aug", "seed": 29, "set": "sealed_4_new", "acc": 0.359375}
|
| 71 |
+
{"probe": "mlp_ctrl_shuffled", "seed": 29, "set": "sealed_ood", "acc": 0.0}
|
| 72 |
+
{"probe": "mlp_ctrl_shuffled", "seed": 29, "set": "sealed_1_new", "acc": 0.0}
|
| 73 |
+
{"probe": "mlp_ctrl_shuffled", "seed": 29, "set": "sealed_2_new", "acc": 0.0}
|
| 74 |
+
{"probe": "mlp_ctrl_shuffled", "seed": 29, "set": "sealed_4_new", "acc": 0.0}
|
| 75 |
+
{"probe": "ridge_aug", "seed": 43, "set": "sealed_ood", "acc": 1.0}
|
| 76 |
+
{"probe": "ridge_aug", "seed": 43, "set": "sealed_1_new", "acc": 0.8802083333333334}
|
| 77 |
+
{"probe": "ridge_aug", "seed": 43, "set": "sealed_2_new", "acc": 0.7083333333333334}
|
| 78 |
+
{"probe": "ridge_aug", "seed": 43, "set": "sealed_4_new", "acc": 0.4791666666666667}
|
| 79 |
+
{"probe": "mlp_aug", "seed": 43, "set": "sealed_ood", "acc": 1.0}
|
| 80 |
+
{"probe": "mlp_aug", "seed": 43, "set": "sealed_1_new", "acc": 1.0}
|
| 81 |
+
{"probe": "mlp_aug", "seed": 43, "set": "sealed_2_new", "acc": 0.9635416666666666}
|
| 82 |
+
{"probe": "mlp_aug", "seed": 43, "set": "sealed_4_new", "acc": 0.3229166666666667}
|
| 83 |
+
{"probe": "mlp_ctrl_shuffled", "seed": 43, "set": "sealed_ood", "acc": 0.0}
|
| 84 |
+
{"probe": "mlp_ctrl_shuffled", "seed": 43, "set": "sealed_1_new", "acc": 0.0}
|
| 85 |
+
{"probe": "mlp_ctrl_shuffled", "seed": 43, "set": "sealed_2_new", "acc": 0.0}
|
| 86 |
+
{"probe": "mlp_ctrl_shuffled", "seed": 43, "set": "sealed_4_new", "acc": 0.0}
|
| 87 |
+
{"control": "random_receiver", "set": "sealed_ood", "text_template": 0.0, "k2_021_s17": 0.0, "k2_021_s29": 0.0, "k2_021_s43": 0.0}
|
| 88 |
+
{"control": "random_receiver", "set": "sealed_4_new", "text_template": 0.0, "k2_021_s17": 0.0, "k2_021_s29": 0.0, "k2_021_s43": 0.0}
|
021/resultsB.jsonl
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"positions_per_event": "values", "sealed_id": 19.0, "sealed_ood": 19.0, "sealed_1_new": 19.4, "sealed_2_new": 19.95, "sealed_4_new": 20.85}
|
| 2 |
+
{"positions_per_event": "values_csv", "sealed_id": 22.0, "sealed_ood": 22.0, "sealed_1_new": 22.4, "sealed_2_new": 22.95, "sealed_4_new": 23.85}
|
| 3 |
+
{"positions_per_event": "kv", "sealed_id": 31.0, "sealed_ood": 31.0, "sealed_1_new": 31.21, "sealed_2_new": 31.41, "sealed_4_new": 32.01}
|
| 4 |
+
{"positions_per_event": "template", "sealed_id": 35.0, "sealed_ood": 35.0, "sealed_1_new": 35.21, "sealed_2_new": 35.41, "sealed_4_new": 36.01}
|
| 5 |
+
{"sys": "fewshot_values", "shared_prefix_positions": 99, "sealed_id": 0.345, "sealed_ood": 0.325, "sealed_1_new": 0.03125, "sealed_2_new": 0.0, "sealed_4_new": 0.0}
|
| 6 |
+
{"sys": "fewshot_values_csv", "shared_prefix_positions": 105, "sealed_id": 0.89, "sealed_ood": 0.9075, "sealed_1_new": 0.5, "sealed_2_new": 0.24479166666666666, "sealed_4_new": 0.046875}
|
| 7 |
+
{"sys": "fewshot_kv", "shared_prefix_positions": 123, "sealed_id": 0.945, "sealed_ood": 0.9175, "sealed_1_new": 0.84375, "sealed_2_new": 0.8645833333333334, "sealed_4_new": 0.8958333333333334}
|
| 8 |
+
{"sys": "fewshot_template", "shared_prefix_positions": 131, "sealed_id": 0.9, "sealed_ood": 0.865, "sealed_1_new": 0.6666666666666666, "sealed_2_new": 0.5833333333333334, "sealed_4_new": 0.3854166666666667}
|
| 9 |
+
{"run": "sp_values_s17", "step": 0, "val": 0.0}
|
| 10 |
+
{"run": "sp_values_s17", "step": 250, "val": 1.6145833333333333, "loss": 3.044680595397949, "s": 87}
|
| 11 |
+
{"run": "sp_values_s17", "step": 500, "val": 1.6666666666666667, "loss": 3.143272638320923, "s": 174}
|
| 12 |
+
{"run": "sp_values_s17", "step": 750, "val": 1.6770833333333335, "loss": 2.6209170818328857, "s": 261}
|
| 13 |
+
{"run": "sp_values_s17", "step": 1000, "val": 1.6041666666666665, "loss": 3.025043487548828, "s": 348}
|
| 14 |
+
{"run": "sp_values_s17", "step": 1250, "val": 1.7083333333333335, "loss": 2.137320041656494, "s": 435}
|
| 15 |
+
{"run": "sp_values_s17", "step": 1500, "val": 1.71875, "loss": 1.954566240310669, "s": 523}
|
| 16 |
+
{"run": "sp_values_s17", "step": 1750, "val": 1.6354166666666665, "loss": 3.326815128326416, "s": 610}
|
| 17 |
+
{"run": "sp_values_s17", "step": 2000, "val": 1.7291666666666665, "loss": 2.481618642807007, "s": 699}
|
| 18 |
+
{"run": "sp_values_s17", "step": 2250, "val": 1.7083333333333335, "loss": 2.0035409927368164, "s": 786}
|
| 19 |
+
{"run": "sp_values_s17", "step": 2500, "val": 1.7395833333333335, "loss": 1.278429627418518, "s": 872}
|
| 20 |
+
{"run": "sp_values_s17", "step": 2750, "val": 1.7708333333333335, "loss": 1.8047574758529663, "s": 960}
|
| 21 |
+
{"run": "sp_values_s17", "step": 3000, "val": 1.7395833333333335, "loss": 1.2187349796295166, "s": 1047}
|
| 22 |
+
{"sys": "sp_values_s17", "params": 3842368, "selected_step": 2750, "val": 1.7708333333333335, "soft_prompt_positions": 8, "sealed_id": 1.0, "sealed_ood": 1.0, "sealed_1_new": 1.0, "sealed_2_new": 0.984375, "sealed_4_new": 0.96875}
|
| 23 |
+
{"run": "sp_values_s29", "step": 0, "val": 0.0}
|
| 24 |
+
{"run": "sp_values_s29", "step": 250, "val": 1.375, "loss": 4.450610160827637, "s": 87}
|
| 25 |
+
{"run": "sp_values_s29", "step": 500, "val": 1.6145833333333335, "loss": 2.346332550048828, "s": 174}
|
| 26 |
+
{"run": "sp_values_s29", "step": 750, "val": 1.6145833333333335, "loss": 2.190868854522705, "s": 261}
|
| 27 |
+
{"run": "sp_values_s29", "step": 1000, "val": 1.6666666666666665, "loss": 1.3257291316986084, "s": 348}
|
| 28 |
+
{"run": "sp_values_s29", "step": 1250, "val": 1.6458333333333335, "loss": 2.51035737991333, "s": 436}
|
| 29 |
+
{"run": "sp_values_s29", "step": 1500, "val": 1.6875, "loss": 2.5555827617645264, "s": 523}
|
| 30 |
+
{"run": "sp_values_s29", "step": 1750, "val": 1.6041666666666665, "loss": 1.5838260650634766, "s": 609}
|
| 31 |
+
{"run": "sp_values_s29", "step": 2000, "val": 1.6770833333333335, "loss": 2.4831151962280273, "s": 697}
|
| 32 |
+
{"run": "sp_values_s29", "step": 2250, "val": 1.7083333333333335, "loss": 1.9796416759490967, "s": 784}
|
| 33 |
+
{"run": "sp_values_s29", "step": 2500, "val": 1.6666666666666665, "loss": 1.4970152378082275, "s": 871}
|
| 34 |
+
{"run": "sp_values_s29", "step": 2750, "val": 1.6875, "loss": 2.13800311088562, "s": 958}
|
| 35 |
+
{"run": "sp_values_s29", "step": 3000, "val": 1.6875, "loss": 2.1700429916381836, "s": 1044}
|
| 36 |
+
{"sys": "sp_values_s29", "params": 3842368, "selected_step": 2250, "val": 1.7083333333333335, "soft_prompt_positions": 8, "sealed_id": 1.0, "sealed_ood": 1.0, "sealed_1_new": 1.0, "sealed_2_new": 0.9895833333333334, "sealed_4_new": 0.9635416666666666}
|
| 37 |
+
{"run": "sp_values_s43", "step": 0, "val": 0.0}
|
| 38 |
+
{"run": "sp_values_s43", "step": 250, "val": 1.4583333333333333, "loss": 2.9020471572875977, "s": 87}
|
| 39 |
+
{"run": "sp_values_s43", "step": 500, "val": 1.46875, "loss": 2.5136404037475586, "s": 173}
|
| 40 |
+
{"run": "sp_values_s43", "step": 750, "val": 1.625, "loss": 2.120654821395874, "s": 262}
|
| 41 |
+
{"run": "sp_values_s43", "step": 1000, "val": 1.6770833333333335, "loss": 3.3165745735168457, "s": 349}
|
| 42 |
+
{"run": "sp_values_s43", "step": 1250, "val": 1.78125, "loss": 2.017547607421875, "s": 437}
|
| 43 |
+
{"run": "sp_values_s43", "step": 1500, "val": 1.78125, "loss": 2.347735643386841, "s": 524}
|
| 44 |
+
{"run": "sp_values_s43", "step": 1750, "val": 1.7708333333333335, "loss": 2.394887924194336, "s": 612}
|
| 45 |
+
{"run": "sp_values_s43", "step": 2000, "val": 1.625, "loss": 1.9778445959091187, "s": 699}
|
| 46 |
+
{"run": "sp_values_s43", "step": 2250, "val": 1.78125, "loss": 2.360875129699707, "s": 785}
|
| 47 |
+
{"run": "sp_values_s43", "step": 2500, "val": 1.7916666666666665, "loss": 1.7670954465866089, "s": 872}
|
| 48 |
+
{"run": "sp_values_s43", "step": 2750, "val": 1.75, "loss": 2.02799654006958, "s": 959}
|
| 49 |
+
{"run": "sp_values_s43", "step": 3000, "val": 1.7395833333333335, "loss": 1.7585619688034058, "s": 1046}
|
| 50 |
+
{"sys": "sp_values_s43", "params": 3842368, "selected_step": 2500, "val": 1.7916666666666665, "soft_prompt_positions": 8, "sealed_id": 1.0, "sealed_ood": 1.0, "sealed_1_new": 1.0, "sealed_2_new": 0.984375, "sealed_4_new": 0.9322916666666666}
|
| 51 |
+
{"run": "sp_values_csv_s17", "step": 0, "val": 0.0}
|
| 52 |
+
{"run": "sp_values_csv_s17", "step": 250, "val": 1.90625, "loss": 2.3863179683685303, "s": 95}
|
| 53 |
+
{"run": "sp_values_csv_s17", "step": 500, "val": 1.9583333333333335, "loss": 2.344421863555908, "s": 189}
|
| 54 |
+
{"run": "sp_values_csv_s17", "step": 750, "val": 1.9791666666666665, "loss": 1.9774900674819946, "s": 283}
|
| 55 |
+
{"run": "sp_values_csv_s17", "step": 1000, "val": 1.9583333333333335, "loss": 1.8494818210601807, "s": 377}
|
| 56 |
+
{"run": "sp_values_csv_s17", "step": 1250, "val": 1.9479166666666665, "loss": 1.641819715499878, "s": 472}
|
| 57 |
+
{"run": "sp_values_csv_s17", "step": 1500, "val": 1.9895833333333335, "loss": 1.3760128021240234, "s": 567}
|
| 58 |
+
{"run": "sp_values_csv_s17", "step": 1750, "val": 1.9895833333333335, "loss": 1.9606835842132568, "s": 662}
|
| 59 |
+
{"run": "sp_values_csv_s17", "step": 2000, "val": 2.0, "loss": 1.9302983283996582, "s": 758}
|
| 60 |
+
{"run": "sp_values_csv_s17", "step": 2250, "val": 1.9895833333333335, "loss": 1.5998188257217407, "s": 852}
|
| 61 |
+
{"run": "sp_values_csv_s17", "step": 2500, "val": 1.9895833333333335, "loss": 1.07892906665802, "s": 946}
|
| 62 |
+
{"run": "sp_values_csv_s17", "step": 2750, "val": 2.0, "loss": 1.3307654857635498, "s": 1040}
|
| 63 |
+
{"run": "sp_values_csv_s17", "step": 3000, "val": 2.0, "loss": 0.9787193536758423, "s": 1134}
|
| 64 |
+
{"sys": "sp_values_csv_s17", "params": 3842368, "selected_step": 2000, "val": 2.0, "soft_prompt_positions": 8, "sealed_id": 1.0, "sealed_ood": 1.0, "sealed_1_new": 1.0, "sealed_2_new": 1.0, "sealed_4_new": 1.0}
|
| 65 |
+
{"run": "sp_values_csv_s29", "step": 0, "val": 0.0}
|
| 66 |
+
{"run": "sp_values_csv_s29", "step": 250, "val": 1.9583333333333335, "loss": 2.902418851852417, "s": 94}
|
| 67 |
+
{"run": "sp_values_csv_s29", "step": 500, "val": 1.9375, "loss": 1.829451084136963, "s": 190}
|
| 68 |
+
{"run": "sp_values_csv_s29", "step": 750, "val": 1.9375, "loss": 1.5734684467315674, "s": 283}
|
| 69 |
+
{"run": "sp_values_csv_s29", "step": 1000, "val": 1.96875, "loss": 0.8627220988273621, "s": 378}
|
| 70 |
+
{"run": "sp_values_csv_s29", "step": 1250, "val": 1.9791666666666665, "loss": 1.8343243598937988, "s": 473}
|
| 71 |
+
{"run": "sp_values_csv_s29", "step": 1500, "val": 1.9895833333333335, "loss": 1.6268024444580078, "s": 567}
|
| 72 |
+
{"run": "sp_values_csv_s29", "step": 1750, "val": 1.96875, "loss": 1.224697232246399, "s": 661}
|
| 73 |
+
{"run": "sp_values_csv_s29", "step": 2000, "val": 1.9583333333333335, "loss": 1.8627893924713135, "s": 756}
|
| 74 |
+
{"run": "sp_values_csv_s29", "step": 2250, "val": 1.9583333333333335, "loss": 1.6173484325408936, "s": 851}
|
| 75 |
+
{"run": "sp_values_csv_s29", "step": 2500, "val": 1.9791666666666665, "loss": 1.2948013544082642, "s": 945}
|
| 76 |
+
{"run": "sp_values_csv_s29", "step": 2750, "val": 1.96875, "loss": 1.518986463546753, "s": 1040}
|
| 77 |
+
{"run": "sp_values_csv_s29", "step": 3000, "val": 1.96875, "loss": 1.6026052236557007, "s": 1134}
|
| 78 |
+
{"sys": "sp_values_csv_s29", "params": 3842368, "selected_step": 1500, "val": 1.9895833333333335, "soft_prompt_positions": 8, "sealed_id": 1.0, "sealed_ood": 1.0, "sealed_1_new": 1.0, "sealed_2_new": 1.0, "sealed_4_new": 1.0}
|
| 79 |
+
{"run": "sp_values_csv_s43", "step": 0, "val": 0.0}
|
| 80 |
+
{"run": "sp_values_csv_s43", "step": 250, "val": 1.8854166666666665, "loss": 2.186886787414551, "s": 95}
|
| 81 |
+
{"run": "sp_values_csv_s43", "step": 500, "val": 1.96875, "loss": 1.741999864578247, "s": 189}
|
| 82 |
+
{"run": "sp_values_csv_s43", "step": 750, "val": 1.8645833333333335, "loss": 1.5374534130096436, "s": 284}
|
| 83 |
+
{"run": "sp_values_csv_s43", "step": 1000, "val": 1.9583333333333335, "loss": 2.0583114624023438, "s": 379}
|
| 84 |
+
{"run": "sp_values_csv_s43", "step": 1250, "val": 1.9166666666666665, "loss": 1.2399407625198364, "s": 475}
|
| 85 |
+
{"run": "sp_values_csv_s43", "step": 1500, "val": 1.96875, "loss": 1.6992583274841309, "s": 570}
|
| 86 |
+
{"run": "sp_values_csv_s43", "step": 1750, "val": 1.9895833333333335, "loss": 1.7322700023651123, "s": 665}
|
| 87 |
+
{"run": "sp_values_csv_s43", "step": 2000, "val": 1.9583333333333335, "loss": 1.5983898639678955, "s": 759}
|
| 88 |
+
{"run": "sp_values_csv_s43", "step": 2250, "val": 1.96875, "loss": 1.6161710023880005, "s": 853}
|
| 89 |
+
{"run": "sp_values_csv_s43", "step": 2500, "val": 1.96875, "loss": 1.3451036214828491, "s": 947}
|
| 90 |
+
{"run": "sp_values_csv_s43", "step": 2750, "val": 1.9791666666666665, "loss": 1.2667261362075806, "s": 1042}
|
| 91 |
+
{"run": "sp_values_csv_s43", "step": 3000, "val": 1.96875, "loss": 1.4136821031570435, "s": 1137}
|
| 92 |
+
{"sys": "sp_values_csv_s43", "params": 3842368, "selected_step": 1750, "val": 1.9895833333333335, "soft_prompt_positions": 8, "sealed_id": 1.0, "sealed_ood": 1.0, "sealed_1_new": 1.0, "sealed_2_new": 1.0, "sealed_4_new": 1.0}
|
| 93 |
+
{"run": "sp_kv_s17", "step": 0, "val": 0.0625}
|
| 94 |
+
{"run": "sp_kv_s17", "step": 250, "val": 2.0, "loss": 1.9125851392745972, "s": 115}
|
| 95 |
+
{"run": "sp_kv_s17", "step": 500, "val": 2.0, "loss": 2.0021212100982666, "s": 230}
|
| 96 |
+
{"run": "sp_kv_s17", "step": 750, "val": 2.0, "loss": 1.5451703071594238, "s": 346}
|
| 97 |
+
{"run": "sp_kv_s17", "step": 1000, "val": 2.0, "loss": 1.4742144346237183, "s": 461}
|
| 98 |
+
{"run": "sp_kv_s17", "step": 1250, "val": 2.0, "loss": 1.4599279165267944, "s": 577}
|
| 99 |
+
{"run": "sp_kv_s17", "step": 1500, "val": 2.0, "loss": 1.2338306903839111, "s": 693}
|
| 100 |
+
{"run": "sp_kv_s17", "step": 1750, "val": 2.0, "loss": 1.8669570684432983, "s": 809}
|
| 101 |
+
{"run": "sp_kv_s17", "step": 2000, "val": 2.0, "loss": 1.7696266174316406, "s": 926}
|
| 102 |
+
{"run": "sp_kv_s17", "step": 2250, "val": 2.0, "loss": 1.350313425064087, "s": 1041}
|
| 103 |
+
{"run": "sp_kv_s17", "step": 2500, "val": 2.0, "loss": 0.9131432175636292, "s": 1156}
|
| 104 |
+
{"run": "sp_kv_s17", "step": 2750, "val": 2.0, "loss": 1.2375553846359253, "s": 1271}
|
| 105 |
+
{"run": "sp_kv_s17", "step": 3000, "val": 2.0, "loss": 0.8844823837280273, "s": 1386}
|
| 106 |
+
{"sys": "sp_kv_s17", "params": 3842368, "selected_step": 250, "val": 2.0, "soft_prompt_positions": 8, "sealed_id": 1.0, "sealed_ood": 1.0, "sealed_1_new": 1.0, "sealed_2_new": 1.0, "sealed_4_new": 1.0}
|
| 107 |
+
{"run": "sp_kv_s29", "step": 0, "val": 0.07291666666666667}
|
| 108 |
+
{"run": "sp_kv_s29", "step": 250, "val": 1.9895833333333335, "loss": 2.1503524780273438, "s": 115}
|
| 109 |
+
{"run": "sp_kv_s29", "step": 500, "val": 2.0, "loss": 1.5459184646606445, "s": 231}
|
| 110 |
+
{"run": "sp_kv_s29", "step": 750, "val": 1.9895833333333335, "loss": 1.2727928161621094, "s": 347}
|
| 111 |
+
{"run": "sp_kv_s29", "step": 1000, "val": 2.0, "loss": 0.745368242263794, "s": 462}
|
| 112 |
+
{"run": "sp_kv_s29", "step": 1250, "val": 2.0, "loss": 1.6049551963806152, "s": 579}
|
| 113 |
+
{"run": "sp_kv_s29", "step": 1500, "val": 2.0, "loss": 1.5594961643218994, "s": 694}
|
| 114 |
+
{"run": "sp_kv_s29", "step": 1750, "val": 2.0, "loss": 1.1384079456329346, "s": 809}
|
| 115 |
+
{"run": "sp_kv_s29", "step": 2000, "val": 1.96875, "loss": 1.4778153896331787, "s": 924}
|
| 116 |
+
{"run": "sp_kv_s29", "step": 2250, "val": 2.0, "loss": 1.5377285480499268, "s": 1040}
|
| 117 |
+
{"run": "sp_kv_s29", "step": 2500, "val": 2.0, "loss": 1.1550800800323486, "s": 1156}
|
| 118 |
+
{"run": "sp_kv_s29", "step": 2750, "val": 2.0, "loss": 1.3703510761260986, "s": 1271}
|
| 119 |
+
{"run": "sp_kv_s29", "step": 3000, "val": 2.0, "loss": 1.3216395378112793, "s": 1386}
|
| 120 |
+
{"sys": "sp_kv_s29", "params": 3842368, "selected_step": 500, "val": 2.0, "soft_prompt_positions": 8, "sealed_id": 1.0, "sealed_ood": 1.0, "sealed_1_new": 1.0, "sealed_2_new": 1.0, "sealed_4_new": 1.0}
|
| 121 |
+
{"run": "sp_kv_s43", "step": 0, "val": 0.07291666666666667}
|
| 122 |
+
{"run": "sp_kv_s43", "step": 250, "val": 2.0, "loss": 1.6502081155776978, "s": 116}
|
| 123 |
+
{"run": "sp_kv_s43", "step": 500, "val": 2.0, "loss": 1.304010272026062, "s": 231}
|
| 124 |
+
{"run": "sp_kv_s43", "step": 750, "val": 2.0, "loss": 1.260686993598938, "s": 347}
|
| 125 |
+
{"run": "sp_kv_s43", "step": 1000, "val": 2.0, "loss": 1.5032280683517456, "s": 463}
|
| 126 |
+
{"run": "sp_kv_s43", "step": 1250, "val": 2.0, "loss": 1.139223337173462, "s": 580}
|
| 127 |
+
{"run": "sp_kv_s43", "step": 1500, "val": 2.0, "loss": 1.4122791290283203, "s": 695}
|
| 128 |
+
{"run": "sp_kv_s43", "step": 1750, "val": 2.0, "loss": 1.3497402667999268, "s": 811}
|
| 129 |
+
{"run": "sp_kv_s43", "step": 2000, "val": 2.0, "loss": 1.427781581878662, "s": 927}
|
| 130 |
+
{"run": "sp_kv_s43", "step": 2250, "val": 2.0, "loss": 1.3728115558624268, "s": 1041}
|
| 131 |
+
{"run": "sp_kv_s43", "step": 2500, "val": 2.0, "loss": 1.1903347969055176, "s": 1157}
|
| 132 |
+
{"run": "sp_kv_s43", "step": 2750, "val": 2.0, "loss": 1.029153823852539, "s": 1273}
|
| 133 |
+
{"run": "sp_kv_s43", "step": 3000, "val": 2.0, "loss": 1.0028929710388184, "s": 1388}
|
| 134 |
+
{"sys": "sp_kv_s43", "params": 3842368, "selected_step": 250, "val": 2.0, "soft_prompt_positions": 8, "sealed_id": 1.0, "sealed_ood": 1.0, "sealed_1_new": 1.0, "sealed_2_new": 1.0, "sealed_4_new": 1.0}
|
021/sealed021.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"exclude_token_ids": [477, 2269, 4187, 5649, 8223, 9685, 10373, 10848, 14127, 15864, 15882, 17744, 19583, 19842, 19853, 19965, 21450, 22736, 24332, 24334, 24459, 24519, 25322, 25360, 25459, 25527, 29795, 30747, 30967, 32142, 33430, 34171, 36526, 36976, 37622, 38509, 43642, 46191, 46832, 47575], "new_values": [["tube", "remotely", "calming", "Insect", "Benin", "Mom", "purchases nat", "comfortablyifera"], ["interpolation", "heroine", "equine", "Bodies", "Fischer", "Gaussian", "electrolyte benefited", "geospatialard"], ["sarc", "downloads", "tackle", "Archived", "Filipino", "Fred", "linestyle seen", "identifierbud"], ["aspire", "careful", "manners", "Answer", "Director", "Prepar", "choice suffrage", "brickbc"]], "sealed_1_new": [["report", "customer", "active", "manners"], ["query", "Gaussian", "active", "minute"], ["delete", "machine", "closed", "choice suffrage"], ["create", "valve", "running", "Prepar"], ["update", "machine", "tackle", "today"], ["stop", "equine", "paid", "hour"], ["Benin", "motor", "ready", "hour"], ["report", "motor", "open", "aspire"], ["write", "electrolyte benefited", "active", "day"], ["create", "Fischer", "high", "hour"], ["purchases nat", "valve", "ready", "now"], ["read", "heroine", "pending", "day"], ["query", "electrolyte benefited", "idle", "week"], ["report", "Bodies", "active", "minute"], ["execute", "motor", "paid", "Director"], ["Insect", "motor", "running", "none"], ["tube", "pump", "ready", "hour"], ["read", "invoice", "tackle", "today"], ["stop", "Gaussian", "open", "minute"], ["notify", "customer", "pending", "aspire"], ["write", "Gaussian", "failed", "hour"], ["write", "valve", "complete", "Director"], ["write", "heroine", "high", "hour"], ["query", "file", "linestyle seen", "today"], ["update", "order", "open", "Director"], ["write", "invoice", "paid", "aspire"], ["stop", "email", "active", "aspire"], ["Insect", "inventory", "high", "today"], ["stop", "motor", "blocked", "manners"], ["read", "geospatialard", "active", "year"], ["Benin", "motor", "blocked", "none"], ["stop", "valve", "high", "Prepar"], ["Insect", "meeting", "overload", "month"], ["purchases nat", "machine", "paid", "today"], ["Insect", "meeting", "failed", "hour"], ["create", "heroine", "complete", "year"], ["Insect", "meeting", "failed", "today"], ["update", "motor", "closed", "Prepar"], ["write", "machine", "ready", "choice suffrage"], ["update", "file", "blocked", "careful"], ["write", "file", "overload", "careful"], ["stop", "equine", "overload", "month"], ["create", "Gaussian", "high", "hour"], ["read", "job", "tackle", "day"], ["stop", "heroine", "overload", "tomorrow"], ["create", "Fischer", "ready", "now"], ["read", "customer", "linestyle seen", "tomorrow"], ["read", "geospatialard", "closed", "day"], ["stop", "Bodies", "complete", "today"], ["comfortablyifera", "valve", "active", "day"], ["notify", "heroine", "none", "hour"], ["purchases nat", "customer", "failed", "today"], ["execute", "order", "sarc", "minute"], ["stop", "invoice", "Archived", "year"], ["execute", "Bodies", "ready", "hour"], ["calming", "inventory", "open", "tomorrow"], ["check", "file", "linestyle seen", "month"], ["query", "file", "Filipino", "minute"], ["tube", "motor", "low", "year"], ["notify", "job", "active", "aspire"], ["notify", "pump", "active", "careful"], ["start", "invoice", "tackle", "tomorrow"], ["purchases nat", "meeting", "normal", "year"], ["write", "customer", "Filipino", "year"], ["check", "customer", "tackle", "year"], ["delete", "motor", "tackle", "year"], ["query", "file", "overload", "manners"], ["notify", "pump", "idle", "careful"], ["read", "valve", "open", "Director"], ["check", "motor", "Filipino", "week"], ["Mom", "valve", "ready", "month"], ["check", "heroine", "running", "hour"], ["start", "customer", "Archived", "day"], ["read", "pump", "pending", "brickbc"], ["read", "Bodies", "active", "year"], ["write", "valve", "closed", "choice suffrage"], ["comfortablyifera", "customer", "ready", "month"], ["check", "motor", "idle", "careful"], ["start", "job", "Filipino", "year"], ["update", "file", "high", "Director"], ["execute", "interpolation", "active", "now"], ["execute", "invoice", "Filipino", "minute"], ["stop", "inventory", "idle", "Director"], ["comfortablyifera", "inventory", "open", "tomorrow"], ["write", "valve", "closed", "brickbc"], ["Mom", "order", "closed", "day"], ["Mom", "inventory", "running", "now"], ["query", "meeting", "active", "Prepar"], ["write", "machine", "Filipino", "none"], ["Insect", "file", "high", "today"], ["create", "order", "linestyle seen", "year"], ["query", "valve", "failed", "brickbc"], ["write", "interpolation", "ready", "day"], ["write", "customer", "Fred", "minute"], ["write", "heroine", "open", "year"], ["start", "interpolation", "none", "month"], ["execute", "valve", "paid", "careful"], ["check", "Gaussian", "open", "tomorrow"], ["report", "motor", "linestyle seen", "hour"], ["purchases nat", "inventory", "active", "week"], ["notify", "customer", "Fred", "hour"], ["execute", "order", "Filipino", "now"], ["stop", "heroine", "blocked", "none"], ["stop", "email", "closed", "manners"], ["stop", "heroine", "ready", "now"], ["write", "Bodies", "normal", "year"], ["calming", "file", "none", "hour"], ["Mom", "invoice", "closed", "now"], ["update", "invoice", "closed", "Prepar"], ["check", "geospatialard", "idle", "tomorrow"], ["delete", "Gaussian", "ready", "hour"], ["create", "electrolyte benefited", "ready", "tomorrow"], ["create", "order", "low", "manners"], ["write", "valve", "Archived", "minute"], ["notify", "pump", "Filipino", "minute"], ["Benin", "inventory", "closed", "week"], ["query", "machine", "open", "manners"], ["stop", "email", "tackle", "year"], ["write", "email", "Filipino", "none"], ["delete", "inventory", "failed", "Answer"], ["read", "geospatialard", "overload", "hour"], ["update", "email", "Filipino", "tomorrow"], ["check", "pump", "ready", "Prepar"], ["purchases nat", "order", "ready", "month"], ["query", "machine", "downloads", "today"], ["write", "pump", "pending", "Prepar"], ["check", "interpolation", "none", "day"], ["start", "machine", "overload", "aspire"], ["delete", "order", "low", "choice suffrage"], ["comfortablyifera", "meeting", "complete", "hour"], ["start", "job", "closed", "manners"], ["execute", "file", "none", "Answer"], ["Benin", "motor", "overload", "month"], ["check", "file", "downloads", "week"], ["create", "equine", "closed", "day"], ["check", "motor", "idle", "careful"], ["check", "customer", "Filipino", "week"], ["delete", "Fischer", "blocked", "day"], ["Mom", "valve", "pending", "day"], ["execute", "order", "Archived", "minute"], ["create", "inventory", "failed", "careful"], ["delete", "equine", "active", "day"], ["read", "invoice", "Archived", "tomorrow"], ["query", "Fischer", "active", "year"], ["read", "heroine", "low", "day"], ["delete", "machine", "running", "brickbc"], ["purchases nat", "inventory", "active", "month"], ["Benin", "order", "complete", "day"], ["write", "email", "tackle", "week"], ["create", "job", "tackle", "week"], ["delete", "electrolyte benefited", "active", "none"], ["stop", "Gaussian", "ready", "now"], ["tube", "pump", "idle", "day"], ["update", "order", "Filipino", "hour"], ["Mom", "invoice", "open", "month"], ["notify", "motor", "normal", "Director"], ["Mom", "valve", "ready", "hour"], ["read", "invoice", "overload", "aspire"], ["update", "Bodies", "idle", "none"], ["comfortablyifera", "email", "normal", "week"], ["execute", "geospatialard", "high", "minute"], ["read", "equine", "high", "month"], ["report", "pump", "identifierbud", "tomorrow"], ["query", "electrolyte benefited", "blocked", "tomorrow"], ["report", "email", "Archived", "hour"], ["purchases nat", "invoice", "none", "day"], ["Insect", "email", "pending", "hour"], ["start", "file", "downloads", "tomorrow"], ["report", "motor", "Filipino", "day"], ["stop", "inventory", "linestyle seen", "tomorrow"], ["start", "Gaussian", "high", "hour"], ["check", "machine", "sarc", "tomorrow"], ["start", "meeting", "blocked", "brickbc"], ["query", "heroine", "ready", "week"], ["query", "machine", "closed", "manners"], ["notify", "job", "open", "aspire"], ["notify", "meeting", "failed", "Answer"], ["write", "motor", "sarc", "minute"], ["start", "meeting", "idle", "Prepar"], ["write", "valve", "Filipino", "minute"], ["read", "pump", "linestyle seen", "now"], ["read", "order", "pending", "Answer"], ["Benin", "machine", "low", "year"], ["create", "valve", "blocked", "aspire"], ["read", "pump", "open", "careful"], ["start", "meeting", "Fred", "now"], ["calming", "pump", "pending", "day"], ["query", "motor", "high", "careful"], ["query", "equine", "paid", "hour"], ["execute", "valve", "normal", "Answer"], ["Mom", "pump", "none", "minute"], ["report", "equine", "complete", "day"]], "sealed_2_new": [["report", "equine", "active", "Director"], ["Benin", "valve", "active", "careful"], ["delete", "electrolyte benefited", "closed", "manners"], ["create", "Fischer", "running", "Answer"], ["update", "Fischer", "high", "aspire"], ["comfortablyifera", "Gaussian", "paid", "hour"], ["write", "electrolyte benefited", "ready", "brickbc"], ["Mom", "motor", "identifierbud", "none"], ["comfortablyifera", "equine", "active", "day"], ["Mom", "job", "high", "choice suffrage"], ["calming", "valve", "ready", "choice suffrage"], ["Insect", "motor", "sarc", "day"], ["comfortablyifera", "machine", "Filipino", "week"], ["report", "electrolyte benefited", "Fred", "minute"], ["remotely", "motor", "identifierbud", "month"], ["report", "Gaussian", "running", "Prepar"], ["stop", "heroine", "Archived", "hour"], ["read", "electrolyte benefited", "low", "Answer"], ["Insect", "interpolation", "open", "minute"], ["tube", "customer", "Archived", "day"], ["write", "Bodies", "failed", "manners"], ["write", "interpolation", "Filipino", "today"], ["write", "Fischer", "Fred", "hour"], ["query", "Bodies", "linestyle seen", "today"], ["update", "order", "linestyle seen", "careful"], ["write", "Bodies", "tackle", "month"], ["stop", "geospatialard", "downloads", "hour"], ["query", "equine", "Archived", "today"], ["purchases nat", "Gaussian", "blocked", "none"], ["tube", "pump", "sarc", "year"], ["calming", "equine", "blocked", "none"], ["Mom", "valve", "high", "manners"], ["remotely", "meeting", "overload", "Director"], ["Mom", "machine", "Archived", "today"], ["notify", "Bodies", "failed", "manners"], ["Benin", "job", "complete", "Answer"], ["write", "meeting", "Archived", "Director"], ["tube", "motor", "closed", "manners"], ["write", "machine", "Filipino", "Prepar"], ["purchases nat", "Bodies", "blocked", "none"], ["Mom", "heroine", "overload", "now"], ["stop", "job", "identifierbud", "Answer"], ["create", "pump", "Fred", "choice suffrage"], ["read", "interpolation", "downloads", "day"], ["Mom", "email", "Filipino", "tomorrow"], ["calming", "job", "ready", "Director"], ["read", "customer", "tackle", "careful"], ["calming", "meeting", "closed", "Answer"], ["stop", "Bodies", "complete", "careful"], ["comfortablyifera", "valve", "linestyle seen", "day"], ["notify", "Fischer", "none", "Director"], ["Mom", "customer", "failed", "Answer"], ["comfortablyifera", "order", "normal", "choice suffrage"], ["stop", "heroine", "Archived", "year"], ["execute", "equine", "linestyle seen", "hour"], ["execute", "Bodies", "open", "aspire"], ["check", "Bodies", "identifierbud", "month"], ["query", "file", "Filipino", "Prepar"], ["create", "Gaussian", "low", "manners"], ["remotely", "job", "tackle", "minute"], ["tube", "pump", "active", "Answer"], ["start", "Gaussian", "Filipino", "tomorrow"], ["Mom", "Fischer", "normal", "year"], ["Mom", "Gaussian", "low", "year"], ["check", "Bodies", "normal", "manners"], ["delete", "heroine", "normal", "choice suffrage"], ["query", "file", "Fred", "Prepar"], ["notify", "Fischer", "tackle", "now"], ["read", "equine", "open", "aspire"], ["check", "Bodies", "tackle", "week"], ["stop", "valve", "Filipino", "brickbc"], ["Benin", "motor", "Archived", "hour"], ["tube", "customer", "Archived", "day"], ["calming", "pump", "pending", "Answer"], ["read", "valve", "Fred", "aspire"], ["write", "heroine", "Archived", "none"], ["start", "geospatialard", "Archived", "month"], ["Benin", "motor", "idle", "careful"], ["purchases nat", "job", "active", "Answer"], ["Benin", "file", "high", "Answer"], ["Insect", "order", "Archived", "now"], ["comfortablyifera", "invoice", "identifierbud", "minute"], ["calming", "inventory", "Fred", "week"], ["remotely", "inventory", "open", "Director"], ["comfortablyifera", "valve", "sarc", "today"], ["create", "interpolation", "Filipino", "day"], ["update", "Bodies", "linestyle seen", "now"], ["remotely", "meeting", "active", "Answer"], ["Insect", "Fischer", "open", "none"], ["execute", "interpolation", "high", "careful"], ["create", "order", "tackle", "brickbc"], ["query", "electrolyte benefited", "failed", "Director"], ["purchases nat", "electrolyte benefited", "ready", "day"], ["Benin", "customer", "Archived", "minute"], ["comfortablyifera", "Fischer", "open", "year"], ["Mom", "Gaussian", "none", "month"], ["calming", "valve", "paid", "Answer"], ["check", "Fischer", "Fred", "tomorrow"], ["calming", "motor", "tackle", "hour"], ["stop", "equine", "tackle", "week"], ["notify", "interpolation", "linestyle seen", "hour"], ["remotely", "order", "normal", "careful"], ["Benin", "machine", "sarc", "none"], ["remotely", "email", "closed", "Director"], ["stop", "electrolyte benefited", "tackle", "now"], ["Mom", "pump", "normal", "Prepar"], ["stop", "electrolyte benefited", "none", "Answer"], ["calming", "invoice", "closed", "Prepar"], ["update", "Fischer", "closed", "brickbc"], ["check", "heroine", "identifierbud", "tomorrow"], ["delete", "email", "identifierbud", "Prepar"], ["create", "geospatialard", "ready", "careful"], ["tube", "Bodies", "low", "year"], ["purchases nat", "valve", "running", "Answer"], ["purchases nat", "electrolyte benefited", "ready", "minute"], ["read", "interpolation", "closed", "careful"], ["Insect", "machine", "open", "Prepar"], ["stop", "interpolation", "failed", "careful"], ["write", "email", "downloads", "aspire"], ["comfortablyifera", "inventory", "failed", "choice suffrage"], ["Insect", "geospatialard", "overload", "hour"], ["update", "email", "Filipino", "aspire"], ["Mom", "pump", "sarc", "hour"], ["start", "order", "downloads", "choice suffrage"], ["query", "machine", "Filipino", "manners"], ["remotely", "interpolation", "pending", "day"], ["check", "motor", "tackle", "Director"], ["calming", "electrolyte benefited", "overload", "now"], ["delete", "order", "downloads", "Prepar"], ["start", "Bodies", "complete", "choice suffrage"], ["start", "Gaussian", "closed", "Prepar"], ["calming", "interpolation", "none", "hour"], ["Benin", "motor", "overload", "Answer"], ["check", "Gaussian", "Fred", "week"], ["purchases nat", "machine", "Archived", "day"], ["check", "geospatialard", "Archived", "tomorrow"], ["check", "customer", "linestyle seen", "Answer"], ["delete", "machine", "identifierbud", "Director"], ["create", "valve", "Filipino", "careful"], ["Insect", "Gaussian", "failed", "minute"], ["create", "Fischer", "sarc", "hour"], ["delete", "customer", "downloads", "manners"], ["tube", "invoice", "complete", "Prepar"], ["query", "geospatialard", "active", "Prepar"], ["remotely", "meeting", "sarc", "day"], ["comfortablyifera", "Fischer", "running", "none"], ["stop", "interpolation", "identifierbud", "month"], ["calming", "heroine", "complete", "day"], ["write", "Gaussian", "linestyle seen", "week"], ["Benin", "job", "running", "manners"], ["remotely", "Fischer", "active", "none"], ["stop", "email", "tackle", "aspire"], ["tube", "pump", "Filipino", "day"], ["calming", "order", "identifierbud", "hour"], ["calming", "Fischer", "open", "month"], ["purchases nat", "motor", "tackle", "day"], ["remotely", "valve", "tackle", "hour"], ["comfortablyifera", "Fischer", "overload", "week"], ["Mom", "order", "idle", "manners"], ["remotely", "Gaussian", "normal", "week"], ["execute", "order", "linestyle seen", "Answer"], ["purchases nat", "pump", "downloads", "month"], ["calming", "pump", "identifierbud", "tomorrow"], ["remotely", "electrolyte benefited", "blocked", "tomorrow"], ["remotely", "email", "high", "aspire"], ["Mom", "invoice", "none", "Answer"], ["calming", "interpolation", "pending", "hour"], ["start", "interpolation", "identifierbud", "tomorrow"], ["purchases nat", "motor", "ready", "Director"], ["comfortablyifera", "inventory", "failed", "brickbc"], ["start", "motor", "Filipino", "manners"], ["check", "Fischer", "blocked", "Prepar"], ["calming", "meeting", "tackle", "none"], ["query", "Bodies", "tackle", "week"], ["Insect", "machine", "closed", "aspire"], ["Benin", "job", "open", "Answer"], ["notify", "interpolation", "Fred", "today"], ["purchases nat", "motor", "Fred", "minute"], ["start", "interpolation", "sarc", "hour"], ["write", "valve", "Filipino", "Prepar"], ["read", "pump", "Fred", "Prepar"], ["Insect", "order", "pending", "brickbc"], ["notify", "Bodies", "sarc", "year"], ["comfortablyifera", "valve", "blocked", "manners"], ["Benin", "Bodies", "open", "hour"], ["Insect", "meeting", "failed", "Answer"], ["Benin", "pump", "tackle", "day"], ["query", "heroine", "high", "Prepar"], ["tube", "file", "paid", "careful"], ["execute", "Gaussian", "linestyle seen", "today"], ["comfortablyifera", "pump", "none", "choice suffrage"], ["report", "heroine", "downloads", "day"]], "sealed_4_new": [["Mom", "electrolyte benefited", "sarc", "choice suffrage"], ["Mom", "heroine", "Archived", "brickbc"], ["Mom", "electrolyte benefited", "sarc", "choice suffrage"], ["tube", "Gaussian", "downloads", "brickbc"], ["Insect", "electrolyte benefited", "sarc", "choice suffrage"], ["comfortablyifera", "equine", "Filipino", "Answer"], ["Insect", "electrolyte benefited", "identifierbud", "manners"], ["tube", "Bodies", "Filipino", "manners"], ["remotely", "Fischer", "Archived", "brickbc"], ["purchases nat", "electrolyte benefited", "sarc", "Director"], ["purchases nat", "Bodies", "downloads", "manners"], ["tube", "electrolyte benefited", "downloads", "Prepar"], ["comfortablyifera", "interpolation", "downloads", "Answer"], ["Mom", "Fischer", "linestyle seen", "Director"], ["Insect", "Bodies", "tackle", "aspire"], ["Mom", "geospatialard", "Filipino", "Answer"], ["Benin", "Gaussian", "Filipino", "Prepar"], ["purchases nat", "heroine", "Filipino", "manners"], ["tube", "interpolation", "identifierbud", "brickbc"], ["tube", "Fischer", "Archived", "brickbc"], ["comfortablyifera", "Gaussian", "tackle", "manners"], ["Mom", "electrolyte benefited", "sarc", "brickbc"], ["Mom", "electrolyte benefited", "sarc", "Prepar"], ["Insect", "Bodies", "downloads", "Answer"], ["calming", "equine", "Filipino", "careful"], ["remotely", "Gaussian", "sarc", "Director"], ["calming", "heroine", "sarc", "Prepar"], ["Mom", "interpolation", "identifierbud", "Prepar"], ["calming", "Fischer", "Filipino", "careful"], ["calming", "electrolyte benefited", "downloads", "Answer"], ["comfortablyifera", "electrolyte benefited", "Fred", "brickbc"], ["Mom", "heroine", "tackle", "Prepar"], ["comfortablyifera", "Fischer", "linestyle seen", "careful"], ["comfortablyifera", "equine", "Filipino", "choice suffrage"], ["tube", "Bodies", "identifierbud", "Answer"], ["Insect", "equine", "Filipino", "aspire"], ["Benin", "Gaussian", "identifierbud", "brickbc"], ["comfortablyifera", "Bodies", "sarc", "Prepar"], ["Mom", "geospatialard", "linestyle seen", "choice suffrage"], ["Mom", "heroine", "sarc", "Director"], ["Benin", "Fischer", "downloads", "manners"], ["remotely", "Fischer", "sarc", "choice suffrage"], ["Mom", "geospatialard", "Fred", "Director"], ["purchases nat", "Fischer", "Fred", "Prepar"], ["Insect", "geospatialard", "sarc", "aspire"], ["comfortablyifera", "heroine", "downloads", "choice suffrage"], ["purchases nat", "Fischer", "Filipino", "manners"], ["comfortablyifera", "Bodies", "downloads", "Director"], ["comfortablyifera", "Fischer", "Fred", "aspire"], ["Mom", "heroine", "identifierbud", "Answer"], ["tube", "geospatialard", "Filipino", "careful"], ["purchases nat", "Gaussian", "identifierbud", "choice suffrage"], ["Mom", "electrolyte benefited", "linestyle seen", "Director"], ["Benin", "Fischer", "tackle", "choice suffrage"], ["purchases nat", "equine", "Archived", "Director"], ["tube", "Gaussian", "tackle", "Prepar"], ["comfortablyifera", "heroine", "Fred", "Prepar"], ["Benin", "Fischer", "tackle", "aspire"], ["purchases nat", "geospatialard", "linestyle seen", "Director"], ["Mom", "Fischer", "Filipino", "aspire"], ["tube", "equine", "tackle", "brickbc"], ["comfortablyifera", "equine", "sarc", "aspire"], ["Mom", "interpolation", "Filipino", "Director"], ["Insect", "electrolyte benefited", "linestyle seen", "manners"], ["tube", "Fischer", "Filipino", "Answer"], ["comfortablyifera", "Fischer", "Filipino", "manners"], ["remotely", "geospatialard", "downloads", "brickbc"], ["Benin", "heroine", "sarc", "Answer"], ["Insect", "heroine", "sarc", "choice suffrage"], ["Benin", "interpolation", "Archived", "Prepar"], ["tube", "interpolation", "Fred", "Director"], ["Mom", "Bodies", "Fred", "Director"], ["tube", "Fischer", "Filipino", "brickbc"], ["Mom", "equine", "downloads", "aspire"], ["Mom", "Gaussian", "sarc", "Director"], ["tube", "Bodies", "downloads", "Answer"], ["Mom", "geospatialard", "Filipino", "manners"], ["Benin", "electrolyte benefited", "Filipino", "manners"], ["Mom", "electrolyte benefited", "Fred", "Director"], ["comfortablyifera", "equine", "linestyle seen", "Prepar"], ["Benin", "Fischer", "downloads", "careful"], ["Mom", "equine", "sarc", "aspire"], ["tube", "Bodies", "tackle", "aspire"], ["Insect", "electrolyte benefited", "Fred", "careful"], ["calming", "interpolation", "Filipino", "manners"], ["Insect", "Bodies", "tackle", "brickbc"], ["Benin", "electrolyte benefited", "identifierbud", "brickbc"], ["Mom", "geospatialard", "Filipino", "Director"], ["calming", "Fischer", "sarc", "careful"], ["comfortablyifera", "Gaussian", "sarc", "Director"], ["tube", "Fischer", "downloads", "Answer"], ["comfortablyifera", "equine", "Fred", "careful"], ["comfortablyifera", "geospatialard", "Fred", "Answer"], ["comfortablyifera", "Gaussian", "Archived", "manners"], ["Mom", "geospatialard", "Filipino", "Director"], ["tube", "geospatialard", "linestyle seen", "manners"], ["calming", "Bodies", "Fred", "choice suffrage"], ["Insect", "Gaussian", "tackle", "Answer"], ["tube", "electrolyte benefited", "downloads", "manners"], ["Insect", "interpolation", "downloads", "choice suffrage"], ["Benin", "heroine", "Archived", "careful"], ["tube", "heroine", "sarc", "Answer"], ["tube", "geospatialard", "downloads", "brickbc"], ["Benin", "heroine", "sarc", "careful"], ["tube", "Fischer", "downloads", "careful"], ["calming", "heroine", "Archived", "Answer"], ["Benin", "heroine", "Fred", "choice suffrage"], ["Insect", "equine", "Filipino", "aspire"], ["remotely", "equine", "Filipino", "Director"], ["remotely", "geospatialard", "tackle", "Prepar"], ["calming", "electrolyte benefited", "identifierbud", "Answer"], ["comfortablyifera", "Fischer", "downloads", "Director"], ["Mom", "Bodies", "downloads", "brickbc"], ["Benin", "Fischer", "Archived", "manners"], ["comfortablyifera", "electrolyte benefited", "identifierbud", "Prepar"], ["calming", "Gaussian", "Fred", "aspire"], ["remotely", "electrolyte benefited", "identifierbud", "choice suffrage"], ["calming", "heroine", "sarc", "careful"], ["Benin", "interpolation", "downloads", "manners"], ["tube", "Gaussian", "Archived", "Director"], ["tube", "equine", "tackle", "brickbc"], ["calming", "electrolyte benefited", "Filipino", "Answer"], ["Insect", "Gaussian", "downloads", "Answer"], ["tube", "electrolyte benefited", "identifierbud", "choice suffrage"], ["tube", "geospatialard", "downloads", "Prepar"], ["remotely", "equine", "Archived", "Prepar"], ["calming", "electrolyte benefited", "sarc", "careful"], ["Mom", "heroine", "linestyle seen", "careful"], ["comfortablyifera", "electrolyte benefited", "Archived", "brickbc"], ["comfortablyifera", "geospatialard", "Filipino", "aspire"], ["Insect", "Bodies", "downloads", "choice suffrage"], ["Mom", "Fischer", "tackle", "choice suffrage"], ["tube", "equine", "tackle", "careful"], ["calming", "Fischer", "Fred", "Answer"], ["tube", "interpolation", "downloads", "choice suffrage"], ["purchases nat", "interpolation", "tackle", "brickbc"], ["purchases nat", "Fischer", "linestyle seen", "choice suffrage"], ["comfortablyifera", "interpolation", "identifierbud", "manners"], ["comfortablyifera", "Bodies", "Filipino", "Prepar"], ["purchases nat", "heroine", "sarc", "manners"], ["calming", "heroine", "linestyle seen", "aspire"], ["tube", "equine", "Fred", "careful"], ["purchases nat", "geospatialard", "Fred", "Prepar"], ["purchases nat", "equine", "downloads", "choice suffrage"], ["Mom", "Fischer", "Archived", "manners"], ["comfortablyifera", "equine", "Fred", "careful"], ["Benin", "Gaussian", "Archived", "careful"], ["Insect", "interpolation", "sarc", "choice suffrage"], ["comfortablyifera", "geospatialard", "linestyle seen", "careful"], ["remotely", "Gaussian", "Archived", "Answer"], ["purchases nat", "Bodies", "Archived", "Answer"], ["tube", "Gaussian", "linestyle seen", "Answer"], ["comfortablyifera", "heroine", "linestyle seen", "Prepar"], ["remotely", "electrolyte benefited", "linestyle seen", "Director"], ["purchases nat", "Gaussian", "downloads", "Director"], ["purchases nat", "heroine", "tackle", "brickbc"], ["calming", "interpolation", "sarc", "Prepar"], ["purchases nat", "geospatialard", "downloads", "manners"], ["Benin", "geospatialard", "downloads", "brickbc"], ["comfortablyifera", "Fischer", "Fred", "choice suffrage"], ["Insect", "Fischer", "downloads", "careful"], ["tube", "Fischer", "Archived", "careful"], ["Benin", "heroine", "identifierbud", "choice suffrage"], ["calming", "Gaussian", "identifierbud", "manners"], ["calming", "heroine", "tackle", "manners"], ["Benin", "Bodies", "tackle", "Prepar"], ["Benin", "Fischer", "identifierbud", "choice suffrage"], ["Mom", "electrolyte benefited", "Fred", "Answer"], ["comfortablyifera", "Gaussian", "downloads", "brickbc"], ["remotely", "equine", "Archived", "manners"], ["calming", "Bodies", "sarc", "Director"], ["purchases nat", "interpolation", "linestyle seen", "choice suffrage"], ["comfortablyifera", "interpolation", "downloads", "careful"], ["calming", "interpolation", "Fred", "brickbc"], ["remotely", "heroine", "sarc", "brickbc"], ["Mom", "geospatialard", "Filipino", "Prepar"], ["Benin", "geospatialard", "identifierbud", "brickbc"], ["remotely", "Fischer", "downloads", "manners"], ["Mom", "Fischer", "Filipino", "aspire"], ["calming", "Gaussian", "tackle", "careful"], ["comfortablyifera", "Fischer", "sarc", "choice suffrage"], ["Mom", "Bodies", "Fred", "choice suffrage"], ["remotely", "geospatialard", "tackle", "manners"], ["Insect", "Fischer", "tackle", "Director"], ["Insect", "Fischer", "identifierbud", "Prepar"], ["Benin", "Gaussian", "Archived", "choice suffrage"], ["tube", "Bodies", "sarc", "brickbc"], ["Insect", "electrolyte benefited", "Fred", "careful"], ["comfortablyifera", "geospatialard", "Fred", "brickbc"], ["comfortablyifera", "geospatialard", "sarc", "brickbc"], ["Insect", "interpolation", "sarc", "aspire"], ["comfortablyifera", "geospatialard", "sarc", "Answer"]], "sealed_id": [["delete", "email", "active", "tomorrow"], ["check", "invoice", "failed", "now"], ["stop", "motor", "active", "week"], ["stop", "motor", "normal", "hour"], ["notify", "invoice", "normal", "now"], ["read", "meeting", "closed", "none"], ["stop", "meeting", "complete", "now"], ["check", "meeting", "ready", "minute"], ["query", "meeting", "idle", "minute"], ["notify", "meeting", "high", "day"], ["report", "job", "running", "none"], ["read", "meeting", "none", "day"], ["update", "job", "high", "none"], ["stop", "meeting", "complete", "tomorrow"], ["notify", "machine", "blocked", "minute"], ["start", "valve", "high", "month"], ["write", "motor", "blocked", "minute"], ["execute", "machine", "none", "year"], ["check", "email", "pending", "minute"], ["delete", "invoice", "ready", "minute"], ["report", "machine", "open", "today"], ["execute", "pump", "closed", "minute"], ["execute", "pump", "blocked", "year"], ["query", "meeting", "open", "hour"], ["check", "inventory", "open", "week"], ["report", "email", "idle", "week"], ["start", "motor", "normal", "minute"], ["delete", "meeting", "low", "tomorrow"], ["query", "inventory", "complete", "minute"], ["write", "order", "open", "hour"], ["write", "machine", "overload", "none"], ["query", "meeting", "blocked", "today"], ["notify", "email", "overload", "today"], ["execute", "pump", "none", "none"], ["check", "invoice", "running", "tomorrow"], ["check", "inventory", "paid", "tomorrow"], ["delete", "order", "none", "minute"], ["check", "pump", "high", "year"], ["update", "valve", "blocked", "day"], ["delete", "motor", "blocked", "now"], ["stop", "order", "normal", "today"], ["report", "invoice", "pending", "hour"], ["read", "machine", "none", "week"], ["read", "inventory", "idle", "year"], ["write", "invoice", "normal", "hour"], ["start", "email", "failed", "minute"], ["write", "pump", "high", "minute"], ["check", "inventory", "complete", "today"], ["notify", "valve", "open", "week"], ["read", "customer", "low", "hour"], ["execute", "file", "open", "year"], ["start", "motor", "ready", "tomorrow"], ["update", "motor", "low", "hour"], ["notify", "motor", "closed", "hour"], ["report", "meeting", "pending", "today"], ["execute", "pump", "paid", "day"], ["start", "order", "ready", "tomorrow"], ["check", "meeting", "failed", "day"], ["delete", "motor", "closed", "tomorrow"], ["notify", "invoice", "ready", "tomorrow"], ["notify", "machine", "active", "tomorrow"], ["create", "file", "normal", "today"], ["execute", "customer", "paid", "minute"], ["create", "valve", "high", "none"], ["report", "meeting", "paid", "tomorrow"], ["start", "customer", "failed", "now"], ["query", "meeting", "active", "none"], ["read", "email", "pending", "today"], ["notify", "machine", "complete", "tomorrow"], ["check", "valve", "running", "none"], ["write", "invoice", "low", "today"], ["update", "machine", "ready", "today"], ["execute", "meeting", "none", "minute"], ["execute", "customer", "closed", "month"], ["query", "email", "blocked", "day"], ["read", "motor", "low", "hour"], ["update", "machine", "idle", "none"], ["write", "order", "failed", "now"], ["report", "file", "idle", "none"], ["query", "customer", "low", "month"], ["check", "email", "low", "tomorrow"], ["query", "invoice", "closed", "today"], ["create", "machine", "ready", "now"], ["start", "motor", "none", "today"], ["check", "customer", "overload", "day"], ["execute", "inventory", "blocked", "hour"], ["start", "machine", "closed", "none"], ["write", "customer", "running", "hour"], ["stop", "valve", "ready", "none"], ["notify", "order", "idle", "day"], ["update", "customer", "open", "now"], ["delete", "meeting", "complete", "tomorrow"], ["delete", "order", "normal", "today"], ["read", "customer", "high", "minute"], ["write", "meeting", "idle", "none"], ["start", "email", "closed", "hour"], ["report", "valve", "blocked", "year"], ["write", "pump", "complete", "minute"], ["stop", "meeting", "complete", "hour"], ["stop", "invoice", "closed", "now"], ["execute", "meeting", "closed", "now"], ["execute", "motor", "active", "hour"], ["start", "email", "failed", "tomorrow"], ["create", "valve", "overload", "day"], ["read", "file", "failed", "year"], ["delete", "file", "blocked", "minute"], ["report", "order", "blocked", "year"], ["check", "job", "complete", "now"], ["update", "job", "paid", "today"], ["read", "valve", "pending", "tomorrow"], ["notify", "file", "normal", "month"], ["update", "motor", "active", "tomorrow"], ["write", "job", "normal", "week"], ["notify", "meeting", "closed", "minute"], ["execute", "machine", "normal", "tomorrow"], ["check", "order", "normal", "none"], ["execute", "job", "active", "week"], ["create", "motor", "failed", "week"], ["write", "pump", "ready", "none"], ["delete", "order", "active", "week"], ["query", "job", "closed", "year"], ["report", "order", "ready", "tomorrow"], ["delete", "invoice", "ready", "tomorrow"], ["write", "motor", "running", "hour"], ["delete", "order", "open", "hour"], ["stop", "invoice", "active", "hour"], ["start", "valve", "complete", "week"], ["check", "valve", "running", "today"], ["notify", "motor", "blocked", "now"], ["read", "customer", "blocked", "day"], ["start", "email", "blocked", "now"], ["write", "customer", "failed", "year"], ["check", "valve", "complete", "tomorrow"], ["report", "machine", "complete", "tomorrow"], ["stop", "invoice", "high", "now"], ["notify", "customer", "idle", "none"], ["stop", "job", "normal", "now"], ["stop", "order", "complete", "now"], ["notify", "machine", "ready", "now"], ["create", "inventory", "blocked", "year"], ["create", "pump", "blocked", "now"], ["start", "order", "blocked", "now"], ["read", "motor", "ready", "year"], ["start", "pump", "running", "tomorrow"], ["delete", "valve", "low", "none"], ["read", "job", "idle", "minute"], ["start", "customer", "overload", "day"], ["execute", "job", "paid", "year"], ["read", "meeting", "overload", "minute"], ["read", "motor", "overload", "hour"], ["update", "meeting", "overload", "year"], ["read", "machine", "complete", "minute"], ["execute", "machine", "high", "day"], ["write", "motor", "blocked", "now"], ["query", "inventory", "failed", "tomorrow"], ["query", "order", "open", "month"], ["delete", "job", "complete", "minute"], ["start", "customer", "idle", "month"], ["notify", "email", "normal", "today"], ["update", "motor", "high", "minute"], ["delete", "invoice", "none", "year"], ["notify", "file", "blocked", "year"], ["delete", "email", "low", "hour"], ["stop", "valve", "running", "minute"], ["query", "motor", "normal", "week"], ["notify", "order", "blocked", "hour"], ["create", "invoice", "active", "week"], ["stop", "invoice", "blocked", "month"], ["execute", "meeting", "ready", "week"], ["delete", "motor", "pending", "now"], ["create", "valve", "complete", "today"], ["report", "email", "pending", "year"], ["start", "pump", "active", "day"], ["report", "pump", "overload", "tomorrow"], ["write", "meeting", "paid", "day"], ["create", "motor", "complete", "minute"], ["create", "inventory", "running", "now"], ["read", "valve", "idle", "year"], ["query", "inventory", "normal", "hour"], ["execute", "inventory", "pending", "minute"], ["notify", "order", "open", "week"], ["start", "email", "closed", "none"], ["read", "machine", "failed", "now"], ["check", "pump", "high", "none"], ["notify", "machine", "paid", "minute"], ["stop", "job", "ready", "week"], ["report", "order", "high", "week"], ["report", "pump", "normal", "month"], ["notify", "order", "complete", "tomorrow"], ["update", "pump", "open", "month"], ["delete", "file", "idle", "day"], ["create", "customer", "overload", "week"], ["report", "machine", "paid", "minute"], ["read", "email", "none", "day"], ["check", "order", "none", "now"], ["create", "inventory", "ready", "tomorrow"], ["write", "customer", "paid", "today"], ["report", "file", "normal", "today"], ["create", "pump", "paid", "tomorrow"], ["update", "meeting", "blocked", "month"]], "sealed_ood": [["report", "customer", "active", "now"], ["query", "valve", "active", "minute"], ["delete", "machine", "closed", "today"], ["create", "valve", "running", "year"], ["update", "machine", "high", "today"], ["stop", "email", "paid", "hour"], ["write", "motor", "ready", "hour"], ["report", "motor", "open", "none"], ["write", "email", "active", "day"], ["create", "job", "high", "hour"], ["execute", "valve", "ready", "now"], ["read", "motor", "pending", "day"], ["query", "machine", "idle", "week"], ["report", "machine", "active", "minute"], ["execute", "motor", "paid", "month"], ["report", "motor", "running", "none"], ["stop", "pump", "ready", "hour"], ["read", "invoice", "low", "today"], ["stop", "email", "open", "minute"], ["notify", "customer", "pending", "day"], ["write", "valve", "failed", "hour"], ["write", "valve", "complete", "today"], ["write", "job", "high", "hour"], ["query", "file", "low", "today"], ["update", "order", "open", "now"], ["write", "invoice", "paid", "month"], ["stop", "email", "active", "hour"], ["query", "inventory", "high", "today"], ["stop", "motor", "blocked", "none"], ["read", "pump", "active", "year"], ["update", "motor", "blocked", "none"], ["stop", "valve", "high", "hour"], ["check", "meeting", "overload", "month"], ["query", "machine", "paid", "today"], ["notify", "meeting", "failed", "hour"], ["create", "job", "complete", "year"], ["write", "meeting", "failed", "today"], ["update", "motor", "closed", "day"], ["write", "machine", "ready", "hour"], ["update", "file", "blocked", "none"], ["write", "file", "overload", "now"], ["stop", "job", "overload", "month"], ["create", "pump", "high", "hour"], ["read", "job", "pending", "day"], ["stop", "email", "overload", "tomorrow"], ["create", "job", "ready", "now"], ["read", "customer", "blocked", "tomorrow"], ["read", "meeting", "closed", "day"], ["stop", "email", "complete", "today"], ["write", "valve", "active", "day"], ["notify", "order", "none", "hour"], ["query", "customer", "failed", "today"], ["execute", "order", "normal", "minute"], ["stop", "invoice", "running", "year"], ["execute", "pump", "ready", "hour"], ["execute", "inventory", "open", "tomorrow"], ["check", "file", "failed", "month"], ["query", "file", "ready", "minute"], ["create", "motor", "low", "year"], ["notify", "job", "active", "minute"], ["notify", "pump", "active", "today"], ["start", "invoice", "high", "tomorrow"], ["execute", "meeting", "normal", "year"], ["write", "customer", "low", "year"], ["check", "customer", "normal", "year"], ["delete", "motor", "normal", "year"], ["query", "file", "overload", "minute"], ["notify", "pump", "idle", "now"], ["read", "valve", "open", "tomorrow"], ["check", "motor", "paid", "week"], ["stop", "valve", "ready", "month"], ["check", "motor", "running", "hour"], ["start", "customer", "complete", "day"], ["read", "pump", "pending", "hour"], ["read", "valve", "active", "year"], ["write", "valve", "closed", "none"], ["start", "customer", "ready", "month"], ["check", "motor", "idle", "month"], ["start", "job", "active", "year"], ["update", "file", "high", "today"], ["execute", "order", "active", "now"], ["execute", "invoice", "active", "minute"], ["stop", "inventory", "idle", "week"], ["notify", "inventory", "open", "tomorrow"], ["write", "valve", "closed", "today"], ["create", "order", "closed", "day"], ["update", "inventory", "running", "now"], ["query", "meeting", "active", "minute"], ["write", "machine", "open", "none"], ["execute", "file", "high", "today"], ["create", "order", "active", "year"], ["query", "valve", "failed", "today"], ["write", "valve", "ready", "day"], ["write", "customer", "active", "minute"], ["write", "valve", "open", "year"], ["start", "valve", "none", "month"], ["execute", "valve", "paid", "week"], ["check", "valve", "open", "tomorrow"], ["report", "motor", "normal", "hour"], ["stop", "inventory", "active", "week"], ["notify", "customer", "paid", "hour"], ["execute", "order", "normal", "now"], ["stop", "machine", "blocked", "none"], ["stop", "email", "closed", "week"], ["stop", "inventory", "ready", "now"], ["write", "pump", "normal", "year"], ["stop", "file", "none", "hour"], ["start", "invoice", "closed", "now"], ["update", "invoice", "closed", "day"], ["check", "order", "idle", "tomorrow"], ["delete", "email", "ready", "hour"], ["create", "meeting", "ready", "tomorrow"], ["create", "order", "low", "year"], ["write", "valve", "running", "minute"], ["notify", "pump", "ready", "minute"], ["read", "inventory", "closed", "week"], ["query", "machine", "open", "year"], ["stop", "email", "failed", "year"], ["write", "email", "none", "none"], ["delete", "inventory", "failed", "today"], ["read", "pump", "overload", "hour"], ["update", "email", "blocked", "tomorrow"], ["check", "pump", "ready", "hour"], ["start", "order", "ready", "month"], ["query", "machine", "open", "today"], ["write", "pump", "pending", "day"], ["check", "motor", "none", "day"], ["start", "machine", "overload", "now"], ["delete", "order", "low", "year"], ["start", "meeting", "complete", "hour"], ["start", "job", "closed", "day"], ["execute", "file", "none", "hour"], ["update", "motor", "overload", "month"], ["check", "file", "high", "week"], ["create", "machine", "closed", "day"], ["check", "motor", "idle", "tomorrow"], ["check", "customer", "pending", "week"], ["delete", "machine", "blocked", "day"], ["create", "valve", "pending", "day"], ["execute", "order", "failed", "minute"], ["create", "inventory", "failed", "hour"], ["delete", "customer", "active", "day"], ["read", "invoice", "complete", "tomorrow"], ["query", "machine", "active", "year"], ["read", "meeting", "low", "day"], ["delete", "machine", "running", "none"], ["stop", "inventory", "active", "month"], ["report", "order", "complete", "day"], ["write", "email", "failed", "week"], ["create", "job", "running", "week"], ["delete", "machine", "active", "none"], ["stop", "email", "ready", "now"], ["read", "pump", "idle", "day"], ["update", "order", "paid", "hour"], ["read", "invoice", "open", "month"], ["notify", "motor", "normal", "day"], ["check", "valve", "ready", "hour"], ["read", "invoice", "overload", "week"], ["update", "order", "idle", "none"], ["write", "email", "normal", "week"], ["execute", "order", "high", "minute"], ["read", "pump", "high", "month"], ["report", "pump", "idle", "tomorrow"], ["query", "motor", "blocked", "tomorrow"], ["report", "email", "high", "hour"], ["read", "invoice", "none", "day"], ["write", "email", "pending", "hour"], ["start", "file", "idle", "tomorrow"], ["report", "motor", "ready", "day"], ["stop", "inventory", "failed", "tomorrow"], ["start", "motor", "high", "hour"], ["check", "machine", "blocked", "tomorrow"], ["start", "meeting", "blocked", "none"], ["query", "file", "ready", "week"], ["query", "machine", "closed", "week"], ["notify", "job", "open", "hour"], ["notify", "meeting", "failed", "today"], ["write", "motor", "active", "minute"], ["start", "meeting", "idle", "hour"], ["write", "valve", "closed", "minute"], ["read", "pump", "ready", "now"], ["read", "order", "pending", "day"], ["notify", "machine", "low", "year"], ["create", "valve", "blocked", "none"], ["read", "pump", "open", "hour"], ["start", "meeting", "failed", "now"], ["update", "pump", "pending", "day"], ["query", "motor", "high", "hour"], ["query", "file", "paid", "hour"], ["execute", "valve", "normal", "today"], ["read", "pump", "none", "minute"], ["report", "pump", "complete", "day"], ["check", "pump", "overload", "now"], ["start", "meeting", "none", "month"], ["create", "order", "high", "today"], ["execute", "valve", "open", "year"], ["create", "pump", "open", "none"], ["write", "email", "paid", "week"], ["report", "file", "complete", "year"], ["check", "motor", "idle", "day"], ["notify", "customer", "running", "year"], ["start", "pump", "active", "minute"], ["update", "order", "complete", "year"], ["delete", "machine", "blocked", "year"], ["stop", "pump", "running", "year"], ["check", "valve", "paid", "hour"], ["write", "customer", "open", "none"], ["check", "motor", "overload", "month"], ["check", "motor", "high", "tomorrow"], ["read", "invoice", "low", "month"], ["check", "motor", "none", "tomorrow"], ["update", "inventory", "normal", "month"], ["query", "order", "low", "year"], ["write", "email", "overload", "year"], ["report", "meeting", "blocked", "tomorrow"], ["start", "invoice", "ready", "today"], ["write", "email", "complete", "week"], ["write", "job", "active", "year"], ["read", "pump", "failed", "none"], ["write", "email", "active", "week"], ["start", "motor", "overload", "month"], ["read", "invoice", "normal", "none"], ["report", "customer", "ready", "none"], ["notify", "pump", "paid", "minute"], ["delete", "order", "ready", "hour"], ["read", "pump", "low", "minute"], ["create", "job", "normal", "none"], ["read", "machine", "active", "minute"], ["report", "motor", "open", "today"], ["stop", "inventory", "running", "week"], ["delete", "machine", "idle", "year"], ["execute", "valve", "running", "year"], ["write", "meeting", "overload", "month"], ["notify", "valve", "running", "day"], ["write", "motor", "low", "day"], ["read", "pump", "ready", "today"], ["check", "file", "paid", "tomorrow"], ["write", "machine", "closed", "week"], ["update", "motor", "none", "hour"], ["update", "order", "none", "now"], ["create", "customer", "pending", "week"], ["check", "customer", "open", "none"], ["write", "machine", "failed", "today"], ["create", "job", "closed", "week"], ["query", "file", "pending", "today"], ["check", "valve", "blocked", "tomorrow"], ["stop", "email", "failed", "minute"], ["notify", "job", "open", "tomorrow"], ["execute", "order", "idle", "hour"], ["start", "job", "low", "day"], ["check", "email", "open", "none"], ["start", "job", "pending", "week"], ["update", "inventory", "open", "hour"], ["stop", "order", "paid", "hour"], ["start", "meeting", "idle", "none"], ["write", "valve", "paid", "year"], ["report", "customer", "active", "week"], ["start", "pump", "running", "day"], ["start", "meeting", "normal", "hour"], ["report", "customer", "complete", "week"], ["update", "inventory", "complete", "tomorrow"], ["write", "email", "overload", "day"], ["execute", "valve", "idle", "month"], ["delete", "invoice", "ready", "hour"], ["check", "meeting", "normal", "year"], ["delete", "inventory", "high", "hour"], ["stop", "order", "failed", "hour"], ["write", "file", "paid", "hour"], ["delete", "inventory", "paid", "hour"], ["notify", "pump", "paid", "today"], ["query", "file", "failed", "tomorrow"], ["delete", "valve", "high", "today"], ["read", "motor", "normal", "day"], ["start", "invoice", "low", "now"], ["query", "file", "normal", "minute"], ["stop", "pump", "high", "hour"], ["notify", "order", "none", "month"], ["stop", "inventory", "none", "month"], ["report", "pump", "high", "today"], ["notify", "pump", "paid", "hour"], ["stop", "inventory", "paid", "hour"], ["report", "customer", "none", "year"], ["execute", "inventory", "failed", "today"], ["delete", "machine", "normal", "none"], ["stop", "email", "active", "today"], ["stop", "inventory", "low", "today"], ["stop", "meeting", "complete", "year"], ["create", "customer", "high", "today"], ["stop", "inventory", "complete", "today"], ["read", "pump", "none", "year"], ["report", "motor", "complete", "today"], ["update", "inventory", "paid", "minute"], ["report", "customer", "overload", "hour"], ["delete", "customer", "paid", "minute"], ["notify", "pump", "pending", "now"], ["delete", "file", "ready", "month"], ["stop", "email", "overload", "year"], ["write", "invoice", "complete", "day"], ["delete", "customer", "closed", "month"], ["stop", "invoice", "paid", "hour"], ["check", "file", "failed", "day"], ["query", "file", "normal", "month"], ["query", "file", "blocked", "month"], ["start", "valve", "overload", "month"], ["start", "meeting", "failed", "tomorrow"], ["read", "invoice", "open", "week"], ["execute", "pump", "normal", "year"], ["notify", "pump", "pending", "month"], ["write", "inventory", "ready", "month"], ["report", "motor", "none", "hour"], ["write", "pump", "normal", "day"], ["notify", "pump", "low", "hour"], ["execute", "file", "closed", "week"], ["update", "valve", "normal", "year"], ["stop", "customer", "complete", "day"], ["stop", "inventory", "ready", "hour"], ["start", "email", "overload", "month"], ["read", "invoice", "open", "now"], ["write", "pump", "active", "year"], ["create", "meeting", "running", "year"], ["write", "motor", "pending", "day"], ["notify", "order", "ready", "month"], ["check", "file", "blocked", "day"], ["query", "machine", "paid", "hour"], ["start", "meeting", "low", "none"], ["create", "inventory", "failed", "today"], ["write", "order", "pending", "day"], ["delete", "customer", "running", "now"], ["query", "job", "failed", "hour"], ["report", "motor", "ready", "none"], ["check", "meeting", "running", "day"], ["execute", "invoice", "closed", "day"], ["delete", "valve", "blocked", "tomorrow"], ["execute", "valve", "paid", "now"], ["notify", "pump", "pending", "hour"], ["stop", "inventory", "pending", "now"], ["execute", "email", "complete", "year"], ["query", "file", "paid", "tomorrow"], ["delete", "machine", "high", "year"], ["execute", "motor", "complete", "day"], ["create", "machine", "high", "today"], ["notify", "valve", "failed", "hour"], ["check", "file", "active", "week"], ["delete", "file", "low", "year"], ["report", "customer", "running", "today"], ["write", "valve", "active", "hour"], ["update", "email", "running", "year"], ["write", "pump", "open", "none"], ["report", "motor", "failed", "year"], ["write", "invoice", "low", "day"], ["update", "order", "high", "none"], ["write", "inventory", "none", "hour"], ["write", "email", "pending", "week"], ["create", "job", "low", "minute"], ["read", "invoice", "blocked", "month"], ["create", "customer", "failed", "today"], ["notify", "email", "low", "year"], ["stop", "order", "none", "hour"], ["read", "invoice", "high", "year"], ["start", "inventory", "closed", "week"], ["write", "order", "none", "month"], ["check", "file", "complete", "month"], ["create", "pump", "none", "hour"], ["execute", "motor", "complete", "year"], ["check", "inventory", "complete", "year"], ["report", "motor", "open", "hour"], ["read", "pump", "closed", "year"], ["execute", "inventory", "active", "minute"], ["start", "invoice", "failed", "tomorrow"], ["start", "email", "blocked", "tomorrow"], ["start", "valve", "ready", "month"], ["write", "job", "active", "minute"], ["execute", "valve", "blocked", "now"], ["execute", "motor", "running", "year"], ["start", "pump", "none", "month"], ["create", "meeting", "paid", "hour"], ["update", "motor", "idle", "tomorrow"], ["read", "job", "pending", "week"], ["execute", "order", "pending", "hour"], ["read", "pump", "none", "now"], ["create", "motor", "failed", "today"], ["update", "inventory", "running", "day"], ["start", "pump", "overload", "now"], ["execute", "meeting", "active", "minute"], ["notify", "order", "high", "today"], ["notify", "pump", "high", "week"], ["check", "file", "idle", "year"], ["query", "customer", "running", "day"], ["start", "valve", "high", "today"], ["query", "job", "ready", "month"], ["start", "invoice", "idle", "none"], ["query", "valve", "complete", "year"], ["start", "motor", "ready", "hour"], ["read", "meeting", "blocked", "tomorrow"], ["execute", "order", "open", "year"], ["read", "order", "blocked", "tomorrow"], ["query", "order", "overload", "month"], ["delete", "invoice", "blocked", "none"], ["update", "inventory", "idle", "tomorrow"], ["query", "file", "low", "tomorrow"]], "seed": 20261017, "types": {"Answer": "name", "Archived": "name", "Benin": "name", "Bodies": "name", "Director": "name", "Filipino": "name", "Fischer": "name", "Fred": "name", "Gaussian": "name", "Insect": "name", "Mom": "name", "Prepar": "name", "aspire": "word", "brickbc": "multitoken_word", "calming": "word", "careful": "word", "choice suffrage": "phrase", "comfortablyifera": "multitoken_word", "downloads": "word", "electrolyte benefited": "phrase", "equine": "word", "geospatialard": "multitoken_word", "heroine": "word", "identifierbud": "multitoken_word", "interpolation": "word", "linestyle seen": "phrase", "manners": "word", "purchases nat": "phrase", "remotely": "word", "sarc": "word", "tackle": "word", "tube": "word"}}
|
021/stats021.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json, math, random, collections, sys
|
| 2 |
+
P = collections.defaultdict(dict)
|
| 3 |
+
for f in ('gpu/predictionsA.jsonl', 'gpu/predictionsB.jsonl'):
|
| 4 |
+
import os
|
| 5 |
+
if not os.path.exists(f): continue
|
| 6 |
+
for l in open(f):
|
| 7 |
+
d = json.loads(l); P[d['sys']].setdefault(d['set'], {})[d['i']] = float(d['correct'])
|
| 8 |
+
L = lambda s, st: [P[s][st][i] for i in sorted(P[s][st])]
|
| 9 |
+
def wil(a):
|
| 10 |
+
n = len(a); p = sum(a)/n; z = 1.96; d = 1+z*z/n; c = (p+z*z/(2*n))/d; h = z*math.sqrt(p*(1-p)/n+z*z/(4*n*n))/d; return f"{100*p:.2f} [{100*(c-h):.1f}, {100*(c+h):.1f}]"
|
| 11 |
+
def boot(a, b, seed=0, B=10000):
|
| 12 |
+
r = random.Random(seed); n = len(a); d = [x-y for x, y in zip(a, b)]; bs = sorted(sum(d[r.randrange(n)] for _ in range(n))/n*100 for _ in range(B))
|
| 13 |
+
return f"{sum(d)/n*100:+.2f} [{bs[int(.025*B)]:+.2f}, {bs[int(.975*B)]:+.2f}]"
|
| 14 |
+
mean = lambda names, st: [sum(x)/len(x) for x in zip(*[L(n, st) for n in names])]
|
| 15 |
+
SETS = ['sealed_id', 'sealed_ood', 'sealed_1_new', 'sealed_2_new', 'sealed_4_new']
|
| 16 |
+
groups = {'k2_021': [f'k2_021_s{s}' for s in (17, 29, 43)], 'k2_018': [f'k2_018_s{s}' for s in (17, 29, 43)]}
|
| 17 |
+
for fn in ('values', 'values_csv', 'kv'): groups[f'sp_{fn}'] = [f'sp_{fn}_s{s}' for s in (17, 29, 43)]
|
| 18 |
+
for p in ('ridge_aug', 'mlp_aug', 'mlp_ctrl_shuffled'): groups[p] = [f'{p}_s{s}' for s in (17, 29, 43)]
|
| 19 |
+
singles = ['text_template'] + [k for k in P if k.startswith('fewshot_')]
|
| 20 |
+
for st in SETS:
|
| 21 |
+
print('==', st); T = L('text_template', st)
|
| 22 |
+
for s in singles: print(f' {s}: {wil(L(s, st))} vs template {boot(L(s, st), T)}')
|
| 23 |
+
for g, ns in groups.items():
|
| 24 |
+
if not all(st in P[n] for n in ns): continue
|
| 25 |
+
per = ' / '.join(f"{100*sum(L(n, st))/len(L(n, st)):.1f}" for n in ns); m = mean(ns, st)
|
| 26 |
+
extra = f" vs k2_021 {boot(m, mean(groups['k2_021'], st), 2)}" if g != 'k2_021' else ''
|
| 27 |
+
print(f' {g}: {per} | mean {100*sum(m)/len(m):.2f} vs template {boot(m, T, 1)}{extra}')
|
021/stats021.txt
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
== sealed_id
|
| 2 |
+
text_template: 100.00 [98.1, 100.0] vs template +0.00 [+0.00, +0.00]
|
| 3 |
+
fewshot_values: 34.50 [28.3, 41.3] vs template -65.50 [-72.00, -59.00]
|
| 4 |
+
fewshot_values_csv: 89.00 [83.9, 92.6] vs template -11.00 [-15.50, -7.00]
|
| 5 |
+
fewshot_kv: 94.50 [90.4, 96.9] vs template -5.50 [-9.00, -2.50]
|
| 6 |
+
fewshot_template: 90.00 [85.1, 93.4] vs template -10.00 [-14.50, -6.00]
|
| 7 |
+
k2_021: 100.0 / 99.0 / 99.5 | mean 99.50 vs template -0.50 [-1.17, +0.00]
|
| 8 |
+
k2_018: 100.0 / 99.5 / 97.5 | mean 99.00 vs template -1.00 [-1.83, -0.33] vs k2_021 -0.50 [-1.50, +0.50]
|
| 9 |
+
sp_values: 100.0 / 100.0 / 100.0 | mean 100.00 vs template +0.00 [+0.00, +0.00] vs k2_021 +0.50 [+0.00, +1.17]
|
| 10 |
+
sp_values_csv: 100.0 / 100.0 / 100.0 | mean 100.00 vs template +0.00 [+0.00, +0.00] vs k2_021 +0.50 [+0.00, +1.17]
|
| 11 |
+
sp_kv: 100.0 / 100.0 / 100.0 | mean 100.00 vs template +0.00 [+0.00, +0.00] vs k2_021 +0.50 [+0.00, +1.17]
|
| 12 |
+
== sealed_ood
|
| 13 |
+
text_template: 98.50 [96.8, 99.3] vs template +0.00 [+0.00, +0.00]
|
| 14 |
+
fewshot_values: 32.50 [28.1, 37.2] vs template -66.00 [-70.75, -61.25]
|
| 15 |
+
fewshot_values_csv: 90.75 [87.5, 93.2] vs template -7.75 [-11.00, -4.75]
|
| 16 |
+
fewshot_kv: 91.75 [88.6, 94.1] vs template -6.75 [-9.75, -3.75]
|
| 17 |
+
fewshot_template: 86.50 [82.8, 89.5] vs template -12.00 [-15.50, -8.75]
|
| 18 |
+
k2_021: 99.8 / 98.5 / 97.8 | mean 98.67 vs template +0.17 [-1.25, +1.67]
|
| 19 |
+
k2_018: 99.0 / 99.0 / 97.8 | mean 98.58 vs template +0.08 [-1.17, +1.42] vs k2_021 -0.08 [-1.08, +0.92]
|
| 20 |
+
sp_values: 100.0 / 100.0 / 100.0 | mean 100.00 vs template +1.50 [+0.50, +2.75] vs k2_021 +1.33 [+0.58, +2.17]
|
| 21 |
+
sp_values_csv: 100.0 / 100.0 / 100.0 | mean 100.00 vs template +1.50 [+0.50, +2.75] vs k2_021 +1.33 [+0.58, +2.17]
|
| 22 |
+
sp_kv: 100.0 / 100.0 / 100.0 | mean 100.00 vs template +1.50 [+0.50, +2.75] vs k2_021 +1.33 [+0.58, +2.17]
|
| 23 |
+
ridge_aug: 100.0 / 100.0 / 100.0 | mean 100.00 vs template +1.50 [+0.50, +2.75] vs k2_021 +1.33 [+0.58, +2.17]
|
| 24 |
+
mlp_aug: 100.0 / 100.0 / 100.0 | mean 100.00 vs template +1.50 [+0.50, +2.75] vs k2_021 +1.33 [+0.58, +2.17]
|
| 25 |
+
mlp_ctrl_shuffled: 0.0 / 0.0 / 0.0 | mean 0.00 vs template -98.50 [-99.50, -97.25] vs k2_021 -98.67 [-99.42, -97.83]
|
| 26 |
+
== sealed_1_new
|
| 27 |
+
text_template: 96.88 [93.4, 98.6] vs template +0.00 [+0.00, +0.00]
|
| 28 |
+
fewshot_values: 3.12 [1.4, 6.6] vs template -93.75 [-96.88, -90.10]
|
| 29 |
+
fewshot_values_csv: 50.00 [43.0, 57.0] vs template -46.88 [-54.17, -39.58]
|
| 30 |
+
fewshot_kv: 84.38 [78.6, 88.8] vs template -12.50 [-17.71, -7.81]
|
| 31 |
+
fewshot_template: 66.67 [59.7, 73.0] vs template -30.21 [-36.98, -23.44]
|
| 32 |
+
k2_021: 90.6 / 88.5 / 66.1 | mean 81.77 vs template -15.10 [-19.97, -10.24]
|
| 33 |
+
k2_018: 87.5 / 87.5 / 75.5 | mean 83.51 vs template -13.37 [-18.06, -9.03] vs k2_021 +1.74 [-1.39, +4.86]
|
| 34 |
+
sp_values: 100.0 / 100.0 / 100.0 | mean 100.00 vs template +3.12 [+1.04, +5.73] vs k2_021 +18.23 [+14.41, +22.40]
|
| 35 |
+
sp_values_csv: 100.0 / 100.0 / 100.0 | mean 100.00 vs template +3.12 [+1.04, +5.73] vs k2_021 +18.23 [+14.41, +22.40]
|
| 36 |
+
sp_kv: 100.0 / 100.0 / 100.0 | mean 100.00 vs template +3.12 [+1.04, +5.73] vs k2_021 +18.23 [+14.41, +22.40]
|
| 37 |
+
ridge_aug: 89.1 / 91.1 / 88.0 | mean 89.41 vs template -7.47 [-11.98, -2.78] vs k2_021 +7.64 [+1.74, +13.54]
|
| 38 |
+
mlp_aug: 100.0 / 100.0 / 100.0 | mean 100.00 vs template +3.12 [+1.04, +5.73] vs k2_021 +18.23 [+14.41, +22.40]
|
| 39 |
+
mlp_ctrl_shuffled: 0.0 / 0.0 / 0.0 | mean 0.00 vs template -96.88 [-98.96, -94.27] vs k2_021 -81.77 [-85.59, -77.60]
|
| 40 |
+
== sealed_2_new
|
| 41 |
+
text_template: 97.40 [94.0, 98.9] vs template +0.00 [+0.00, +0.00]
|
| 42 |
+
fewshot_values: 0.00 [0.0, 2.0] vs template -97.40 [-99.48, -94.79]
|
| 43 |
+
fewshot_values_csv: 24.48 [18.9, 31.0] vs template -72.92 [-79.17, -66.67]
|
| 44 |
+
fewshot_kv: 86.46 [80.9, 90.6] vs template -10.94 [-16.15, -5.73]
|
| 45 |
+
fewshot_template: 58.33 [51.3, 65.1] vs template -39.06 [-45.83, -32.29]
|
| 46 |
+
k2_021: 63.0 / 56.8 / 38.0 | mean 52.60 vs template -44.79 [-50.87, -38.89]
|
| 47 |
+
k2_018: 62.5 / 57.8 / 45.8 | mean 55.38 vs template -42.01 [-48.26, -35.94] vs k2_021 +2.78 [-0.87, +6.42]
|
| 48 |
+
sp_values: 98.4 / 99.0 / 98.4 | mean 98.61 vs template +1.22 [-1.22, +3.99] vs k2_021 +46.01 [+40.28, +51.74]
|
| 49 |
+
sp_values_csv: 100.0 / 100.0 / 100.0 | mean 100.00 vs template +2.60 [+0.52, +5.21] vs k2_021 +47.40 [+41.67, +53.12]
|
| 50 |
+
sp_kv: 100.0 / 100.0 / 100.0 | mean 100.00 vs template +2.60 [+0.52, +5.21] vs k2_021 +47.40 [+41.67, +53.12]
|
| 51 |
+
ridge_aug: 71.4 / 76.0 / 70.8 | mean 72.74 vs template -24.65 [-30.90, -18.58] vs k2_021 +20.14 [+11.98, +28.30]
|
| 52 |
+
mlp_aug: 98.4 / 98.4 / 96.4 | mean 97.74 vs template +0.35 [-2.08, +3.12] vs k2_021 +45.14 [+39.41, +50.69]
|
| 53 |
+
mlp_ctrl_shuffled: 0.0 / 0.0 / 0.0 | mean 0.00 vs template -97.40 [-99.48, -94.79] vs k2_021 -52.60 [-58.33, -46.88]
|
| 54 |
+
== sealed_4_new
|
| 55 |
+
text_template: 98.44 [95.5, 99.5] vs template +0.00 [+0.00, +0.00]
|
| 56 |
+
fewshot_values: 0.00 [0.0, 2.0] vs template -98.44 [-100.00, -96.35]
|
| 57 |
+
fewshot_values_csv: 4.69 [2.5, 8.7] vs template -93.75 [-96.88, -90.10]
|
| 58 |
+
fewshot_kv: 89.58 [84.5, 93.2] vs template -8.85 [-13.54, -4.69]
|
| 59 |
+
fewshot_template: 38.54 [31.9, 45.6] vs template -59.90 [-66.67, -53.12]
|
| 60 |
+
k2_021: 31.2 / 30.2 / 26.6 | mean 29.34 vs template -69.10 [-75.00, -63.02]
|
| 61 |
+
k2_018: 28.6 / 26.6 / 25.0 | mean 26.74 vs template -71.70 [-77.26, -65.97] vs k2_021 -2.60 [-6.25, +1.04]
|
| 62 |
+
sp_values: 96.9 / 96.4 / 93.2 | mean 95.49 vs template -2.95 [-5.56, -0.52] vs k2_021 +66.15 [+59.90, +72.05]
|
| 63 |
+
sp_values_csv: 100.0 / 100.0 / 100.0 | mean 100.00 vs template +1.56 [+0.00, +3.65] vs k2_021 +70.66 [+64.93, +76.22]
|
| 64 |
+
sp_kv: 100.0 / 100.0 / 100.0 | mean 100.00 vs template +1.56 [+0.00, +3.65] vs k2_021 +70.66 [+64.93, +76.22]
|
| 65 |
+
ridge_aug: 43.2 / 42.2 / 47.9 | mean 44.44 vs template -53.99 [-60.24, -47.57] vs k2_021 +15.10 [+6.77, +23.44]
|
| 66 |
+
mlp_aug: 24.5 / 35.9 / 32.3 | mean 30.90 vs template -67.53 [-72.57, -62.15] vs k2_021 +1.56 [-5.73, +8.85]
|
| 67 |
+
mlp_ctrl_shuffled: 0.0 / 0.0 / 0.0 | mean 0.00 vs template -98.44 [-100.00, -96.35] vs k2_021 -29.34 [-35.07, -23.78]
|
CITATION.cff
CHANGED
|
@@ -1,38 +1,42 @@
|
|
| 1 |
cff-version: 1.2.0
|
| 2 |
message: "If you use this work, please cite it as below."
|
| 3 |
-
title: "
|
| 4 |
type: software
|
| 5 |
authors:
|
| 6 |
- family-names: "Sanchez"
|
| 7 |
given-names: "Erik"
|
| 8 |
affiliation: "Independent researcher"
|
| 9 |
email: "sanchezerik836@gmail.com"
|
| 10 |
-
version: "2026-10-16"
|
| 11 |
license:
|
| 12 |
- MIT
|
| 13 |
- CC-BY-4.0
|
| 14 |
repository-artifact: "https://huggingface.co/Eriksanchezolvera/cuerdas"
|
| 15 |
url: "https://huggingface.co/Eriksanchezolvera/cuerdas"
|
| 16 |
abstract: >-
|
| 17 |
-
Technical report
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
|
|
|
|
|
|
|
|
|
| 26 |
keywords:
|
| 27 |
- soft prompts
|
| 28 |
- context compression
|
|
|
|
| 29 |
- frozen language model
|
| 30 |
- knowledge distillation
|
| 31 |
- SmolLM2
|
| 32 |
repository-code: "https://huggingface.co/Eriksanchezolvera/cuerdas"
|
| 33 |
preferred-citation:
|
| 34 |
type: report
|
| 35 |
-
title: "
|
| 36 |
authors:
|
| 37 |
- family-names: "Sanchez"
|
| 38 |
given-names: "Erik"
|
|
@@ -41,7 +45,7 @@ preferred-citation:
|
|
| 41 |
year: 2026
|
| 42 |
month: 10
|
| 43 |
status: preprint
|
| 44 |
-
notes: "Technical Report. Not peer reviewed. DOI pending."
|
| 45 |
url: "https://huggingface.co/Eriksanchezolvera/cuerdas"
|
| 46 |
license: CC-BY-4.0
|
| 47 |
references:
|
|
|
|
| 1 |
cff-version: 1.2.0
|
| 2 |
message: "If you use this work, please cite it as below."
|
| 3 |
+
title: "Semantic Strands: A Pre-registered Negative Result on Learned Soft-Slot Context Compression — Trained Compact Text Wins"
|
| 4 |
type: software
|
| 5 |
authors:
|
| 6 |
- family-names: "Sanchez"
|
| 7 |
given-names: "Erik"
|
| 8 |
affiliation: "Independent researcher"
|
| 9 |
email: "sanchezerik836@gmail.com"
|
| 10 |
+
version: "2026-10-16-rev5"
|
| 11 |
license:
|
| 12 |
- MIT
|
| 13 |
- CC-BY-4.0
|
| 14 |
repository-artifact: "https://huggingface.co/Eriksanchezolvera/cuerdas"
|
| 15 |
url: "https://huggingface.co/Eriksanchezolvera/cuerdas"
|
| 16 |
abstract: >-
|
| 17 |
+
Technical report (revision 5) presenting a pre-registered negative result.
|
| 18 |
+
A 3.84M-parameter encoder maps four-field structured records to K=2 soft
|
| 19 |
+
input slots for a frozen SmolLM2-135M-Instruct (17 vs 35 positions per
|
| 20 |
+
event against a verbal template). On unseen combinations of known words it
|
| 21 |
+
matches the template on two hash-sealed tests evaluated once (99.0 vs
|
| 22 |
+
99.25%; 98.7 vs 98.5%). It fails on unseen values (29% vs 98% with four
|
| 23 |
+
unseen names, phrases or multi-token values), a larger training pool does
|
| 24 |
+
not help, and compact text with a matched trained soft prompt (3.84M
|
| 25 |
+
parameters) scores 100% at 22 (+8 shared) positions. Probes read the slots
|
| 26 |
+
better than the frozen LLM. Recommendation: compare soft-slot encoders
|
| 27 |
+
against compact text with a matched trainable budget before claiming
|
| 28 |
+
compression.
|
| 29 |
keywords:
|
| 30 |
- soft prompts
|
| 31 |
- context compression
|
| 32 |
+
- prompt compression
|
| 33 |
- frozen language model
|
| 34 |
- knowledge distillation
|
| 35 |
- SmolLM2
|
| 36 |
repository-code: "https://huggingface.co/Eriksanchezolvera/cuerdas"
|
| 37 |
preferred-citation:
|
| 38 |
type: report
|
| 39 |
+
title: "Semantic Strands: A Pre-registered Negative Result on Learned Soft-Slot Context Compression — Trained Compact Text Wins"
|
| 40 |
authors:
|
| 41 |
- family-names: "Sanchez"
|
| 42 |
given-names: "Erik"
|
|
|
|
| 45 |
year: 2026
|
| 46 |
month: 10
|
| 47 |
status: preprint
|
| 48 |
+
notes: "Technical Report, revision 5. Not peer reviewed. DOI pending."
|
| 49 |
url: "https://huggingface.co/Eriksanchezolvera/cuerdas"
|
| 50 |
license: CC-BY-4.0
|
| 51 |
references:
|
README.md
CHANGED
|
@@ -1,74 +1,134 @@
|
|
| 1 |
---
|
| 2 |
license: mit
|
|
|
|
|
|
|
|
|
|
| 3 |
base_model: HuggingFaceTB/SmolLM2-135M-Instruct
|
| 4 |
-
tags:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
---
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
|
| 36 |
-
|
|
| 37 |
-
|
|
| 38 |
-
|
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
##
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
license: mit
|
| 3 |
+
language:
|
| 4 |
+
- en
|
| 5 |
+
- es
|
| 6 |
base_model: HuggingFaceTB/SmolLM2-135M-Instruct
|
| 7 |
+
tags:
|
| 8 |
+
- soft-prompt
|
| 9 |
+
- context-compression
|
| 10 |
+
- prompt-compression
|
| 11 |
+
- negative-result
|
| 12 |
+
- pre-registered
|
| 13 |
+
- research
|
| 14 |
---
|
| 15 |
+
|
| 16 |
+
# Semantic Strands ("Cuerdas"): a pre-registered negative result on learned soft-slot context compression
|
| 17 |
+
|
| 18 |
+
**Technical Report, Revision 5 (16 October 2026). Not peer reviewed.**
|
| 19 |
+
Erik Sanchez, independent researcher · sanchezerik836@gmail.com
|
| 20 |
+
|
| 21 |
+
> **ES.** Les presento los resultados de mi primer reto personal en ciencia computacional. En este punto mi conocimiento todavía es nulo, pero lo compenso con una gran curiosidad. Espero que les sea de utilidad.
|
| 22 |
+
>
|
| 23 |
+
> **EN.** These are the results of my first personal challenge in computer science. At this point my knowledge is still close to zero, but I make up for it with a lot of curiosity. I hope you find it useful.
|
| 24 |
+
|
| 25 |
+
"Cuerdas" (Spanish for "strings") is the internal project name. It has nothing to do with string theory; the English title is *Semantic Strands*.
|
| 26 |
+
|
| 27 |
+
**Bottom line: this approach did not pay off.** A competent compact-text baseline with the same trainable budget beats it. The repository is published so that others can skip this path and reuse the controls that exposed the problem.
|
| 28 |
+
|
| 29 |
+
- Full report: [`paper/cuerdas-technical-report.pdf`](paper/cuerdas-technical-report.pdf) (Markdown source: [`paper/cuerdas-technical-report.md`](paper/cuerdas-technical-report.md))
|
| 30 |
+
- Citation metadata: [`CITATION.cff`](CITATION.cff)
|
| 31 |
+
|
| 32 |
+
> Note: README files from earlier revisions (1–4) presented the method as a success. That framing is withdrawn. Revision 5 supersedes it.
|
| 33 |
+
|
| 34 |
+
## What was tried
|
| 35 |
+
|
| 36 |
+
We trained a small encoder ("PackK", K=2, 3,842,952 trainable parameters) to map four-field structured records (action, target, status, window) to **2 learned input vectors ("soft slots")** read by a **frozen** SmolLM2-135M-Instruct (revision `12fd25f77366fa6b3b4b768ec3050bf629380bac`; no weights updated, no LoRA). The goal was to replace a 20-token verbal template and so shorten the sequence from **35 to 17 positions per event** (counting a shared query prefix and four query tails).
|
| 37 |
+
|
| 38 |
+
Accuracy is closed-set, first-token, exact match over all four fields. Nothing is generated.
|
| 39 |
+
|
| 40 |
+
## What held
|
| 41 |
+
|
| 42 |
+
On unseen combinations of **known** words, the slots match the verbal template on two pre-registered, hash-sealed tests evaluated once (seed means, n=400, 3 seeds):
|
| 43 |
+
|
| 44 |
+
| Sealed test | Verbal template (35 pos.) | K=2 slots (17 pos.) |
|
| 45 |
+
|---|---:|---:|
|
| 46 |
+
| Iteration 020 | 99.25% | 99.0% |
|
| 47 |
+
| Iteration 021 | 98.5% | 98.7% |
|
| 48 |
+
|
| 49 |
+
## What failed
|
| 50 |
+
|
| 51 |
+
1. **Unseen values.** On the 021 sealed test (unseen words, names, phrases and multi-token values), accuracy with four unseen values falls to **29%**, against **98%** for the template. Retraining with a much larger, more varied replacement pool did not help (+2.6 pp, interval includes zero).
|
| 52 |
+
2. **A fair compact-text baseline wins.** With a matched trainable budget (a reparameterized soft prompt of 3,842,368 parameters; same data, steps, loss, teacher and validation-only selection), **values separated by commas score 100% on every sealed set**, including four unseen values, at 22 positions per event plus 8 shareable prompt positions. Against this baseline the slots save only 2–5 positions per event and lose up to 70.7 pp (four unseen values).
|
| 53 |
+
3. **The receiver is the bottleneck.** Probes trained with replacement words read unseen values from the slots better than the frozen LLM does (for example 97.7% vs 52.6% with two unseen values). The information is in the slots; the receiver does not read it.
|
| 54 |
+
|
| 55 |
+
Zero-shot compact text fails with this receiver (0–8%), which is why comparing only against zero-shot or verbose text overstates the benefit of soft slots.
|
| 56 |
+
|
| 57 |
+
Other documented pitfalls: an inflated latency figure from a non-shared prefix (017, corrected in 018), weak probes that made the slots look non-decodable, and a re-saved `config.json` that silently changed the RoPE base under transformers 4.46.3 (see `019/resultados-019.md` and Section 7.6 of the report).
|
| 58 |
+
|
| 59 |
+
## Recommendations for others
|
| 60 |
+
|
| 61 |
+
Before claiming compression from learned soft slots or soft tokens:
|
| 62 |
+
|
| 63 |
+
1. Compare against **compact text with a matched trainable budget** (soft prompt, prefix or LoRA of equal size, same data, steps, loss and selection). Zero-shot or verbose text is not a fair comparison.
|
| 64 |
+
2. Count positions honestly, including shared prefixes, query tails and soft-prompt positions.
|
| 65 |
+
3. Seal the test: pre-register metrics and decision rules, generate the test with a new seed, hash it and evaluate once.
|
| 66 |
+
4. Test unseen values of several kinds (words, names, phrases, multi-token values) and report accuracy by the number of unseen fields.
|
| 67 |
+
5. Train probes on the same augmented distribution as the encoder, with shuffled-target controls.
|
| 68 |
+
6. Keep a random-weight receiver control, and report per-example predictions with paired bootstrap intervals.
|
| 69 |
+
7. Measure latency with a shared KV prefix on every route.
|
| 70 |
+
8. Pin model revisions and configuration files.
|
| 71 |
+
|
| 72 |
+
Scope: one synthetic, closed-vocabulary, four-field task, one 135M receiver, English only. No claim is made beyond it.
|
| 73 |
+
|
| 74 |
+
## Repository contents
|
| 75 |
+
|
| 76 |
+
| Path | What it is |
|
| 77 |
+
|---|---|
|
| 78 |
+
| `013/` | Iteration 013 (CPU, SmolLM2-135M base, smaller universe): turned out to be a lookup table |
|
| 79 |
+
| `014-auditoria/` | Audit of iteration 014 (run by ChatGPT) |
|
| 80 |
+
| `015/` … `018/` | Training scripts (`run01X.py`), exact Kaggle kernels, raw logs (`results01X.jsonl`) and result notes |
|
| 81 |
+
| `019/` | Re-evaluation of the 018 checkpoints: per-example predictions, paired bootstrap, random-weight receiver, RoPE-config diagnosis |
|
| 82 |
+
| `020/` | Pre-registration, sealed test (`sealed020.json`, SHA-256 `4c7737f6…3393`), compact-text baselines and stronger probes |
|
| 83 |
+
| `021/` | Pre-registration, second sealed test (`sealed021.json`, SHA-256 `a86a6a5e…72bc`), retraining with a varied pool (`c021.py`, `evalA.py`) and matched soft-prompt baselines (`evalB.py`), logs, predictions and statistics |
|
| 84 |
+
| `checkpoints/` | `binder4_1000_s*.pt` (015 teacher), `k1_s*.pt` (017 K=1), `k2_s*.pt` (017 K=2, start point for 018 and 021), `k2long_s*.pt` (018 K=2), `k2v_s*.pt` (021 K=2 retrained), `sp_{values,values_csv,kv}_s*.pt` (021 soft-prompt baselines) |
|
| 85 |
+
| `make_dataset.py` | Regenerates the 014 dataset (standard library only) |
|
| 86 |
+
| `paper/` | Technical report, revision 5 (PDF, Markdown, Figure 1) |
|
| 87 |
+
|
| 88 |
+
The checkpoints are small encoder state dicts, **not** standalone language models. They are loaded with the `Binder` / `PackK` classes in `018/run018.py` (and the soft-prompt class in `021/evalB.py`), together with the frozen receiver. Seeds: 17, 29, 43.
|
| 89 |
+
|
| 90 |
+
## How to reproduce
|
| 91 |
+
|
| 92 |
+
1. Python 3.11+, PyTorch and `transformers==4.46.3`. The 019 re-evaluation on a Kaggle Tesla T4 recorded PyTorch 2.11.0+cu128, CUDA 12.8, Python 3.13.15, and reproduced every 018 accuracy exactly.
|
| 93 |
+
2. Regenerate the dataset: `python make_dataset.py --out $SRC/data/expanded.json` (verify with `--check`). The original `expanded.json`, generated in iteration 014 with ChatGPT, is **not redistributed**; the script produces the same tuples and split indices.
|
| 94 |
+
3. Download the receiver at the pinned revision into `$MP`. Do **not** load a copy re-saved by transformers ≥ 5 with transformers 4.46.3: the RoPE base silently falls back to 10000.
|
| 95 |
+
4. Training: `SRC=... MP=... OD=out python 015/run015.py base binder4:1000`, then `016/run016.py`, `017/run017.py` and `018/run018.py` with `INP=checkpoints`.
|
| 96 |
+
5. Evaluations without training: `019/eval019.py` and `020/eval020.py` (`INP` must contain the checkpoints, `run018.py` and the sealed file). Iteration 021: `021/evalA.py` (retraining, needs `c021.py`) and `021/evalB.py` (soft-prompt baselines). The `kaggle_kernel*.py` files show exactly how each was run on a Kaggle T4.
|
| 97 |
+
|
| 98 |
+
Compute: about 2.6 Kaggle T4 GPU-hours for 015–020 (free tier; Section 4.7 of the report), plus CPU runs for 013–015.
|
| 99 |
+
|
| 100 |
+
## Use of AI tools
|
| 101 |
+
|
| 102 |
+
AI systems are not authors. Grok Bot (AI assistant) helped design the experiments, wrote and ran most of the code and drafted the report. ChatGPT ran iteration 014, generated its dataset and reviewed earlier revisions. The author set the goals and acceptance criteria, chose what to run next and approved the claims. Details: Section 11 of the report.
|
| 103 |
+
|
| 104 |
+
## License
|
| 105 |
+
|
| 106 |
+
- **Code** (scripts, kernels, `make_dataset.py`) and **encoder checkpoints**: MIT (see `LICENSE`).
|
| 107 |
+
- **Report text** (including Figure 1): CC BY 4.0.
|
| 108 |
+
- SmolLM2-135M-Instruct is not included and keeps its Apache-2.0 license. The original ChatGPT-generated 014 dataset is not redistributed.
|
| 109 |
+
|
| 110 |
+
## How to cite
|
| 111 |
+
|
| 112 |
+
> Sanchez, E. (2026). *Semantic Strands: A Pre-registered Negative Result on Learned Soft-Slot Context Compression — Trained Compact Text Wins.* Technical Report, revision 5. https://huggingface.co/Eriksanchezolvera/cuerdas
|
| 113 |
+
|
| 114 |
+
DOI: pending
|
| 115 |
+
|
| 116 |
+
```bibtex
|
| 117 |
+
@techreport{sanchez2026cuerdas,
|
| 118 |
+
author = {Sanchez, Erik},
|
| 119 |
+
title = {Semantic Strands: A Pre-registered Negative Result on
|
| 120 |
+
Learned Soft-Slot Context Compression --- Trained
|
| 121 |
+
Compact Text Wins},
|
| 122 |
+
institution = {Independent researcher},
|
| 123 |
+
type = {Technical Report},
|
| 124 |
+
number = {Revision 5},
|
| 125 |
+
year = {2026},
|
| 126 |
+
month = oct,
|
| 127 |
+
url = {https://huggingface.co/Eriksanchezolvera/cuerdas},
|
| 128 |
+
note = {DOI pending}
|
| 129 |
+
}
|
| 130 |
+
```
|
| 131 |
+
|
| 132 |
+
## Contact
|
| 133 |
+
|
| 134 |
+
Erik Sanchez (independent researcher) · sanchezerik836@gmail.com
|
checkpoints/k2_s17.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b7deb5aead22c265b1fd11bba19c00768e1f5d1d0d63faa92f0555fc8883c659
|
| 3 |
+
size 15375601
|
checkpoints/k2_s29.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a06de687aa3de8e5252f58f6b403119be2810b0b157a1c85541b70e55496566e
|
| 3 |
+
size 15375601
|
checkpoints/k2_s43.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:19f665e0d16688e253b7a1c98c48803fd6cd23babc10f02f39586e3b98676418
|
| 3 |
+
size 15375601
|
checkpoints/k2v_s17.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e9666602831ae1954dfa26871ec6918670f5171f0a340d5f95168d9da4192788
|
| 3 |
+
size 15375746
|
checkpoints/k2v_s29.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6fe4e70e747e29f3f7f2b57f3766489eecc5c74d03d2010b8a1402b647faa5b2
|
| 3 |
+
size 15375746
|
checkpoints/k2v_s43.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e9df3ebcfaeea902c3d05e4c8b736d366b4cf260b5773e4ed31d08d586e76e0c
|
| 3 |
+
size 15375746
|
checkpoints/sp_kv_s17.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a64f53e9416380b0064e4b2b79446ff48027867c4b2902f570358099366857d6
|
| 3 |
+
size 15372335
|
checkpoints/sp_kv_s29.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:22cbed2f6f231d2c6f17d94d81ce69a8ffd6234ec9f20e895cef136c0155e82e
|
| 3 |
+
size 15372335
|
checkpoints/sp_kv_s43.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3568c35b04a42bdb2f80838137987674e9fe69526db090b5e5047ed178b5b2b1
|
| 3 |
+
size 15372335
|
checkpoints/sp_values_csv_s17.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:99e8b81d05f974f38cc464c4c72c3ae2ed6941b61be77c2a0b31428e3ee0aaf6
|
| 3 |
+
size 15372423
|
checkpoints/sp_values_csv_s29.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9a7e69db920cc50f1b8f17f1afab805ba3e85b645f0348718b4dbbf5722f3b7e
|
| 3 |
+
size 15372423
|
checkpoints/sp_values_csv_s43.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:13979deb620a6836805c017c1263016a85a857ff420dbbd0e34d51cc7a6d01e0
|
| 3 |
+
size 15372423
|
checkpoints/sp_values_s17.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b57f329cef963875aaceaeab5c4492baec2a136c98751cb4eb5145cd73f6ab8b
|
| 3 |
+
size 15372379
|
checkpoints/sp_values_s29.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6f4b9b8ee1a766a9756446063f77fa63e28ce54de7606073997195e68f80d561
|
| 3 |
+
size 15372379
|
checkpoints/sp_values_s43.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0db03dbc657a69071a20edf34c5595678e5cb6df7ce509646ec2a4f548dfcd5d
|
| 3 |
+
size 15372379
|
paper/cuerdas-technical-report.md
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
paper/cuerdas-technical-report.pdf
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c2ac543fc4f49adf33d90c598aa1f0332bcf75e7070f462a98c241a68e217797
|
| 3 |
+
size 181206
|
paper/fig_architecture.svg
ADDED
|
|