Update Qwen2.5-3B-Instruct PersonaMem-128K reader from validation-selected optimization
adbf177 verified | { | |
| "status": "passed", | |
| "bundles": [ | |
| { | |
| "bundle": "personamem-32k/qwen3-4b", | |
| "strict_component_load": true, | |
| "synthetic_forward_shape": [ | |
| 1, | |
| 256, | |
| 2560 | |
| ], | |
| "peft_config_load": true | |
| }, | |
| { | |
| "bundle": "personamem-32k/qwen2.5-3b", | |
| "strict_component_load": true, | |
| "synthetic_forward_shape": [ | |
| 1, | |
| 256, | |
| 2048 | |
| ], | |
| "peft_config_load": true | |
| }, | |
| { | |
| "bundle": "personamem-32k/qwen2.5-7b", | |
| "strict_component_load": true, | |
| "synthetic_forward_shape": [ | |
| 1, | |
| 256, | |
| 3584 | |
| ], | |
| "peft_config_load": true | |
| }, | |
| { | |
| "bundle": "personamem-128k/qwen3-4b", | |
| "strict_component_load": true, | |
| "synthetic_forward_shape": [ | |
| 1, | |
| 256, | |
| 2560 | |
| ], | |
| "peft_config_load": true | |
| }, | |
| { | |
| "bundle": "personamem-128k/qwen2.5-3b", | |
| "strict_component_load": true, | |
| "synthetic_forward_shape": [ | |
| 1, | |
| 256, | |
| 2048 | |
| ], | |
| "peft_config_load": true | |
| }, | |
| { | |
| "bundle": "personamem-128k/qwen2.5-7b", | |
| "strict_component_load": true, | |
| "synthetic_forward_shape": [ | |
| 1, | |
| 256, | |
| 3584 | |
| ], | |
| "peft_config_load": true | |
| }, | |
| { | |
| "bundle": "locomo/qwen3-4b", | |
| "strict_component_load": true, | |
| "synthetic_forward_shape": [ | |
| 1, | |
| 512, | |
| 2560 | |
| ], | |
| "peft_config_load": true | |
| }, | |
| { | |
| "bundle": "locomo/qwen2.5-3b", | |
| "strict_component_load": true, | |
| "synthetic_forward_shape": [ | |
| 1, | |
| 512, | |
| 2048 | |
| ], | |
| "peft_config_load": true | |
| }, | |
| { | |
| "bundle": "locomo/qwen2.5-7b", | |
| "strict_component_load": true, | |
| "synthetic_forward_shape": [ | |
| 1, | |
| 512, | |
| 3584 | |
| ], | |
| "peft_config_load": true | |
| } | |
| ], | |
| "public_payload_scan": "passed", | |
| "artifact_hashes": "passed", | |
| "scope": "Original release checks are retained as historical records. optimized_default describes the currently selected PersonaMem-32K Qwen3-4B reader; other bundles are unchanged.", | |
| "code_revision": "d049fd293e3232358594cba7cdae0f7a138bb58d", | |
| "gpu_validation": { | |
| "date": "2026-09-29", | |
| "dataset": "PersonaMem-32K", | |
| "backbone": "Qwen/Qwen3-4B", | |
| "examples": 50, | |
| "trials": 5, | |
| "decoding": "greedy with deterministic answer-option permutations", | |
| "code_revision": "527d5748402460246453e457ab050d6add91c50a", | |
| "weight_revision": "a5548705e97e137942268e92c1209fb8517bc316", | |
| "setup": "Fresh public code clone and public HF reader/compressor download; existing local base model, benchmark inputs, and Python environment.", | |
| "end_to_end": { | |
| "accuracy_by_trial": [ | |
| 0.72, | |
| 0.76, | |
| 0.62, | |
| 0.68, | |
| 0.74 | |
| ], | |
| "mean_accuracy": 0.704, | |
| "best_accuracy": 0.76, | |
| "memory": "newly generated and encoded" | |
| }, | |
| "historical_memory_control": { | |
| "accuracy_by_trial": [ | |
| 0.8, | |
| 0.78, | |
| 0.64, | |
| 0.76, | |
| 0.76 | |
| ], | |
| "mean_accuracy": 0.748, | |
| "best_accuracy": 0.8, | |
| "memory": "historical memory; encoder states recomputed", | |
| "prediction_matches": 250, | |
| "predictions": 250 | |
| }, | |
| "limitation": "The end-to-end run does not reproduce the webpage best accuracy of 84%. The exact reason for regenerated-memory differences has not been established." | |
| }, | |
| "simplified_code_regression": { | |
| "code_revision": "d049fd293e3232358594cba7cdae0f7a138bb58d", | |
| "same_fixed_memories_and_cache": true, | |
| "identical_predictions": 250, | |
| "predictions": 250, | |
| "cpu_tests_passed": 19, | |
| "cli_help_checks_passed": 18, | |
| "memory_component_bundles_loaded": 9, | |
| "scope": "GPU reader regression on Qwen3-4B PersonaMem-32K; component loading for all nine bundles, not GPU evaluation for all bundles." | |
| }, | |
| "optimized_default": { | |
| "scope": "Validation-selected optimization candidate; other bundles are unchanged from base_release.", | |
| "base_release": "d39842eeca8132288c5e2814fdd66733fa92fc3d", | |
| "bundle": "personamem-32k/qwen3-4b", | |
| "adapter_finite": true, | |
| "adapter_config_load": true, | |
| "frozen_compressor_tensors_match": true, | |
| "gpu_test": { | |
| "name": "selected", | |
| "trials": [ | |
| { | |
| "trial": 0, | |
| "correct": 41, | |
| "total": 50, | |
| "accuracy": 0.82 | |
| }, | |
| { | |
| "trial": 1, | |
| "correct": 38, | |
| "total": 50, | |
| "accuracy": 0.76 | |
| }, | |
| { | |
| "trial": 2, | |
| "correct": 34, | |
| "total": 50, | |
| "accuracy": 0.68 | |
| }, | |
| { | |
| "trial": 3, | |
| "correct": 39, | |
| "total": 50, | |
| "accuracy": 0.78 | |
| }, | |
| { | |
| "trial": 4, | |
| "correct": 38, | |
| "total": 50, | |
| "accuracy": 0.76 | |
| } | |
| ], | |
| "mean_accuracy": 0.76, | |
| "best_accuracy": 0.82, | |
| "seconds": 50.37944988813251 | |
| }, | |
| "protocol": "OPTIMIZATION.json" | |
| }, | |
| "optimized_defaults": { | |
| "personamem-32k/qwen2.5-3b": { | |
| "source_revision": "45b50a33bbcf55013f0c720a6980ea4fcc5e5db3", | |
| "validation": { | |
| "trials": [ | |
| { | |
| "trial": 0, | |
| "correct": 32, | |
| "total": 50, | |
| "accuracy": 0.64 | |
| }, | |
| { | |
| "trial": 1, | |
| "correct": 34, | |
| "total": 50, | |
| "accuracy": 0.68 | |
| }, | |
| { | |
| "trial": 2, | |
| "correct": 34, | |
| "total": 50, | |
| "accuracy": 0.68 | |
| }, | |
| { | |
| "trial": 3, | |
| "correct": 38, | |
| "total": 50, | |
| "accuracy": 0.76 | |
| }, | |
| { | |
| "trial": 4, | |
| "correct": 31, | |
| "total": 50, | |
| "accuracy": 0.62 | |
| } | |
| ], | |
| "mean_accuracy": 0.6759999999999999, | |
| "best_accuracy": 0.76 | |
| }, | |
| "test": { | |
| "trials": [ | |
| { | |
| "trial": 0, | |
| "correct": 32, | |
| "total": 50, | |
| "accuracy": 0.64 | |
| }, | |
| { | |
| "trial": 1, | |
| "correct": 28, | |
| "total": 50, | |
| "accuracy": 0.56 | |
| }, | |
| { | |
| "trial": 2, | |
| "correct": 30, | |
| "total": 50, | |
| "accuracy": 0.6 | |
| }, | |
| { | |
| "trial": 3, | |
| "correct": 30, | |
| "total": 50, | |
| "accuracy": 0.6 | |
| }, | |
| { | |
| "trial": 4, | |
| "correct": 33, | |
| "total": 50, | |
| "accuracy": 0.66 | |
| } | |
| ], | |
| "mean_accuracy": 0.6120000000000001, | |
| "best_accuracy": 0.66 | |
| }, | |
| "details": "personamem-32k/qwen2.5-3b/optimization.json" | |
| }, | |
| "personamem-32k/qwen2.5-7b": { | |
| "source_revision": "95b195098f933b18b6170f33b37d007a862554f9", | |
| "validation": { | |
| "trials": [ | |
| { | |
| "trial": 0, | |
| "correct": 39, | |
| "total": 50, | |
| "accuracy": 0.78 | |
| }, | |
| { | |
| "trial": 1, | |
| "correct": 38, | |
| "total": 50, | |
| "accuracy": 0.76 | |
| }, | |
| { | |
| "trial": 2, | |
| "correct": 36, | |
| "total": 50, | |
| "accuracy": 0.72 | |
| }, | |
| { | |
| "trial": 3, | |
| "correct": 36, | |
| "total": 50, | |
| "accuracy": 0.72 | |
| }, | |
| { | |
| "trial": 4, | |
| "correct": 37, | |
| "total": 50, | |
| "accuracy": 0.74 | |
| } | |
| ], | |
| "mean_accuracy": 0.744, | |
| "best_accuracy": 0.78 | |
| }, | |
| "test": { | |
| "trials": [ | |
| { | |
| "trial": 0, | |
| "correct": 40, | |
| "total": 50, | |
| "accuracy": 0.8 | |
| }, | |
| { | |
| "trial": 1, | |
| "correct": 39, | |
| "total": 50, | |
| "accuracy": 0.78 | |
| }, | |
| { | |
| "trial": 2, | |
| "correct": 38, | |
| "total": 50, | |
| "accuracy": 0.76 | |
| }, | |
| { | |
| "trial": 3, | |
| "correct": 43, | |
| "total": 50, | |
| "accuracy": 0.86 | |
| }, | |
| { | |
| "trial": 4, | |
| "correct": 40, | |
| "total": 50, | |
| "accuracy": 0.8 | |
| } | |
| ], | |
| "mean_accuracy": 0.8, | |
| "best_accuracy": 0.86 | |
| }, | |
| "details": "personamem-32k/qwen2.5-7b/optimization.json" | |
| }, | |
| "personamem-128k/qwen3-4b": { | |
| "validation": { | |
| "name": "highest-lr-step-1000", | |
| "trials": [ | |
| { | |
| "trial": 0, | |
| "correct": 235, | |
| "total": 273, | |
| "accuracy": 0.8608058608058609 | |
| }, | |
| { | |
| "trial": 1, | |
| "correct": 244, | |
| "total": 273, | |
| "accuracy": 0.8937728937728938 | |
| }, | |
| { | |
| "trial": 2, | |
| "correct": 242, | |
| "total": 273, | |
| "accuracy": 0.8864468864468864 | |
| }, | |
| { | |
| "trial": 3, | |
| "correct": 251, | |
| "total": 273, | |
| "accuracy": 0.9194139194139194 | |
| }, | |
| { | |
| "trial": 4, | |
| "correct": 245, | |
| "total": 273, | |
| "accuracy": 0.8974358974358975 | |
| } | |
| ], | |
| "mean_accuracy": 0.8915750915750916, | |
| "best_accuracy": 0.9194139194139194, | |
| "pipeline_seconds": 528.7673501800746 | |
| }, | |
| "test": { | |
| "name": "selected", | |
| "trials": [ | |
| { | |
| "trial": 0, | |
| "correct": 205, | |
| "total": 233, | |
| "accuracy": 0.8798283261802575 | |
| }, | |
| { | |
| "trial": 1, | |
| "correct": 197, | |
| "total": 233, | |
| "accuracy": 0.8454935622317596 | |
| }, | |
| { | |
| "trial": 2, | |
| "correct": 209, | |
| "total": 233, | |
| "accuracy": 0.8969957081545065 | |
| }, | |
| { | |
| "trial": 3, | |
| "correct": 204, | |
| "total": 233, | |
| "accuracy": 0.8755364806866953 | |
| }, | |
| { | |
| "trial": 4, | |
| "correct": 206, | |
| "total": 233, | |
| "accuracy": 0.8841201716738197 | |
| } | |
| ], | |
| "mean_accuracy": 0.8763948497854077, | |
| "best_accuracy": 0.8969957081545065, | |
| "pipeline_seconds": 488.22901280317456 | |
| }, | |
| "adapter_config_load": true, | |
| "finite_adapter": true, | |
| "matching_frozen_compressor": true, | |
| "details": "personamem-128k/qwen3-4b/optimization.json" | |
| }, | |
| "personamem-128k/qwen2.5-3b": { | |
| "validation": { | |
| "name": "current-ratio-step-600", | |
| "trials": [ | |
| { | |
| "trial": 0, | |
| "correct": 245, | |
| "total": 273, | |
| "accuracy": 0.8974358974358975 | |
| }, | |
| { | |
| "trial": 1, | |
| "correct": 244, | |
| "total": 273, | |
| "accuracy": 0.8937728937728938 | |
| }, | |
| { | |
| "trial": 2, | |
| "correct": 240, | |
| "total": 273, | |
| "accuracy": 0.8791208791208791 | |
| }, | |
| { | |
| "trial": 3, | |
| "correct": 248, | |
| "total": 273, | |
| "accuracy": 0.9084249084249084 | |
| }, | |
| { | |
| "trial": 4, | |
| "correct": 241, | |
| "total": 273, | |
| "accuracy": 0.8827838827838828 | |
| } | |
| ], | |
| "mean_accuracy": 0.8923076923076924, | |
| "best_accuracy": 0.9084249084249084, | |
| "pipeline_seconds": 517.7519376315176 | |
| }, | |
| "test": { | |
| "name": "selected", | |
| "trials": [ | |
| { | |
| "trial": 0, | |
| "correct": 203, | |
| "total": 233, | |
| "accuracy": 0.871244635193133 | |
| }, | |
| { | |
| "trial": 1, | |
| "correct": 203, | |
| "total": 233, | |
| "accuracy": 0.871244635193133 | |
| }, | |
| { | |
| "trial": 2, | |
| "correct": 208, | |
| "total": 233, | |
| "accuracy": 0.8927038626609443 | |
| }, | |
| { | |
| "trial": 3, | |
| "correct": 205, | |
| "total": 233, | |
| "accuracy": 0.8798283261802575 | |
| }, | |
| { | |
| "trial": 4, | |
| "correct": 202, | |
| "total": 233, | |
| "accuracy": 0.8669527896995708 | |
| } | |
| ], | |
| "mean_accuracy": 0.8763948497854077, | |
| "best_accuracy": 0.8927038626609443, | |
| "pipeline_seconds": 370.2440517041832 | |
| }, | |
| "adapter_config_load": true, | |
| "finite_adapter": true, | |
| "matching_frozen_compressor": true, | |
| "details": "personamem-128k/qwen2.5-3b/optimization.json" | |
| } | |
| } | |
| } | |