Text Generation
Transformers
Safetensors
lfm2
liquid
lfm2.5
weight-editing
model-editing
json
training-free
conversational
Instructions to use yasserrmd/LFM2.5-1.2B-Instruct-WSP-JSONFix with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use yasserrmd/LFM2.5-1.2B-Instruct-WSP-JSONFix with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="yasserrmd/LFM2.5-1.2B-Instruct-WSP-JSONFix") messages = [ {"role": "user", "content": "Who are you?"}, ] pipe(messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("yasserrmd/LFM2.5-1.2B-Instruct-WSP-JSONFix") model = AutoModelForCausalLM.from_pretrained("yasserrmd/LFM2.5-1.2B-Instruct-WSP-JSONFix", device_map="auto") messages = [ {"role": "user", "content": "Who are you?"}, ] inputs = tokenizer.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(tokenizer.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use yasserrmd/LFM2.5-1.2B-Instruct-WSP-JSONFix with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "yasserrmd/LFM2.5-1.2B-Instruct-WSP-JSONFix" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "yasserrmd/LFM2.5-1.2B-Instruct-WSP-JSONFix", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker
docker model run hf.co/yasserrmd/LFM2.5-1.2B-Instruct-WSP-JSONFix
- SGLang
How to use yasserrmd/LFM2.5-1.2B-Instruct-WSP-JSONFix with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "yasserrmd/LFM2.5-1.2B-Instruct-WSP-JSONFix" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "yasserrmd/LFM2.5-1.2B-Instruct-WSP-JSONFix", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "yasserrmd/LFM2.5-1.2B-Instruct-WSP-JSONFix" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "yasserrmd/LFM2.5-1.2B-Instruct-WSP-JSONFix", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }' - Docker Model Runner
How to use yasserrmd/LFM2.5-1.2B-Instruct-WSP-JSONFix with Docker Model Runner:
docker model run hf.co/yasserrmd/LFM2.5-1.2B-Instruct-WSP-JSONFix
Download wsp/test_summary.json from yasserrmd/LFM2.5-1.2B-Instruct-WSP-JSONFix: direct link, hf CLI and curl.
- Browser
- Download file 16.5 kB
-
https://huggingface.co/yasserrmd/LFM2.5-1.2B-Instruct-WSP-JSONFix/resolve/main/wsp/test_summary.json
- Command line
-
hf download hf://yasserrmd/LFM2.5-1.2B-Instruct-WSP-JSONFix/wsp/test_summary.json
-
curl -L -o test_summary.json https://huggingface.co/yasserrmd/LFM2.5-1.2B-Instruct-WSP-JSONFix/resolve/main/wsp/test_summary.json
16.5 kB
| { | |
| "chosen_c11": { | |
| "pooled": { | |
| "B_correct": 196, | |
| "B_strict_parse": 253, | |
| "B_schema": 218, | |
| "B_fence": 0, | |
| "B_start_brace": 129, | |
| "B_start_bracket": 127, | |
| "B_start_fence": 0, | |
| "B_object_exact": 99, | |
| "B_array_exact": 97, | |
| "B_array_wrong_container_start": 1, | |
| "B_object_wrong_container_start": 0, | |
| "A_correct": 35, | |
| "C_correct": 57, | |
| "S_correct": 64, | |
| "F_correct": 48, | |
| "F_balanced": 48, | |
| "length_failures": 0 | |
| }, | |
| "greedy_B": 49, | |
| "ppl_rel": -0.0007307205221399877, | |
| "kl_first_token": 0.42265161871910095, | |
| "kl_passage": 0.0014987997710704803, | |
| "card_B": 50, | |
| "B_pooled": { | |
| "n": 256, | |
| "base": 167, | |
| "edit": 196, | |
| "improved": 29, | |
| "broken": 0, | |
| "mcnemar_exact_p": 3.725290298461914e-09, | |
| "base_ci": [ | |
| 0.5921427947221091, | |
| 0.7080400859251775 | |
| ], | |
| "edit_ci": [ | |
| 0.7100415017359011, | |
| 0.8133542901619322 | |
| ] | |
| }, | |
| "B_greedy": { | |
| "n": 64, | |
| "base": 42, | |
| "edit": 49, | |
| "improved": 7, | |
| "broken": 0, | |
| "mcnemar_exact_p": 0.015625, | |
| "base_ci": [ | |
| 0.5340341181762039, | |
| 0.7607702496453097 | |
| ], | |
| "edit_ci": [ | |
| 0.6486650974929234, | |
| 0.8525023278036498 | |
| ] | |
| }, | |
| "B_card": { | |
| "n": 64, | |
| "base": 42, | |
| "edit": 50, | |
| "improved": 8, | |
| "broken": 0, | |
| "mcnemar_exact_p": 0.0078125, | |
| "base_ci": [ | |
| 0.5340341181762039, | |
| 0.7607702496453097 | |
| ], | |
| "edit_ci": [ | |
| 0.6656698366922221, | |
| 0.8649780253865025 | |
| ] | |
| }, | |
| "B_array_pooled": { | |
| "n": 128, | |
| "base": 76, | |
| "edit": 97, | |
| "improved": 21, | |
| "broken": 0, | |
| "mcnemar_exact_p": 9.5367431640625e-07, | |
| "base_ci": [ | |
| 0.5071382471838717, | |
| 0.674898378585233 | |
| ], | |
| "edit_ci": [ | |
| 0.6767870437261758, | |
| 0.8238136771388622 | |
| ] | |
| }, | |
| "B_object_pooled": { | |
| "n": 128, | |
| "base": 91, | |
| "edit": 99, | |
| "improved": 8, | |
| "broken": 0, | |
| "mcnemar_exact_p": 0.0078125, | |
| "base_ci": [ | |
| 0.6271652314384711, | |
| 0.7824171765420145 | |
| ], | |
| "edit_ci": [ | |
| 0.6935716289452746, | |
| 0.8373685295479476 | |
| ] | |
| }, | |
| "B_nested_pooled": { | |
| "n": 80, | |
| "base": 50, | |
| "edit": 63, | |
| "improved": 13, | |
| "broken": 0, | |
| "mcnemar_exact_p": 0.000244140625, | |
| "base_ci": [ | |
| 0.515485228513904, | |
| 0.7230598374199523 | |
| ], | |
| "edit_ci": [ | |
| 0.6857763938182242, | |
| 0.8628772578296451 | |
| ] | |
| }, | |
| "B_nonascii_pooled": { | |
| "n": 72, | |
| "base": 65, | |
| "edit": 71, | |
| "improved": 6, | |
| "broken": 0, | |
| "mcnemar_exact_p": 0.03125, | |
| "base_ci": [ | |
| 0.812647509746432, | |
| 0.952104288738832 | |
| ], | |
| "edit_ci": [ | |
| 0.9254321971571302, | |
| 0.9975441113595678 | |
| ] | |
| }, | |
| "B_strict_parse_pooled": { | |
| "n": 256, | |
| "base": 211, | |
| "edit": 253, | |
| "improved": 42, | |
| "broken": 0, | |
| "mcnemar_exact_p": 4.547473508864641e-13, | |
| "base_ci": [ | |
| 0.772896116682119, | |
| 0.8659546293108245 | |
| ], | |
| "edit_ci": [ | |
| 0.9661178322810585, | |
| 0.9960067851781936 | |
| ] | |
| }, | |
| "B_schema_pooled": { | |
| "n": 256, | |
| "base": 186, | |
| "edit": 218, | |
| "improved": 32, | |
| "broken": 0, | |
| "mcnemar_exact_p": 4.656612873077393e-10, | |
| "base_ci": [ | |
| 0.6689134962060558, | |
| 0.7775123262950373 | |
| ], | |
| "edit_ci": [ | |
| 0.8028238602392018, | |
| 0.8899058643314598 | |
| ] | |
| }, | |
| "A": { | |
| "n": 64, | |
| "base": 36, | |
| "edit": 35, | |
| "improved": 0, | |
| "broken": 1, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.44085991778287875, | |
| 0.6770618293457267 | |
| ], | |
| "edit_ci": [ | |
| 0.4257338252067978, | |
| 0.6627074851396563 | |
| ] | |
| }, | |
| "C": { | |
| "n": 64, | |
| "base": 57, | |
| "edit": 57, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.7910113670806147, | |
| 0.9459995524731696 | |
| ], | |
| "edit_ci": [ | |
| 0.7910113670806147, | |
| 0.9459995524731696 | |
| ] | |
| }, | |
| "S": { | |
| "n": 64, | |
| "base": 64, | |
| "edit": 64, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.9433739770288436, | |
| 1.0 | |
| ], | |
| "edit_ci": [ | |
| 0.9433739770288436, | |
| 1.0 | |
| ] | |
| }, | |
| "F": { | |
| "n": 48, | |
| "base": 48, | |
| "edit": 48, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.9258973488472576, | |
| 0.9999999999999998 | |
| ], | |
| "edit_ci": [ | |
| 0.9258973488472576, | |
| 0.9999999999999998 | |
| ] | |
| } | |
| }, | |
| "sham": { | |
| "pooled": { | |
| "B_correct": 167, | |
| "B_strict_parse": 211, | |
| "B_schema": 186, | |
| "B_fence": 41, | |
| "B_start_brace": 119, | |
| "B_start_bracket": 96, | |
| "B_start_fence": 41, | |
| "B_object_exact": 91, | |
| "B_array_exact": 76, | |
| "B_array_wrong_container_start": 0, | |
| "B_object_wrong_container_start": 0, | |
| "A_correct": 36, | |
| "C_correct": 57, | |
| "S_correct": 64, | |
| "F_correct": 48, | |
| "F_balanced": 47, | |
| "length_failures": 0 | |
| }, | |
| "greedy_B": 42, | |
| "ppl_rel": 0.0, | |
| "kl_first_token": 0.0, | |
| "kl_passage": 0.0, | |
| "card_B": 42, | |
| "B_pooled": { | |
| "n": 256, | |
| "base": 167, | |
| "edit": 167, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.5921427947221091, | |
| 0.7080400859251775 | |
| ], | |
| "edit_ci": [ | |
| 0.5921427947221091, | |
| 0.7080400859251775 | |
| ] | |
| }, | |
| "B_greedy": { | |
| "n": 64, | |
| "base": 42, | |
| "edit": 42, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.5340341181762039, | |
| 0.7607702496453097 | |
| ], | |
| "edit_ci": [ | |
| 0.5340341181762039, | |
| 0.7607702496453097 | |
| ] | |
| }, | |
| "B_card": { | |
| "n": 64, | |
| "base": 42, | |
| "edit": 42, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.5340341181762039, | |
| 0.7607702496453097 | |
| ], | |
| "edit_ci": [ | |
| 0.5340341181762039, | |
| 0.7607702496453097 | |
| ] | |
| }, | |
| "B_array_pooled": { | |
| "n": 128, | |
| "base": 76, | |
| "edit": 76, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.5071382471838717, | |
| 0.674898378585233 | |
| ], | |
| "edit_ci": [ | |
| 0.5071382471838717, | |
| 0.674898378585233 | |
| ] | |
| }, | |
| "B_object_pooled": { | |
| "n": 128, | |
| "base": 91, | |
| "edit": 91, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.6271652314384711, | |
| 0.7824171765420145 | |
| ], | |
| "edit_ci": [ | |
| 0.6271652314384711, | |
| 0.7824171765420145 | |
| ] | |
| }, | |
| "B_nested_pooled": { | |
| "n": 80, | |
| "base": 50, | |
| "edit": 50, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.515485228513904, | |
| 0.7230598374199523 | |
| ], | |
| "edit_ci": [ | |
| 0.515485228513904, | |
| 0.7230598374199523 | |
| ] | |
| }, | |
| "B_nonascii_pooled": { | |
| "n": 72, | |
| "base": 65, | |
| "edit": 65, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.812647509746432, | |
| 0.952104288738832 | |
| ], | |
| "edit_ci": [ | |
| 0.812647509746432, | |
| 0.952104288738832 | |
| ] | |
| }, | |
| "B_strict_parse_pooled": { | |
| "n": 256, | |
| "base": 211, | |
| "edit": 211, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.772896116682119, | |
| 0.8659546293108245 | |
| ], | |
| "edit_ci": [ | |
| 0.772896116682119, | |
| 0.8659546293108245 | |
| ] | |
| }, | |
| "B_schema_pooled": { | |
| "n": 256, | |
| "base": 186, | |
| "edit": 186, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.6689134962060558, | |
| 0.7775123262950373 | |
| ], | |
| "edit_ci": [ | |
| 0.6689134962060558, | |
| 0.7775123262950373 | |
| ] | |
| }, | |
| "A": { | |
| "n": 64, | |
| "base": 36, | |
| "edit": 36, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.44085991778287875, | |
| 0.6770618293457267 | |
| ], | |
| "edit_ci": [ | |
| 0.44085991778287875, | |
| 0.6770618293457267 | |
| ] | |
| }, | |
| "C": { | |
| "n": 64, | |
| "base": 57, | |
| "edit": 57, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.7910113670806147, | |
| 0.9459995524731696 | |
| ], | |
| "edit_ci": [ | |
| 0.7910113670806147, | |
| 0.9459995524731696 | |
| ] | |
| }, | |
| "S": { | |
| "n": 64, | |
| "base": 64, | |
| "edit": 64, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.9433739770288436, | |
| 1.0 | |
| ], | |
| "edit_ci": [ | |
| 0.9433739770288436, | |
| 1.0 | |
| ] | |
| }, | |
| "F": { | |
| "n": 48, | |
| "base": 48, | |
| "edit": 48, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.9258973488472576, | |
| 0.9999999999999998 | |
| ], | |
| "edit_ci": [ | |
| 0.9258973488472576, | |
| 0.9999999999999998 | |
| ] | |
| } | |
| }, | |
| "c11_rnd": { | |
| "pooled": { | |
| "B_correct": 167, | |
| "B_strict_parse": 212, | |
| "B_schema": 187, | |
| "B_fence": 40, | |
| "B_start_brace": 119, | |
| "B_start_bracket": 97, | |
| "B_start_fence": 40, | |
| "B_object_exact": 91, | |
| "B_array_exact": 76, | |
| "B_array_wrong_container_start": 0, | |
| "B_object_wrong_container_start": 0, | |
| "A_correct": 36, | |
| "C_correct": 56, | |
| "S_correct": 64, | |
| "F_correct": 48, | |
| "F_balanced": 47, | |
| "length_failures": 0 | |
| }, | |
| "greedy_B": 42, | |
| "ppl_rel": 0.002196327451208909, | |
| "kl_first_token": 0.0005274508730508387, | |
| "kl_passage": 0.0006625415990129113, | |
| "card_B": 42, | |
| "B_pooled": { | |
| "n": 256, | |
| "base": 167, | |
| "edit": 167, | |
| "improved": 1, | |
| "broken": 1, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.5921427947221091, | |
| 0.7080400859251775 | |
| ], | |
| "edit_ci": [ | |
| 0.5921427947221091, | |
| 0.7080400859251775 | |
| ] | |
| }, | |
| "B_greedy": { | |
| "n": 64, | |
| "base": 42, | |
| "edit": 42, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.5340341181762039, | |
| 0.7607702496453097 | |
| ], | |
| "edit_ci": [ | |
| 0.5340341181762039, | |
| 0.7607702496453097 | |
| ] | |
| }, | |
| "B_card": { | |
| "n": 64, | |
| "base": 42, | |
| "edit": 42, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.5340341181762039, | |
| 0.7607702496453097 | |
| ], | |
| "edit_ci": [ | |
| 0.5340341181762039, | |
| 0.7607702496453097 | |
| ] | |
| }, | |
| "B_array_pooled": { | |
| "n": 128, | |
| "base": 76, | |
| "edit": 76, | |
| "improved": 1, | |
| "broken": 1, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.5071382471838717, | |
| 0.674898378585233 | |
| ], | |
| "edit_ci": [ | |
| 0.5071382471838717, | |
| 0.674898378585233 | |
| ] | |
| }, | |
| "B_object_pooled": { | |
| "n": 128, | |
| "base": 91, | |
| "edit": 91, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.6271652314384711, | |
| 0.7824171765420145 | |
| ], | |
| "edit_ci": [ | |
| 0.6271652314384711, | |
| 0.7824171765420145 | |
| ] | |
| }, | |
| "B_nested_pooled": { | |
| "n": 80, | |
| "base": 50, | |
| "edit": 49, | |
| "improved": 0, | |
| "broken": 1, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.515485228513904, | |
| 0.7230598374199523 | |
| ], | |
| "edit_ci": [ | |
| 0.5029345562232477, | |
| 0.711756003117223 | |
| ] | |
| }, | |
| "B_nonascii_pooled": { | |
| "n": 72, | |
| "base": 65, | |
| "edit": 65, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.812647509746432, | |
| 0.952104288738832 | |
| ], | |
| "edit_ci": [ | |
| 0.812647509746432, | |
| 0.952104288738832 | |
| ] | |
| }, | |
| "B_strict_parse_pooled": { | |
| "n": 256, | |
| "base": 211, | |
| "edit": 212, | |
| "improved": 1, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.772896116682119, | |
| 0.8659546293108245 | |
| ], | |
| "edit_ci": [ | |
| 0.7771451946116188, | |
| 0.8694025483209986 | |
| ] | |
| }, | |
| "B_schema_pooled": { | |
| "n": 256, | |
| "base": 186, | |
| "edit": 187, | |
| "improved": 2, | |
| "broken": 1, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.6689134962060558, | |
| 0.7775123262950373 | |
| ], | |
| "edit_ci": [ | |
| 0.6730019753620341, | |
| 0.781120844078733 | |
| ] | |
| }, | |
| "A": { | |
| "n": 64, | |
| "base": 36, | |
| "edit": 36, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.44085991778287875, | |
| 0.6770618293457267 | |
| ], | |
| "edit_ci": [ | |
| 0.44085991778287875, | |
| 0.6770618293457267 | |
| ] | |
| }, | |
| "C": { | |
| "n": 64, | |
| "base": 57, | |
| "edit": 56, | |
| "improved": 0, | |
| "broken": 1, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.7910113670806147, | |
| 0.9459995524731696 | |
| ], | |
| "edit_ci": [ | |
| 0.7722521366978408, | |
| 0.935278346073792 | |
| ] | |
| }, | |
| "S": { | |
| "n": 64, | |
| "base": 64, | |
| "edit": 64, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.9433739770288436, | |
| 1.0 | |
| ], | |
| "edit_ci": [ | |
| 0.9433739770288436, | |
| 1.0 | |
| ] | |
| }, | |
| "F": { | |
| "n": 48, | |
| "base": 48, | |
| "edit": 48, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.9258973488472576, | |
| 0.9999999999999998 | |
| ], | |
| "edit_ci": [ | |
| 0.9258973488472576, | |
| 0.9999999999999998 | |
| ] | |
| } | |
| }, | |
| "c11_scd": { | |
| "pooled": { | |
| "B_correct": 166, | |
| "B_strict_parse": 211, | |
| "B_schema": 185, | |
| "B_fence": 40, | |
| "B_start_brace": 120, | |
| "B_start_bracket": 96, | |
| "B_start_fence": 40, | |
| "B_object_exact": 91, | |
| "B_array_exact": 75, | |
| "B_array_wrong_container_start": 0, | |
| "B_object_wrong_container_start": 0, | |
| "A_correct": 36, | |
| "C_correct": 56, | |
| "S_correct": 64, | |
| "F_correct": 48, | |
| "F_balanced": 47, | |
| "length_failures": 0 | |
| }, | |
| "greedy_B": 42, | |
| "ppl_rel": 0.0007641077187694739, | |
| "kl_first_token": 0.0037935180589556694, | |
| "kl_passage": 0.0009359674295410514, | |
| "card_B": 42, | |
| "B_pooled": { | |
| "n": 256, | |
| "base": 167, | |
| "edit": 166, | |
| "improved": 0, | |
| "broken": 1, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.5921427947221091, | |
| 0.7080400859251775 | |
| ], | |
| "edit_ci": [ | |
| 0.5881468207176852, | |
| 0.7043390629899275 | |
| ] | |
| }, | |
| "B_greedy": { | |
| "n": 64, | |
| "base": 42, | |
| "edit": 42, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.5340341181762039, | |
| 0.7607702496453097 | |
| ], | |
| "edit_ci": [ | |
| 0.5340341181762039, | |
| 0.7607702496453097 | |
| ] | |
| }, | |
| "B_card": { | |
| "n": 64, | |
| "base": 42, | |
| "edit": 42, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.5340341181762039, | |
| 0.7607702496453097 | |
| ], | |
| "edit_ci": [ | |
| 0.5340341181762039, | |
| 0.7607702496453097 | |
| ] | |
| }, | |
| "B_array_pooled": { | |
| "n": 128, | |
| "base": 76, | |
| "edit": 75, | |
| "improved": 0, | |
| "broken": 1, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.5071382471838717, | |
| 0.674898378585233 | |
| ], | |
| "edit_ci": [ | |
| 0.4993170001197534, | |
| 0.6675499068352593 | |
| ] | |
| }, | |
| "B_object_pooled": { | |
| "n": 128, | |
| "base": 91, | |
| "edit": 91, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.6271652314384711, | |
| 0.7824171765420145 | |
| ], | |
| "edit_ci": [ | |
| 0.6271652314384711, | |
| 0.7824171765420145 | |
| ] | |
| }, | |
| "B_nested_pooled": { | |
| "n": 80, | |
| "base": 50, | |
| "edit": 50, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.515485228513904, | |
| 0.7230598374199523 | |
| ], | |
| "edit_ci": [ | |
| 0.515485228513904, | |
| 0.7230598374199523 | |
| ] | |
| }, | |
| "B_nonascii_pooled": { | |
| "n": 72, | |
| "base": 65, | |
| "edit": 64, | |
| "improved": 0, | |
| "broken": 1, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.812647509746432, | |
| 0.952104288738832 | |
| ], | |
| "edit_ci": [ | |
| 0.7957685574140959, | |
| 0.9426124893992622 | |
| ] | |
| }, | |
| "B_strict_parse_pooled": { | |
| "n": 256, | |
| "base": 211, | |
| "edit": 211, | |
| "improved": 1, | |
| "broken": 1, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.772896116682119, | |
| 0.8659546293108245 | |
| ], | |
| "edit_ci": [ | |
| 0.772896116682119, | |
| 0.8659546293108245 | |
| ] | |
| }, | |
| "B_schema_pooled": { | |
| "n": 256, | |
| "base": 186, | |
| "edit": 185, | |
| "improved": 0, | |
| "broken": 1, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.6689134962060558, | |
| 0.7775123262950373 | |
| ], | |
| "edit_ci": [ | |
| 0.6648301483971852, | |
| 0.7738986771642338 | |
| ] | |
| }, | |
| "A": { | |
| "n": 64, | |
| "base": 36, | |
| "edit": 36, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.44085991778287875, | |
| 0.6770618293457267 | |
| ], | |
| "edit_ci": [ | |
| 0.44085991778287875, | |
| 0.6770618293457267 | |
| ] | |
| }, | |
| "C": { | |
| "n": 64, | |
| "base": 57, | |
| "edit": 56, | |
| "improved": 0, | |
| "broken": 1, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.7910113670806147, | |
| 0.9459995524731696 | |
| ], | |
| "edit_ci": [ | |
| 0.7722521366978408, | |
| 0.935278346073792 | |
| ] | |
| }, | |
| "S": { | |
| "n": 64, | |
| "base": 64, | |
| "edit": 64, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.9433739770288436, | |
| 1.0 | |
| ], | |
| "edit_ci": [ | |
| 0.9433739770288436, | |
| 1.0 | |
| ] | |
| }, | |
| "F": { | |
| "n": 48, | |
| "base": 48, | |
| "edit": 48, | |
| "improved": 0, | |
| "broken": 0, | |
| "mcnemar_exact_p": 1.0, | |
| "base_ci": [ | |
| 0.9258973488472576, | |
| 0.9999999999999998 | |
| ], | |
| "edit_ci": [ | |
| 0.9258973488472576, | |
| 0.9999999999999998 | |
| ] | |
| } | |
| }, | |
| "base": { | |
| "pooled": { | |
| "B_correct": 167, | |
| "B_strict_parse": 211, | |
| "B_schema": 186, | |
| "B_fence": 41, | |
| "B_start_brace": 119, | |
| "B_start_bracket": 96, | |
| "B_start_fence": 41, | |
| "B_object_exact": 91, | |
| "B_array_exact": 76, | |
| "B_array_wrong_container_start": 0, | |
| "B_object_wrong_container_start": 0, | |
| "A_correct": 36, | |
| "C_correct": 57, | |
| "S_correct": 64, | |
| "F_correct": 48, | |
| "F_balanced": 47, | |
| "length_failures": 0 | |
| }, | |
| "greedy_B": 42, | |
| "ppl": 130.5471182180624, | |
| "card_B": 42 | |
| } | |
| } |