Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

.hydra/config.yaml

@@ -0,0 +1,99 @@
+model:
+  model_args:
+    pretrained_model_name_or_path: Qwen/Qwen2.5-7B
+    torch_dtype: bfloat16
+  peft_config:
+    r: 128
+    target_modules:
+    - down_proj
+    layers_to_transform:
+    - 5
+    - 6
+    - 7
+  tokenizer_args:
+    pretrained_model_name_or_path: Qwen/Qwen2.5-7B
+  template_args:
+    apply_chat_template: false
+trainer:
+  handler: GradDiff
+  args:
+    seed: 42
+    output_dir: ${paths.output_dir}
+    logging_dir: ${trainer.args.output_dir}/logs
+    bf16: true
+    bf16_full_eval: true
+    do_train: true
+    learning_rate: 5.0e-05
+    lr_scheduler_type: constant
+    logging_steps: 5
+    report_to: wandb
+    resume_from_checkpoint: null
+    eval_strategy: 'no'
+    save_strategy: 'no'
+    ddp_find_unused_parameters: true
+    gradient_checkpointing: true
+    per_device_train_batch_size: 1
+    gradient_accumulation_steps: 4
+    num_train_epochs: 1
+    run_name: ${task_name}
+    remove_unused_columns: false
+  method_args:
+    gamma: 1.0
+    alpha: 20
+    retain_loss_type: NLL
+    log_loss_grads: false
+    trainable_params_regex: null
+data:
+  forget:
+    Multilingual:
+      handler: PretrainingForMultilingualDataset
+      args:
+        data_path: data/multilingual
+        language: ${language}
+        books: ${books}
+        chapters: ${chapters}
+        text_key: text
+        max_length: 512
+  retain:
+    Wikitext:
+      handler: PretrainingDataset
+      args:
+        hf_args:
+          path: json
+          data_dir: data/multilingual_wikitext
+          data_files: ${language}.jsonl
+          split: train
+        text_key: text
+        max_length: 512
+  anchor: forget
+collator:
+  DataCollatorForSupervisedDataset:
+    handler: DataCollatorForSupervisedDataset
+    args:
+      padding_side: left
+eval:
+  lm_eval:
+    handler: LMEvalEvaluator
+    output_dir: ${paths.output_dir}
+    overwrite: false
+    tasks:
+    - multilingual_unlearning
+    - mmlu
+    simple_evaluate_args:
+      batch_size: auto
+      system_instruction: null
+      apply_chat_template: false
+    batch_size: 32
+paths:
+  root_dir: .
+  data_dir: ${paths.root_dir}/data/
+  datasets: ${paths.root_dir}/configs/data/datasets
+  output_dir: ${paths.root_dir}/saves/${mode}/${task_name}
+  work_dir: ${hydra:runtime.cwd}
+task_name: multilingual/books.all/chapters.all/GradDiff.NLL/vi
+unlearn_layer: null
+language: vi
+languages: ???
+books: all
+chapters: all
+mode: unlearn

.hydra/hydra.yaml

@@ -0,0 +1,188 @@
+hydra:
+  run:
+    dir: ${paths.output_dir}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      colorlog:
+        (): colorlog.ColoredFormatter
+        format: '[%(cyan)s%(asctime)s%(reset)s][%(purple)sHYDRA%(reset)s] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: colorlog
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+      colorlog:
+        (): colorlog.ColoredFormatter
+        format: '[%(cyan)s%(asctime)s%(reset)s][%(blue)s%(name)s%(reset)s][%(log_color)s%(levelname)s%(reset)s]
+          - %(message)s'
+        log_colors:
+          DEBUG: purple
+          INFO: green
+          WARNING: yellow
+          ERROR: red
+          CRITICAL: red
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: colorlog
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${trainer.handler}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - experiment=unlearn/multilingual/default.yaml
+    - task_name=multilingual/books.all/chapters.all/GradDiff.NLL/vi
+    - language=vi
+    - books=all
+    - chapters=all
+    - trainer=GradDiff
+    - model=Qwen2.5-7B
+    - trainer.method_args.alpha=20
+    - trainer.method_args.retain_loss_type=NLL
+    - model.peft_config.r=128
+    - model.peft_config.target_modules=[down_proj]
+    - model.peft_config.layers_to_transform=[5,6,7]
+  job:
+    name: train
+    chdir: null
+    override_dirname: books=all,chapters=all,experiment=unlearn/multilingual/default.yaml,language=vi,model.peft_config.layers_to_transform=[5,6,7],model.peft_config.r=128,model.peft_config.target_modules=[down_proj],model=Qwen2.5-7B,task_name=multilingual/books.all/chapters.all/GradDiff.NLL/vi,trainer.method_args.alpha=20,trainer.method_args.retain_loss_type=NLL,trainer=GradDiff
+    id: ???
+    num: ???
+    config_name: unlearn.yaml
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.0
+    version_base: '1.3'
+    cwd: /home/ducdm/open-unlearning/open-unlearning
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/ducdm/open-unlearning/open-unlearning/configs
+      schema: file
+      provider: main
+    - path: hydra_plugins.hydra_colorlog.conf
+      schema: pkg
+      provider: hydra-colorlog
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/ducdm/open-unlearning/open-unlearning/saves/unlearn/multilingual/books.all/chapters.all/GradDiff.NLL/vi
+    choices:
+      experiment: unlearn/multilingual/default.yaml
+      paths: default
+      hydra: default
+      eval: lm_eval
+      collator: DataCollatorForSupervisedDataset
+      data: unlearn
+      data/datasets@data.eval: null
+      data/datasets@data.retain: Multilingual_Wikitext
+      data/datasets@data.forget: Multilingual
+      trainer: GradDiff
+      model: Qwen2.5-7B
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: colorlog
+      hydra/hydra_logging: colorlog
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

.hydra/overrides.yaml

@@ -0,0 +1,12 @@
+- experiment=unlearn/multilingual/default.yaml
+- task_name=multilingual/books.all/chapters.all/GradDiff.NLL/vi
+- language=vi
+- books=all
+- chapters=all
+- trainer=GradDiff
+- model=Qwen2.5-7B
+- trainer.method_args.alpha=20
+- trainer.method_args.retain_loss_type=NLL
+- model.peft_config.r=128
+- model.peft_config.target_modules=[down_proj]
+- model.peft_config.layers_to_transform=[5,6,7]

GradDiff.log

@@ -0,0 +1,145 @@
+[2026-06-16 14:23:58,619][trainer][INFO] - GradDiff Trainer loaded, output_dir: ./saves/unlearn/multilingual/books.all/chapters.all/GradDiff.NLL/vi
+[2026-06-16 15:20:15,840][lm_eval.models.huggingface][WARNING] - `pretrained` model kwarg is not of type `str`. Many other model arguments may be ignored. Please do not launch via accelerate or use `parallelize=True` if passing an existing model this way.
+[2026-06-16 15:20:17,159][lm_eval.models.huggingface][WARNING] - Passed an already-initialized model through `pretrained`, assuming single-process call to evaluate() or custom distributed integration
+[2026-06-16 15:20:17,161][lm_eval.evaluator][INFO] - Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234
+[2026-06-16 15:20:17,161][lm_eval.evaluator][INFO] - Using pre-initialized model
+[2026-06-16 15:21:59,994][lm_eval.evaluator_utils][INFO] - Selected tasks:
+[2026-06-16 15:21:59,995][lm_eval.evaluator_utils][INFO] - Group: mmlu
+[2026-06-16 15:21:59,996][lm_eval.evaluator_utils][INFO] -   Group: mmlu_stem
+[2026-06-16 15:21:59,996][lm_eval.evaluator_utils][INFO] -     Task: mmlu_abstract_algebra (mmlu/default/mmlu_abstract_algebra.yaml)
+[2026-06-16 15:21:59,997][lm_eval.evaluator_utils][INFO] -     Task: mmlu_anatomy (mmlu/default/mmlu_anatomy.yaml)
+[2026-06-16 15:21:59,997][lm_eval.evaluator_utils][INFO] -     Task: mmlu_astronomy (mmlu/default/mmlu_astronomy.yaml)
+[2026-06-16 15:21:59,997][lm_eval.evaluator_utils][INFO] -     Task: mmlu_college_biology (mmlu/default/mmlu_college_biology.yaml)
+[2026-06-16 15:21:59,998][lm_eval.evaluator_utils][INFO] -     Task: mmlu_college_chemistry (mmlu/default/mmlu_college_chemistry.yaml)
+[2026-06-16 15:21:59,998][lm_eval.evaluator_utils][INFO] -     Task: mmlu_college_computer_science (mmlu/default/mmlu_college_computer_science.yaml)
+[2026-06-16 15:21:59,998][lm_eval.evaluator_utils][INFO] -     Task: mmlu_college_mathematics (mmlu/default/mmlu_college_mathematics.yaml)
+[2026-06-16 15:21:59,999][lm_eval.evaluator_utils][INFO] -     Task: mmlu_college_physics (mmlu/default/mmlu_college_physics.yaml)
+[2026-06-16 15:21:59,999][lm_eval.evaluator_utils][INFO] -     Task: mmlu_computer_security (mmlu/default/mmlu_computer_security.yaml)
+[2026-06-16 15:21:59,999][lm_eval.evaluator_utils][INFO] -     Task: mmlu_conceptual_physics (mmlu/default/mmlu_conceptual_physics.yaml)
+[2026-06-16 15:22:00,000][lm_eval.evaluator_utils][INFO] -     Task: mmlu_electrical_engineering (mmlu/default/mmlu_electrical_engineering.yaml)
+[2026-06-16 15:22:00,001][lm_eval.evaluator_utils][INFO] -     Task: mmlu_elementary_mathematics (mmlu/default/mmlu_elementary_mathematics.yaml)
+[2026-06-16 15:22:00,001][lm_eval.evaluator_utils][INFO] -     Task: mmlu_high_school_biology (mmlu/default/mmlu_high_school_biology.yaml)
+[2026-06-16 15:22:00,002][lm_eval.evaluator_utils][INFO] -     Task: mmlu_high_school_chemistry (mmlu/default/mmlu_high_school_chemistry.yaml)
+[2026-06-16 15:22:00,002][lm_eval.evaluator_utils][INFO] -     Task: mmlu_high_school_computer_science (mmlu/default/mmlu_high_school_computer_science.yaml)
+[2026-06-16 15:22:00,003][lm_eval.evaluator_utils][INFO] -     Task: mmlu_high_school_mathematics (mmlu/default/mmlu_high_school_mathematics.yaml)
+[2026-06-16 15:22:00,004][lm_eval.evaluator_utils][INFO] -     Task: mmlu_high_school_physics (mmlu/default/mmlu_high_school_physics.yaml)
+[2026-06-16 15:22:00,004][lm_eval.evaluator_utils][INFO] -     Task: mmlu_high_school_statistics (mmlu/default/mmlu_high_school_statistics.yaml)
+[2026-06-16 15:22:00,005][lm_eval.evaluator_utils][INFO] -     Task: mmlu_machine_learning (mmlu/default/mmlu_machine_learning.yaml)
+[2026-06-16 15:22:00,005][lm_eval.evaluator_utils][INFO] -   Group: mmlu_other
+[2026-06-16 15:22:00,006][lm_eval.evaluator_utils][INFO] -     Task: mmlu_business_ethics (mmlu/default/mmlu_business_ethics.yaml)
+[2026-06-16 15:22:00,007][lm_eval.evaluator_utils][INFO] -     Task: mmlu_clinical_knowledge (mmlu/default/mmlu_clinical_knowledge.yaml)
+[2026-06-16 15:22:00,008][lm_eval.evaluator_utils][INFO] -     Task: mmlu_college_medicine (mmlu/default/mmlu_college_medicine.yaml)
+[2026-06-16 15:22:00,009][lm_eval.evaluator_utils][INFO] -     Task: mmlu_global_facts (mmlu/default/mmlu_global_facts.yaml)
+[2026-06-16 15:22:00,009][lm_eval.evaluator_utils][INFO] -     Task: mmlu_human_aging (mmlu/default/mmlu_human_aging.yaml)
+[2026-06-16 15:22:00,010][lm_eval.evaluator_utils][INFO] -     Task: mmlu_management (mmlu/default/mmlu_management.yaml)
+[2026-06-16 15:22:00,011][lm_eval.evaluator_utils][INFO] -     Task: mmlu_marketing (mmlu/default/mmlu_marketing.yaml)
+[2026-06-16 15:22:00,012][lm_eval.evaluator_utils][INFO] -     Task: mmlu_medical_genetics (mmlu/default/mmlu_medical_genetics.yaml)
+[2026-06-16 15:22:00,012][lm_eval.evaluator_utils][INFO] -     Task: mmlu_miscellaneous (mmlu/default/mmlu_miscellaneous.yaml)
+[2026-06-16 15:22:00,013][lm_eval.evaluator_utils][INFO] -     Task: mmlu_nutrition (mmlu/default/mmlu_nutrition.yaml)
+[2026-06-16 15:22:00,014][lm_eval.evaluator_utils][INFO] -     Task: mmlu_professional_accounting (mmlu/default/mmlu_professional_accounting.yaml)
+[2026-06-16 15:22:00,014][lm_eval.evaluator_utils][INFO] -     Task: mmlu_professional_medicine (mmlu/default/mmlu_professional_medicine.yaml)
+[2026-06-16 15:22:00,015][lm_eval.evaluator_utils][INFO] -     Task: mmlu_virology (mmlu/default/mmlu_virology.yaml)
+[2026-06-16 15:22:00,016][lm_eval.evaluator_utils][INFO] -   Group: mmlu_social_sciences
+[2026-06-16 15:22:00,016][lm_eval.evaluator_utils][INFO] -     Task: mmlu_econometrics (mmlu/default/mmlu_econometrics.yaml)
+[2026-06-16 15:22:00,017][lm_eval.evaluator_utils][INFO] -     Task: mmlu_high_school_geography (mmlu/default/mmlu_high_school_geography.yaml)
+[2026-06-16 15:22:00,018][lm_eval.evaluator_utils][INFO] -     Task: mmlu_high_school_government_and_politics (mmlu/default/mmlu_high_school_government_and_politics.yaml)
+[2026-06-16 15:22:00,019][lm_eval.evaluator_utils][INFO] -     Task: mmlu_high_school_macroeconomics (mmlu/default/mmlu_high_school_macroeconomics.yaml)
+[2026-06-16 15:22:00,019][lm_eval.evaluator_utils][INFO] -     Task: mmlu_high_school_microeconomics (mmlu/default/mmlu_high_school_microeconomics.yaml)
+[2026-06-16 15:22:00,020][lm_eval.evaluator_utils][INFO] -     Task: mmlu_high_school_psychology (mmlu/default/mmlu_high_school_psychology.yaml)
+[2026-06-16 15:22:00,021][lm_eval.evaluator_utils][INFO] -     Task: mmlu_human_sexuality (mmlu/default/mmlu_human_sexuality.yaml)
+[2026-06-16 15:22:00,021][lm_eval.evaluator_utils][INFO] -     Task: mmlu_professional_psychology (mmlu/default/mmlu_professional_psychology.yaml)
+[2026-06-16 15:22:00,022][lm_eval.evaluator_utils][INFO] -     Task: mmlu_public_relations (mmlu/default/mmlu_public_relations.yaml)
+[2026-06-16 15:22:00,023][lm_eval.evaluator_utils][INFO] -     Task: mmlu_security_studies (mmlu/default/mmlu_security_studies.yaml)
+[2026-06-16 15:22:00,023][lm_eval.evaluator_utils][INFO] -     Task: mmlu_sociology (mmlu/default/mmlu_sociology.yaml)
+[2026-06-16 15:22:00,024][lm_eval.evaluator_utils][INFO] -     Task: mmlu_us_foreign_policy (mmlu/default/mmlu_us_foreign_policy.yaml)
+[2026-06-16 15:22:00,025][lm_eval.evaluator_utils][INFO] -   Group: mmlu_humanities
+[2026-06-16 15:22:00,025][lm_eval.evaluator_utils][INFO] -     Task: mmlu_formal_logic (mmlu/default/mmlu_formal_logic.yaml)
+[2026-06-16 15:22:00,026][lm_eval.evaluator_utils][INFO] -     Task: mmlu_high_school_european_history (mmlu/default/mmlu_high_school_european_history.yaml)
+[2026-06-16 15:22:00,026][lm_eval.evaluator_utils][INFO] -     Task: mmlu_high_school_us_history (mmlu/default/mmlu_high_school_us_history.yaml)
+[2026-06-16 15:22:00,027][lm_eval.evaluator_utils][INFO] -     Task: mmlu_high_school_world_history (mmlu/default/mmlu_high_school_world_history.yaml)
+[2026-06-16 15:22:00,028][lm_eval.evaluator_utils][INFO] -     Task: mmlu_international_law (mmlu/default/mmlu_international_law.yaml)
+[2026-06-16 15:22:00,028][lm_eval.evaluator_utils][INFO] -     Task: mmlu_jurisprudence (mmlu/default/mmlu_jurisprudence.yaml)
+[2026-06-16 15:22:00,029][lm_eval.evaluator_utils][INFO] -     Task: mmlu_logical_fallacies (mmlu/default/mmlu_logical_fallacies.yaml)
+[2026-06-16 15:22:00,029][lm_eval.evaluator_utils][INFO] -     Task: mmlu_moral_disputes (mmlu/default/mmlu_moral_disputes.yaml)
+[2026-06-16 15:22:00,030][lm_eval.evaluator_utils][INFO] -     Task: mmlu_moral_scenarios (mmlu/default/mmlu_moral_scenarios.yaml)
+[2026-06-16 15:22:00,031][lm_eval.evaluator_utils][INFO] -     Task: mmlu_philosophy (mmlu/default/mmlu_philosophy.yaml)
+[2026-06-16 15:22:00,031][lm_eval.evaluator_utils][INFO] -     Task: mmlu_prehistory (mmlu/default/mmlu_prehistory.yaml)
+[2026-06-16 15:22:00,032][lm_eval.evaluator_utils][INFO] -     Task: mmlu_professional_law (mmlu/default/mmlu_professional_law.yaml)
+[2026-06-16 15:22:00,033][lm_eval.evaluator_utils][INFO] -     Task: mmlu_world_religions (mmlu/default/mmlu_world_religions.yaml)
+[2026-06-16 15:22:00,033][lm_eval.evaluator_utils][INFO] - Group: multilingual_unlearning
+[2026-06-16 15:22:00,034][lm_eval.evaluator_utils][INFO] -   Task: multilingual_unlearning_harry_potter_en (multilingual_unlearning/multilingual_unlearning_harry_potter_en.yaml)
+[2026-06-16 15:22:00,035][lm_eval.evaluator_utils][INFO] -   Task: multilingual_unlearning_harry_potter_fr (multilingual_unlearning/multilingual_unlearning_harry_potter_fr.yaml)
+[2026-06-16 15:22:00,035][lm_eval.evaluator_utils][INFO] -   Task: multilingual_unlearning_harry_potter_ja (multilingual_unlearning/multilingual_unlearning_harry_potter_ja.yaml)
+[2026-06-16 15:22:00,036][lm_eval.evaluator_utils][INFO] -   Task: multilingual_unlearning_harry_potter_vi (multilingual_unlearning/multilingual_unlearning_harry_potter_vi.yaml)
+[2026-06-16 15:22:00,036][lm_eval.evaluator_utils][INFO] -   Task: multilingual_unlearning_harry_potter_zh (multilingual_unlearning/multilingual_unlearning_harry_potter_zh.yaml)
+[2026-06-16 15:22:00,037][lm_eval.evaluator_utils][INFO] -   Task: multilingual_unlearning_harry_potter_ko (multilingual_unlearning/multilingual_unlearning_harry_potter_ko.yaml)
+[2026-06-16 15:22:00,038][lm_eval.evaluator_utils][INFO] -   Task: multilingual_unlearning_harry_potter_es (multilingual_unlearning/multilingual_unlearning_harry_potter_es.yaml)
+[2026-06-16 15:22:00,038][lm_eval.evaluator_utils][INFO] -   Task: multilingual_unlearning_harry_potter_de (multilingual_unlearning/multilingual_unlearning_harry_potter_de.yaml)
+[2026-06-16 15:22:00,039][lm_eval.evaluator_utils][INFO] -   Task: multilingual_unlearning_harry_potter_th (multilingual_unlearning/multilingual_unlearning_harry_potter_th.yaml)
+[2026-06-16 15:22:00,045][lm_eval.api.task][INFO] - Building contexts for multilingual_unlearning_harry_potter_en on rank 0...
+[2026-06-16 15:22:07,623][lm_eval.api.task][INFO] - Building contexts for multilingual_unlearning_harry_potter_fr on rank 0...
+[2026-06-16 15:22:13,769][lm_eval.api.task][INFO] - Building contexts for multilingual_unlearning_harry_potter_ja on rank 0...
+[2026-06-16 15:22:20,282][lm_eval.api.task][INFO] - Building contexts for multilingual_unlearning_harry_potter_vi on rank 0...
+[2026-06-16 15:22:27,216][lm_eval.api.task][INFO] - Building contexts for multilingual_unlearning_harry_potter_zh on rank 0...
+[2026-06-16 15:22:33,440][lm_eval.api.task][INFO] - Building contexts for multilingual_unlearning_harry_potter_ko on rank 0...
+[2026-06-16 15:22:39,997][lm_eval.api.task][INFO] - Building contexts for multilingual_unlearning_harry_potter_es on rank 0...
+[2026-06-16 15:22:46,595][lm_eval.api.task][INFO] - Building contexts for multilingual_unlearning_harry_potter_de on rank 0...
+[2026-06-16 15:22:52,700][lm_eval.api.task][INFO] - Building contexts for multilingual_unlearning_harry_potter_th on rank 0...
+[2026-06-16 15:22:58,938][lm_eval.api.task][INFO] - Building contexts for mmlu_abstract_algebra on rank 0...
+[2026-06-16 15:22:59,046][lm_eval.api.task][INFO] - Building contexts for mmlu_anatomy on rank 0...
+[2026-06-16 15:22:59,191][lm_eval.api.task][INFO] - Building contexts for mmlu_astronomy on rank 0...
+[2026-06-16 15:22:59,354][lm_eval.api.task][INFO] - Building contexts for mmlu_college_biology on rank 0...
+[2026-06-16 15:22:59,506][lm_eval.api.task][INFO] - Building contexts for mmlu_college_chemistry on rank 0...
+[2026-06-16 15:22:59,612][lm_eval.api.task][INFO] - Building contexts for mmlu_college_computer_science on rank 0...
+[2026-06-16 15:22:59,717][lm_eval.api.task][INFO] - Building contexts for mmlu_college_mathematics on rank 0...
+[2026-06-16 15:22:59,823][lm_eval.api.task][INFO] - Building contexts for mmlu_college_physics on rank 0...
+[2026-06-16 15:22:59,930][lm_eval.api.task][INFO] - Building contexts for mmlu_computer_security on rank 0...
+[2026-06-16 15:23:00,034][lm_eval.api.task][INFO] - Building contexts for mmlu_conceptual_physics on rank 0...
+[2026-06-16 15:23:00,281][lm_eval.api.task][INFO] - Building contexts for mmlu_electrical_engineering on rank 0...
+[2026-06-16 15:23:00,434][lm_eval.api.task][INFO] - Building contexts for mmlu_elementary_mathematics on rank 0...
+[2026-06-16 15:23:00,827][lm_eval.api.task][INFO] - Building contexts for mmlu_high_school_biology on rank 0...
+[2026-06-16 15:23:01,153][lm_eval.api.task][INFO] - Building contexts for mmlu_high_school_chemistry on rank 0...
+[2026-06-16 15:23:01,372][lm_eval.api.task][INFO] - Building contexts for mmlu_high_school_computer_science on rank 0...
+[2026-06-16 15:23:01,482][lm_eval.api.task][INFO] - Building contexts for mmlu_high_school_mathematics on rank 0...
+[2026-06-16 15:23:01,763][lm_eval.api.task][INFO] - Building contexts for mmlu_high_school_physics on rank 0...
+[2026-06-16 15:23:01,920][lm_eval.api.task][INFO] - Building contexts for mmlu_high_school_statistics on rank 0...
+[2026-06-16 15:23:02,145][lm_eval.api.task][INFO] - Building contexts for mmlu_machine_learning on rank 0...
+[2026-06-16 15:23:02,262][lm_eval.api.task][INFO] - Building contexts for mmlu_business_ethics on rank 0...
+[2026-06-16 15:23:02,368][lm_eval.api.task][INFO] - Building contexts for mmlu_clinical_knowledge on rank 0...
+[2026-06-16 15:23:02,650][lm_eval.api.task][INFO] - Building contexts for mmlu_college_medicine on rank 0...
+[2026-06-16 15:23:02,832][lm_eval.api.task][INFO] - Building contexts for mmlu_global_facts on rank 0...
+[2026-06-16 15:23:02,936][lm_eval.api.task][INFO] - Building contexts for mmlu_human_aging on rank 0...
+[2026-06-16 15:23:03,169][lm_eval.api.task][INFO] - Building contexts for mmlu_management on rank 0...
+[2026-06-16 15:23:03,278][lm_eval.api.task][INFO] - Building contexts for mmlu_marketing on rank 0...
+[2026-06-16 15:23:03,524][lm_eval.api.task][INFO] - Building contexts for mmlu_medical_genetics on rank 0...
+[2026-06-16 15:23:03,629][lm_eval.api.task][INFO] - Building contexts for mmlu_miscellaneous on rank 0...
+[2026-06-16 15:23:04,443][lm_eval.api.task][INFO] - Building contexts for mmlu_nutrition on rank 0...
+[2026-06-16 15:23:04,764][lm_eval.api.task][INFO] - Building contexts for mmlu_professional_accounting on rank 0...
+[2026-06-16 15:23:05,060][lm_eval.api.task][INFO] - Building contexts for mmlu_professional_medicine on rank 0...
+[2026-06-16 15:23:05,345][lm_eval.api.task][INFO] - Building contexts for mmlu_virology on rank 0...
+[2026-06-16 15:23:05,518][lm_eval.api.task][INFO] - Building contexts for mmlu_econometrics on rank 0...
+[2026-06-16 15:23:05,639][lm_eval.api.task][INFO] - Building contexts for mmlu_high_school_geography on rank 0...
+[2026-06-16 15:23:05,845][lm_eval.api.task][INFO] - Building contexts for mmlu_high_school_government_and_politics on rank 0...
+[2026-06-16 15:23:06,047][lm_eval.api.task][INFO] - Building contexts for mmlu_high_school_macroeconomics on rank 0...
+[2026-06-16 15:23:06,455][lm_eval.api.task][INFO] - Building contexts for mmlu_high_school_microeconomics on rank 0...
+[2026-06-16 15:23:06,703][lm_eval.api.task][INFO] - Building contexts for mmlu_high_school_psychology on rank 0...
+[2026-06-16 15:23:07,270][lm_eval.api.task][INFO] - Building contexts for mmlu_human_sexuality on rank 0...
+[2026-06-16 15:23:07,408][lm_eval.api.task][INFO] - Building contexts for mmlu_professional_psychology on rank 0...
+[2026-06-16 15:23:08,060][lm_eval.api.task][INFO] - Building contexts for mmlu_public_relations on rank 0...
+[2026-06-16 15:23:08,177][lm_eval.api.task][INFO] - Building contexts for mmlu_security_studies on rank 0...
+[2026-06-16 15:23:08,436][lm_eval.api.task][INFO] - Building contexts for mmlu_sociology on rank 0...
+[2026-06-16 15:23:08,647][lm_eval.api.task][INFO] - Building contexts for mmlu_us_foreign_policy on rank 0...
+[2026-06-16 15:23:08,753][lm_eval.api.task][INFO] - Building contexts for mmlu_formal_logic on rank 0...
+[2026-06-16 15:23:08,885][lm_eval.api.task][INFO] - Building contexts for mmlu_high_school_european_history on rank 0...
+[2026-06-16 15:23:09,062][lm_eval.api.task][INFO] - Building contexts for mmlu_high_school_us_history on rank 0...
+[2026-06-16 15:23:09,276][lm_eval.api.task][INFO] - Building contexts for mmlu_high_school_world_history on rank 0...
+[2026-06-16 15:23:09,525][lm_eval.api.task][INFO] - Building contexts for mmlu_international_law on rank 0...
+[2026-06-16 15:23:09,654][lm_eval.api.task][INFO] - Building contexts for mmlu_jurisprudence on rank 0...
+[2026-06-16 15:23:09,770][lm_eval.api.task][INFO] - Building contexts for mmlu_logical_fallacies on rank 0...
+[2026-06-16 15:23:09,942][lm_eval.api.task][INFO] - Building contexts for mmlu_moral_disputes on rank 0...
+[2026-06-16 15:23:10,303][lm_eval.api.task][INFO] - Building contexts for mmlu_moral_scenarios on rank 0...
+[2026-06-16 15:23:11,227][lm_eval.api.task][INFO] - Building contexts for mmlu_philosophy on rank 0...
+[2026-06-16 15:23:11,553][lm_eval.api.task][INFO] - Building contexts for mmlu_prehistory on rank 0...
+[2026-06-16 15:23:12,493][lm_eval.api.task][INFO] - Building contexts for mmlu_professional_law on rank 0...
+[2026-06-16 15:23:14,116][lm_eval.api.task][INFO] - Building contexts for mmlu_world_religions on rank 0...
+[2026-06-16 15:23:14,295][lm_eval.evaluator][INFO] - Running loglikelihood requests

README.md

@@ -0,0 +1,206 @@
+---
+base_model: Qwen/Qwen2.5-7B
+library_name: peft
+tags:
+- base_model:adapter:Qwen/Qwen2.5-7B
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1

adapter_config.json

@@ -0,0 +1,49 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": {
+    "base_model_class": "Qwen2ForCausalLM",
+    "parent_library": "transformers.models.qwen2.modeling_qwen2"
+  },
+  "base_model_name_or_path": "Qwen/Qwen2.5-7B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": [
+    5,
+    6,
+    7
+  ],
+  "loftq_config": {},
+  "lora_alpha": 8,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 128,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "down_proj"
+  ],
+  "target_parameters": null,
+  "task_type": null,
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43f267512656f41c56522c9cab8de41e598caba295823c9db9776764e82d1f17
+size 34603840

chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoint-1634/evals/LMEval_SUMMARY.json

@@ -0,0 +1,524 @@
+{
+    "mmlu": {
+        "alias": "mmlu",
+        "name": "mmlu",
+        "sample_len": 14042,
+        "acc,none": 0.7038883349950149,
+        "acc_stderr,none": 0.003567195594219384,
+        "sample_count": {
+            "acc,none": 14042
+        }
+    },
+    "mmlu_abstract_algebra": {
+        "name": "mmlu_abstract_algebra",
+        "alias": "abstract_algebra",
+        "sample_len": 100,
+        "acc,none": 0.58,
+        "acc_stderr,none": 0.049604496374885836
+    },
+    "mmlu_anatomy": {
+        "name": "mmlu_anatomy",
+        "alias": "anatomy",
+        "sample_len": 135,
+        "acc,none": 0.7333333333333333,
+        "acc_stderr,none": 0.038201699145179055
+    },
+    "mmlu_astronomy": {
+        "name": "mmlu_astronomy",
+        "alias": "astronomy",
+        "sample_len": 152,
+        "acc,none": 0.8092105263157895,
+        "acc_stderr,none": 0.031975658210325
+    },
+    "mmlu_business_ethics": {
+        "name": "mmlu_business_ethics",
+        "alias": "business_ethics",
+        "sample_len": 100,
+        "acc,none": 0.76,
+        "acc_stderr,none": 0.04292346959909281
+    },
+    "mmlu_clinical_knowledge": {
+        "name": "mmlu_clinical_knowledge",
+        "alias": "clinical_knowledge",
+        "sample_len": 265,
+        "acc,none": 0.7849056603773585,
+        "acc_stderr,none": 0.025288394502891363
+    },
+    "mmlu_college_biology": {
+        "name": "mmlu_college_biology",
+        "alias": "college_biology",
+        "sample_len": 144,
+        "acc,none": 0.8402777777777778,
+        "acc_stderr,none": 0.030635578972093278
+    },
+    "mmlu_college_chemistry": {
+        "name": "mmlu_college_chemistry",
+        "alias": "college_chemistry",
+        "sample_len": 100,
+        "acc,none": 0.52,
+        "acc_stderr,none": 0.050211673156867795
+    },
+    "mmlu_college_computer_science": {
+        "name": "mmlu_college_computer_science",
+        "alias": "college_computer_science",
+        "sample_len": 100,
+        "acc,none": 0.7,
+        "acc_stderr,none": 0.046056618647183814
+    },
+    "mmlu_college_mathematics": {
+        "name": "mmlu_college_mathematics",
+        "alias": "college_mathematics",
+        "sample_len": 100,
+        "acc,none": 0.46,
+        "acc_stderr,none": 0.05009082659620333
+    },
+    "mmlu_college_medicine": {
+        "name": "mmlu_college_medicine",
+        "alias": "college_medicine",
+        "sample_len": 173,
+        "acc,none": 0.7052023121387283,
+        "acc_stderr,none": 0.03476599607516478
+    },
+    "mmlu_college_physics": {
+        "name": "mmlu_college_physics",
+        "alias": "college_physics",
+        "sample_len": 102,
+        "acc,none": 0.5,
+        "acc_stderr,none": 0.04975185951049946
+    },
+    "mmlu_computer_security": {
+        "name": "mmlu_computer_security",
+        "alias": "computer_security",
+        "sample_len": 100,
+        "acc,none": 0.85,
+        "acc_stderr,none": 0.03588702812826371
+    },
+    "mmlu_conceptual_physics": {
+        "name": "mmlu_conceptual_physics",
+        "alias": "conceptual_physics",
+        "sample_len": 235,
+        "acc,none": 0.7148936170212766,
+        "acc_stderr,none": 0.029513196625539345
+    },
+    "mmlu_econometrics": {
+        "name": "mmlu_econometrics",
+        "alias": "econometrics",
+        "sample_len": 114,
+        "acc,none": 0.6140350877192983,
+        "acc_stderr,none": 0.04579639422070435
+    },
+    "mmlu_electrical_engineering": {
+        "name": "mmlu_electrical_engineering",
+        "alias": "electrical_engineering",
+        "sample_len": 145,
+        "acc,none": 0.7310344827586207,
+        "acc_stderr,none": 0.036951833116502325
+    },
+    "mmlu_elementary_mathematics": {
+        "name": "mmlu_elementary_mathematics",
+        "alias": "elementary_mathematics",
+        "sample_len": 378,
+        "acc,none": 0.7275132275132276,
+        "acc_stderr,none": 0.022930973071633356
+    },
+    "mmlu_formal_logic": {
+        "name": "mmlu_formal_logic",
+        "alias": "formal_logic",
+        "sample_len": 126,
+        "acc,none": 0.5476190476190477,
+        "acc_stderr,none": 0.044518079590553275
+    },
+    "mmlu_global_facts": {
+        "name": "mmlu_global_facts",
+        "alias": "global_facts",
+        "sample_len": 100,
+        "acc,none": 0.45,
+        "acc_stderr,none": 0.049999999999999996
+    },
+    "mmlu_high_school_biology": {
+        "name": "mmlu_high_school_biology",
+        "alias": "high_school_biology",
+        "sample_len": 310,
+        "acc,none": 0.8419354838709677,
+        "acc_stderr,none": 0.020752831511875278
+    },
+    "mmlu_high_school_chemistry": {
+        "name": "mmlu_high_school_chemistry",
+        "alias": "high_school_chemistry",
+        "sample_len": 203,
+        "acc,none": 0.6699507389162561,
+        "acc_stderr,none": 0.03308530426228258
+    },
+    "mmlu_high_school_computer_science": {
+        "name": "mmlu_high_school_computer_science",
+        "alias": "high_school_computer_science",
+        "sample_len": 100,
+        "acc,none": 0.82,
+        "acc_stderr,none": 0.038612291966536934
+    },
+    "mmlu_high_school_european_history": {
+        "name": "mmlu_high_school_european_history",
+        "alias": "high_school_european_history",
+        "sample_len": 165,
+        "acc,none": 0.7757575757575758,
+        "acc_stderr,none": 0.03256866661681102
+    },
+    "mmlu_high_school_geography": {
+        "name": "mmlu_high_school_geography",
+        "alias": "high_school_geography",
+        "sample_len": 198,
+        "acc,none": 0.8939393939393939,
+        "acc_stderr,none": 0.021938047738853102
+    },
+    "mmlu_high_school_government_and_politics": {
+        "name": "mmlu_high_school_government_and_politics",
+        "alias": "high_school_government_and_politics",
+        "sample_len": 193,
+        "acc,none": 0.9326424870466321,
+        "acc_stderr,none": 0.018088393839078898
+    },
+    "mmlu_high_school_macroeconomics": {
+        "name": "mmlu_high_school_macroeconomics",
+        "alias": "high_school_macroeconomics",
+        "sample_len": 390,
+        "acc,none": 0.7743589743589744,
+        "acc_stderr,none": 0.021193632525148522
+    },
+    "mmlu_high_school_mathematics": {
+        "name": "mmlu_high_school_mathematics",
+        "alias": "high_school_mathematics",
+        "sample_len": 270,
+        "acc,none": 0.5666666666666667,
+        "acc_stderr,none": 0.030213340289237927
+    },
+    "mmlu_high_school_microeconomics": {
+        "name": "mmlu_high_school_microeconomics",
+        "alias": "high_school_microeconomics",
+        "sample_len": 238,
+        "acc,none": 0.8907563025210085,
+        "acc_stderr,none": 0.020262987400605354
+    },
+    "mmlu_high_school_physics": {
+        "name": "mmlu_high_school_physics",
+        "alias": "high_school_physics",
+        "sample_len": 151,
+        "acc,none": 0.5629139072847682,
+        "acc_stderr,none": 0.04050035722230636
+    },
+    "mmlu_high_school_psychology": {
+        "name": "mmlu_high_school_psychology",
+        "alias": "high_school_psychology",
+        "sample_len": 545,
+        "acc,none": 0.8844036697247707,
+        "acc_stderr,none": 0.013708749534172645
+    },
+    "mmlu_high_school_statistics": {
+        "name": "mmlu_high_school_statistics",
+        "alias": "high_school_statistics",
+        "sample_len": 216,
+        "acc,none": 0.6944444444444444,
+        "acc_stderr,none": 0.03141554629402543
+    },
+    "mmlu_high_school_us_history": {
+        "name": "mmlu_high_school_us_history",
+        "alias": "high_school_us_history",
+        "sample_len": 204,
+        "acc,none": 0.8725490196078431,
+        "acc_stderr,none": 0.023405530480846308
+    },
+    "mmlu_high_school_world_history": {
+        "name": "mmlu_high_school_world_history",
+        "alias": "high_school_world_history",
+        "sample_len": 237,
+        "acc,none": 0.7890295358649789,
+        "acc_stderr,none": 0.02655837250266192
+    },
+    "mmlu_human_aging": {
+        "name": "mmlu_human_aging",
+        "alias": "human_aging",
+        "sample_len": 223,
+        "acc,none": 0.7085201793721974,
+        "acc_stderr,none": 0.030500283176545843
+    },
+    "mmlu_human_sexuality": {
+        "name": "mmlu_human_sexuality",
+        "alias": "human_sexuality",
+        "sample_len": 131,
+        "acc,none": 0.816793893129771,
+        "acc_stderr,none": 0.03392770926494733
+    },
+    "mmlu_humanities": {
+        "alias": "humanities",
+        "name": "mmlu_humanities",
+        "sample_len": 4705,
+        "acc,none": 0.5957492029755579,
+        "acc_stderr,none": 0.0064548686587863596,
+        "sample_count": {
+            "acc,none": 4705
+        }
+    },
+    "mmlu_international_law": {
+        "name": "mmlu_international_law",
+        "alias": "international_law",
+        "sample_len": 121,
+        "acc,none": 0.8677685950413223,
+        "acc_stderr,none": 0.030922788320445795
+    },
+    "mmlu_jurisprudence": {
+        "name": "mmlu_jurisprudence",
+        "alias": "jurisprudence",
+        "sample_len": 108,
+        "acc,none": 0.8333333333333334,
+        "acc_stderr,none": 0.036028141763926436
+    },
+    "mmlu_logical_fallacies": {
+        "name": "mmlu_logical_fallacies",
+        "alias": "logical_fallacies",
+        "sample_len": 163,
+        "acc,none": 0.803680981595092,
+        "acc_stderr,none": 0.031207970394709218
+    },
+    "mmlu_machine_learning": {
+        "name": "mmlu_machine_learning",
+        "alias": "machine_learning",
+        "sample_len": 112,
+        "acc,none": 0.5714285714285714,
+        "acc_stderr,none": 0.04697113923010213
+    },
+    "mmlu_management": {
+        "name": "mmlu_management",
+        "alias": "management",
+        "sample_len": 103,
+        "acc,none": 0.9029126213592233,
+        "acc_stderr,none": 0.02931596291881347
+    },
+    "mmlu_marketing": {
+        "name": "mmlu_marketing",
+        "alias": "marketing",
+        "sample_len": 234,
+        "acc,none": 0.9401709401709402,
+        "acc_stderr,none": 0.015537514263253902
+    },
+    "mmlu_medical_genetics": {
+        "name": "mmlu_medical_genetics",
+        "alias": "medical_genetics",
+        "sample_len": 100,
+        "acc,none": 0.88,
+        "acc_stderr,none": 0.03265986323710906
+    },
+    "mmlu_miscellaneous": {
+        "name": "mmlu_miscellaneous",
+        "alias": "miscellaneous",
+        "sample_len": 783,
+        "acc,none": 0.8544061302681992,
+        "acc_stderr,none": 0.012612475800423453
+    },
+    "mmlu_moral_disputes": {
+        "name": "mmlu_moral_disputes",
+        "alias": "moral_disputes",
+        "sample_len": 346,
+        "acc,none": 0.7543352601156069,
+        "acc_stderr,none": 0.023176298203992005
+    },
+    "mmlu_moral_scenarios": {
+        "name": "mmlu_moral_scenarios",
+        "alias": "moral_scenarios",
+        "sample_len": 895,
+        "acc,none": 0.25027932960893856,
+        "acc_stderr,none": 0.014487500852850414
+    },
+    "mmlu_nutrition": {
+        "name": "mmlu_nutrition",
+        "alias": "nutrition",
+        "sample_len": 306,
+        "acc,none": 0.7908496732026143,
+        "acc_stderr,none": 0.02328768531233481
+    },
+    "mmlu_other": {
+        "alias": "other",
+        "name": "mmlu_other",
+        "sample_len": 3107,
+        "acc,none": 0.7640811071773415,
+        "acc_stderr,none": 0.007298524772721928,
+        "sample_count": {
+            "acc,none": 3107
+        }
+    },
+    "mmlu_philosophy": {
+        "name": "mmlu_philosophy",
+        "alias": "philosophy",
+        "sample_len": 311,
+        "acc,none": 0.7459807073954984,
+        "acc_stderr,none": 0.024723861504771686
+    },
+    "mmlu_prehistory": {
+        "name": "mmlu_prehistory",
+        "alias": "prehistory",
+        "sample_len": 324,
+        "acc,none": 0.8179012345679012,
+        "acc_stderr,none": 0.021473491834808338
+    },
+    "mmlu_professional_accounting": {
+        "name": "mmlu_professional_accounting",
+        "alias": "professional_accounting",
+        "sample_len": 282,
+        "acc,none": 0.5638297872340425,
+        "acc_stderr,none": 0.029583452036284062
+    },
+    "mmlu_professional_law": {
+        "name": "mmlu_professional_law",
+        "alias": "professional_law",
+        "sample_len": 1534,
+        "acc,none": 0.5123859191655802,
+        "acc_stderr,none": 0.012766317315473551
+    },
+    "mmlu_professional_medicine": {
+        "name": "mmlu_professional_medicine",
+        "alias": "professional_medicine",
+        "sample_len": 272,
+        "acc,none": 0.75,
+        "acc_stderr,none": 0.026303648393696036
+    },
+    "mmlu_professional_psychology": {
+        "name": "mmlu_professional_psychology",
+        "alias": "professional_psychology",
+        "sample_len": 612,
+        "acc,none": 0.7598039215686274,
+        "acc_stderr,none": 0.017282760695167425
+    },
+    "mmlu_public_relations": {
+        "name": "mmlu_public_relations",
+        "alias": "public_relations",
+        "sample_len": 110,
+        "acc,none": 0.6636363636363637,
+        "acc_stderr,none": 0.04525393596302505
+    },
+    "mmlu_security_studies": {
+        "name": "mmlu_security_studies",
+        "alias": "security_studies",
+        "sample_len": 245,
+        "acc,none": 0.7673469387755102,
+        "acc_stderr,none": 0.02704925791589618
+    },
+    "mmlu_social_sciences": {
+        "alias": "social sciences",
+        "name": "mmlu_social_sciences",
+        "sample_len": 3077,
+        "acc,none": 0.8196295092622684,
+        "acc_stderr,none": 0.006794988989850078,
+        "sample_count": {
+            "acc,none": 3077
+        }
+    },
+    "mmlu_sociology": {
+        "name": "mmlu_sociology",
+        "alias": "sociology",
+        "sample_len": 201,
+        "acc,none": 0.8805970149253731,
+        "acc_stderr,none": 0.02292879327721974
+    },
+    "mmlu_stem": {
+        "alias": "stem",
+        "name": "mmlu_stem",
+        "sample_len": 3153,
+        "acc,none": 0.6929908024104028,
+        "acc_stderr,none": 0.007994591646954078,
+        "sample_count": {
+            "acc,none": 3153
+        }
+    },
+    "mmlu_us_foreign_policy": {
+        "name": "mmlu_us_foreign_policy",
+        "alias": "us_foreign_policy",
+        "sample_len": 100,
+        "acc,none": 0.89,
+        "acc_stderr,none": 0.03144660377352203
+    },
+    "mmlu_virology": {
+        "name": "mmlu_virology",
+        "alias": "virology",
+        "sample_len": 166,
+        "acc,none": 0.5421686746987951,
+        "acc_stderr,none": 0.038786267710023595
+    },
+    "mmlu_world_religions": {
+        "name": "mmlu_world_religions",
+        "alias": "world_religions",
+        "sample_len": 171,
+        "acc,none": 0.8596491228070176,
+        "acc_stderr,none": 0.026640582539133196
+    },
+    "multilingual_unlearning": {
+        "alias": "multilingual_unlearning",
+        "name": "multilingual_unlearning",
+        "sample_len": 64656,
+        "acc,none": 0.2682968324672111,
+        "acc_stderr,none": 0.0017394878315877683,
+        "sample_count": {
+            "acc,none": 64656
+        }
+    },
+    "multilingual_unlearning_harry_potter_de": {
+        "name": "multilingual_unlearning_harry_potter_de",
+        "alias": "multilingual_unlearning_harry_potter_de",
+        "sample_len": 7184,
+        "acc,none": 0.2611358574610245,
+        "acc_stderr,none": 0.005182778127072039
+    },
+    "multilingual_unlearning_harry_potter_en": {
+        "name": "multilingual_unlearning_harry_potter_en",
+        "alias": "multilingual_unlearning_harry_potter_en",
+        "sample_len": 7184,
+        "acc,none": 0.33964365256124723,
+        "acc_stderr,none": 0.005587891907054875
+    },
+    "multilingual_unlearning_harry_potter_es": {
+        "name": "multilingual_unlearning_harry_potter_es",
+        "alias": "multilingual_unlearning_harry_potter_es",
+        "sample_len": 7184,
+        "acc,none": 0.25,
+        "acc_stderr,none": 0.0051091388175553976
+    },
+    "multilingual_unlearning_harry_potter_fr": {
+        "name": "multilingual_unlearning_harry_potter_fr",
+        "alias": "multilingual_unlearning_harry_potter_fr",
+        "sample_len": 7184,
+        "acc,none": 0.25125278396436523,
+        "acc_stderr,none": 0.005117644550585829
+    },
+    "multilingual_unlearning_harry_potter_ja": {
+        "name": "multilingual_unlearning_harry_potter_ja",
+        "alias": "multilingual_unlearning_harry_potter_ja",
+        "sample_len": 7184,
+        "acc,none": 0.2570991091314031,
+        "acc_stderr,none": 0.00515659225811454
+    },
+    "multilingual_unlearning_harry_potter_ko": {
+        "name": "multilingual_unlearning_harry_potter_ko",
+        "alias": "multilingual_unlearning_harry_potter_ko",
+        "sample_len": 7184,
+        "acc,none": 0.2612750556792873,
+        "acc_stderr,none": 0.005183670922970975
+    },
+    "multilingual_unlearning_harry_potter_th": {
+        "name": "multilingual_unlearning_harry_potter_th",
+        "alias": "multilingual_unlearning_harry_potter_th",
+        "sample_len": 7184,
+        "acc,none": 0.2562639198218263,
+        "acc_stderr,none": 0.005151102886970967
+    },
+    "multilingual_unlearning_harry_potter_vi": {
+        "name": "multilingual_unlearning_harry_potter_vi",
+        "alias": "multilingual_unlearning_harry_potter_vi",
+        "sample_len": 7184,
+        "acc,none": 0.25793429844098,
+        "acc_stderr,none": 0.005162056979607864
+    },
+    "multilingual_unlearning_harry_potter_zh": {
+        "name": "multilingual_unlearning_harry_potter_zh",
+        "alias": "multilingual_unlearning_harry_potter_zh",
+        "sample_len": 7184,
+        "acc,none": 0.28006681514476617,
+        "acc_stderr,none": 0.005298146823752324
+    }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

tokenizer_config.json

@@ -0,0 +1,30 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}