Release reproduced country-specific SCPO DeBERTa checkpoint

LICENSE

@@ -0,0 +1,408 @@
+Attribution-NonCommercial 4.0 International
+
+=======================================================================
+
+Creative Commons Corporation ("Creative Commons") is not a law firm and
+does not provide legal services or legal advice. Distribution of
+Creative Commons public licenses does not create a lawyer-client or
+other relationship. Creative Commons makes its licenses and related
+information available on an "as-is" basis. Creative Commons gives no
+warranties regarding its licenses, any material licensed under their
+terms and conditions, or any related information. Creative Commons
+disclaims all liability for damages resulting from their use to the
+fullest extent possible.
+
+Using Creative Commons Public Licenses
+
+Creative Commons public licenses provide a standard set of terms and
+conditions that creators and other rights holders may use to share
+original works of authorship and other material subject to copyright
+and certain other rights specified in the public license below. The
+following considerations are for informational purposes only, are not
+exhaustive, and do not form part of our licenses.
+
+     Considerations for licensors: Our public licenses are
+     intended for use by those authorized to give the public
+     permission to use material in ways otherwise restricted by
+     copyright and certain other rights. Our licenses are
+     irrevocable. Licensors should read and understand the terms
+     and conditions of the license they choose before applying it.
+     Licensors should also secure all rights necessary before
+     applying our licenses so that the public can reuse the
+     material as expected. Licensors should clearly mark any
+     material not subject to the license. This includes other CC-
+     licensed material, or material used under an exception or
+     limitation to copyright. More considerations for licensors:
+    wiki.creativecommons.org/Considerations_for_licensors
+
+     Considerations for the public: By using one of our public
+     licenses, a licensor grants the public permission to use the
+     licensed material under specified terms and conditions. If
+     the licensor's permission is not necessary for any reason--for
+     example, because of any applicable exception or limitation to
+     copyright--then that use is not regulated by the license. Our
+     licenses grant only permissions under copyright and certain
+     other rights that a licensor has authority to grant. Use of
+     the licensed material may still be restricted for other
+     reasons, including because others have copyright or other
+     rights in the material. A licensor may make special requests,
+     such as asking that all changes be marked or described.
+     Although not required by our licenses, you are encouraged to
+     respect those requests where reasonable. More considerations
+     for the public:
+    wiki.creativecommons.org/Considerations_for_licensees
+
+=======================================================================
+
+Creative Commons Attribution-NonCommercial 4.0 International Public
+License
+
+By exercising the Licensed Rights (defined below), You accept and agree
+to be bound by the terms and conditions of this Creative Commons
+Attribution-NonCommercial 4.0 International Public License ("Public
+License"). To the extent this Public License may be interpreted as a
+contract, You are granted the Licensed Rights in consideration of Your
+acceptance of these terms and conditions, and the Licensor grants You
+such rights in consideration of benefits the Licensor receives from
+making the Licensed Material available under these terms and
+conditions.
+
+
+Section 1 -- Definitions.
+
+  a. Adapted Material means material subject to Copyright and Similar
+     Rights that is derived from or based upon the Licensed Material
+     and in which the Licensed Material is translated, altered,
+     arranged, transformed, or otherwise modified in a manner requiring
+     permission under the Copyright and Similar Rights held by the
+     Licensor. For purposes of this Public License, where the Licensed
+     Material is a musical work, performance, or sound recording,
+     Adapted Material is always produced where the Licensed Material is
+     synched in timed relation with a moving image.
+
+  b. Adapter's License means the license You apply to Your Copyright
+     and Similar Rights in Your contributions to Adapted Material in
+     accordance with the terms and conditions of this Public License.
+
+  c. Copyright and Similar Rights means copyright and/or similar rights
+     closely related to copyright including, without limitation,
+     performance, broadcast, sound recording, and Sui Generis Database
+     Rights, without regard to how the rights are labeled or
+     categorized. For purposes of this Public License, the rights
+     specified in Section 2(b)(1)-(2) are not Copyright and Similar
+     Rights.
+  d. Effective Technological Measures means those measures that, in the
+     absence of proper authority, may not be circumvented under laws
+     fulfilling obligations under Article 11 of the WIPO Copyright
+     Treaty adopted on December 20, 1996, and/or similar international
+     agreements.
+
+  e. Exceptions and Limitations means fair use, fair dealing, and/or
+     any other exception or limitation to Copyright and Similar Rights
+     that applies to Your use of the Licensed Material.
+
+  f. Licensed Material means the artistic or literary work, database,
+     or other material to which the Licensor applied this Public
+     License.
+
+  g. Licensed Rights means the rights granted to You subject to the
+     terms and conditions of this Public License, which are limited to
+     all Copyright and Similar Rights that apply to Your use of the
+     Licensed Material and that the Licensor has authority to license.
+
+  h. Licensor means the individual(s) or entity(ies) granting rights
+     under this Public License.
+
+  i. NonCommercial means not primarily intended for or directed towards
+     commercial advantage or monetary compensation. For purposes of
+     this Public License, the exchange of the Licensed Material for
+     other material subject to Copyright and Similar Rights by digital
+     file-sharing or similar means is NonCommercial provided there is
+     no payment of monetary compensation in connection with the
+     exchange.
+
+  j. Share means to provide material to the public by any means or
+     process that requires permission under the Licensed Rights, such
+     as reproduction, public display, public performance, distribution,
+     dissemination, communication, or importation, and to make material
+     available to the public including in ways that members of the
+     public may access the material from a place and at a time
+     individually chosen by them.
+
+  k. Sui Generis Database Rights means rights other than copyright
+     resulting from Directive 96/9/EC of the European Parliament and of
+     the Council of 11 March 1996 on the legal protection of databases,
+     as amended and/or succeeded, as well as other essentially
+     equivalent rights anywhere in the world.
+
+  l. You means the individual or entity exercising the Licensed Rights
+     under this Public License. Your has a corresponding meaning.
+
+
+Section 2 -- Scope.
+
+  a. License grant.
+
+       1. Subject to the terms and conditions of this Public License,
+          the Licensor hereby grants You a worldwide, royalty-free,
+          non-sublicensable, non-exclusive, irrevocable license to
+          exercise the Licensed Rights in the Licensed Material to:
+
+            a. reproduce and Share the Licensed Material, in whole or
+               in part, for NonCommercial purposes only; and
+
+            b. produce, reproduce, and Share Adapted Material for
+               NonCommercial purposes only.
+
+       2. Exceptions and Limitations. For the avoidance of doubt, where
+          Exceptions and Limitations apply to Your use, this Public
+          License does not apply, and You do not need to comply with
+          its terms and conditions.
+
+       3. Term. The term of this Public License is specified in Section
+          6(a).
+
+       4. Media and formats; technical modifications allowed. The
+          Licensor authorizes You to exercise the Licensed Rights in
+          all media and formats whether now known or hereafter created,
+          and to make technical modifications necessary to do so. The
+          Licensor waives and/or agrees not to assert any right or
+          authority to forbid You from making technical modifications
+          necessary to exercise the Licensed Rights, including
+          technical modifications necessary to circumvent Effective
+          Technological Measures. For purposes of this Public License,
+          simply making modifications authorized by this Section 2(a)
+          (4) never produces Adapted Material.
+
+       5. Downstream recipients.
+
+            a. Offer from the Licensor -- Licensed Material. Every
+               recipient of the Licensed Material automatically
+               receives an offer from the Licensor to exercise the
+               Licensed Rights under the terms and conditions of this
+               Public License.
+
+            b. No downstream restrictions. You may not offer or impose
+               any additional or different terms or conditions on, or
+               apply any Effective Technological Measures to, the
+               Licensed Material if doing so restricts exercise of the
+               Licensed Rights by any recipient of the Licensed
+               Material.
+
+       6. No endorsement. Nothing in this Public License constitutes or
+          may be construed as permission to assert or imply that You
+          are, or that Your use of the Licensed Material is, connected
+          with, or sponsored, endorsed, or granted official status by,
+          the Licensor or others designated to receive attribution as
+          provided in Section 3(a)(1)(A)(i).
+
+  b. Other rights.
+
+       1. Moral rights, such as the right of integrity, are not
+          licensed under this Public License, nor are publicity,
+          privacy, and/or other similar personality rights; however, to
+          the extent possible, the Licensor waives and/or agrees not to
+          assert any such rights held by the Licensor to the limited
+          extent necessary to allow You to exercise the Licensed
+          Rights, but not otherwise.
+
+       2. Patent and trademark rights are not licensed under this
+          Public License.
+
+       3. To the extent possible, the Licensor waives any right to
+          collect royalties from You for the exercise of the Licensed
+          Rights, whether directly or through a collecting society
+          under any voluntary or waivable statutory or compulsory
+          licensing scheme. In all other cases the Licensor expressly
+          reserves any right to collect such royalties, including when
+          the Licensed Material is used other than for NonCommercial
+          purposes.
+
+
+Section 3 -- License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the
+following conditions.
+
+  a. Attribution.
+
+       1. If You Share the Licensed Material (including in modified
+          form), You must:
+
+            a. retain the following if it is supplied by the Licensor
+               with the Licensed Material:
+
+                 i. identification of the creator(s) of the Licensed
+                    Material and any others designated to receive
+                    attribution, in any reasonable manner requested by
+                    the Licensor (including by pseudonym if
+                    designated);
+
+                ii. a copyright notice;
+
+               iii. a notice that refers to this Public License;
+
+                iv. a notice that refers to the disclaimer of
+                    warranties;
+
+                 v. a URI or hyperlink to the Licensed Material to the
+                    extent reasonably practicable;
+
+            b. indicate if You modified the Licensed Material and
+               retain an indication of any previous modifications; and
+
+            c. indicate the Licensed Material is licensed under this
+               Public License, and include the text of, or the URI or
+               hyperlink to, this Public License.
+
+       2. You may satisfy the conditions in Section 3(a)(1) in any
+          reasonable manner based on the medium, means, and context in
+          which You Share the Licensed Material. For example, it may be
+          reasonable to satisfy the conditions by providing a URI or
+          hyperlink to a resource that includes the required
+          information.
+
+       3. If requested by the Licensor, You must remove any of the
+          information required by Section 3(a)(1)(A) to the extent
+          reasonably practicable.
+
+       4. If You Share Adapted Material You produce, the Adapter's
+          License You apply must not prevent recipients of the Adapted
+          Material from complying with this Public License.
+
+
+Section 4 -- Sui Generis Database Rights.
+
+Where the Licensed Rights include Sui Generis Database Rights that
+apply to Your use of the Licensed Material:
+
+  a. for the avoidance of doubt, Section 2(a)(1) grants You the right
+     to extract, reuse, reproduce, and Share all or a substantial
+     portion of the contents of the database for NonCommercial purposes
+     only;
+
+  b. if You include all or a substantial portion of the database
+     contents in a database in which You have Sui Generis Database
+     Rights, then the database in which You have Sui Generis Database
+     Rights (but not its individual contents) is Adapted Material; and
+
+  c. You must comply with the conditions in Section 3(a) if You Share
+     all or a substantial portion of the contents of the database.
+
+For the avoidance of doubt, this Section 4 supplements and does not
+replace Your obligations under this Public License where the Licensed
+Rights include other Copyright and Similar Rights.
+
+
+Section 5 -- Disclaimer of Warranties and Limitation of Liability.
+
+  a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
+     EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
+     AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
+     ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
+     IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
+     WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
+     PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
+     ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
+     KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
+     ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
+
+  b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
+     TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
+     NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
+     INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
+     COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
+     USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
+     ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
+     DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
+     IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
+
+  c. The disclaimer of warranties and limitation of liability provided
+     above shall be interpreted in a manner that, to the extent
+     possible, most closely approximates an absolute disclaimer and
+     waiver of all liability.
+
+
+Section 6 -- Term and Termination.
+
+  a. This Public License applies for the term of the Copyright and
+     Similar Rights licensed here. However, if You fail to comply with
+     this Public License, then Your rights under this Public License
+     terminate automatically.
+
+  b. Where Your right to use the Licensed Material has terminated under
+     Section 6(a), it reinstates:
+
+       1. automatically as of the date the violation is cured, provided
+          it is cured within 30 days of Your discovery of the
+          violation; or
+
+       2. upon express reinstatement by the Licensor.
+
+     For the avoidance of doubt, this Section 6(b) does not affect any
+     right the Licensor may have to seek remedies for Your violations
+     of this Public License.
+
+  c. For the avoidance of doubt, the Licensor may also offer the
+     Licensed Material under separate terms or conditions or stop
+     distributing the Licensed Material at any time; however, doing so
+     will not terminate this Public License.
+
+  d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
+     License.
+
+
+Section 7 -- Other Terms and Conditions.
+
+  a. The Licensor shall not be bound by any additional or different
+     terms or conditions communicated by You unless expressly agreed.
+
+  b. Any arrangements, understandings, or agreements regarding the
+     Licensed Material not stated herein are separate from and
+     independent of the terms and conditions of this Public License.
+
+
+Section 8 -- Interpretation.
+
+  a. For the avoidance of doubt, this Public License does not, and
+     shall not be interpreted to, reduce, limit, restrict, or impose
+     conditions on any use of the Licensed Material that could lawfully
+     be made without permission under this Public License.
+
+  b. To the extent possible, if any provision of this Public License is
+     deemed unenforceable, it shall be automatically reformed to the
+     minimum extent necessary to make it enforceable. If the provision
+     cannot be reformed, it shall be severed from this Public License
+     without affecting the enforceability of the remaining terms and
+     conditions.
+
+  c. No term or condition of this Public License will be waived and no
+     failure to comply consented to unless expressly agreed to by the
+     Licensor.
+
+  d. Nothing in this Public License constitutes or may be interpreted
+     as a limitation upon, or waiver of, any privileges and immunities
+     that apply to the Licensor or You, including from the legal
+     processes of any jurisdiction or authority.
+
+=======================================================================
+
+Creative Commons is not a party to its public
+licenses. Notwithstanding, Creative Commons may elect to apply one of
+its public licenses to material it publishes and in those instances
+will be considered the “Licensor.” The text of the Creative Commons
+public licenses is dedicated to the public domain under the CC0 Public
+Domain Dedication. Except for the limited purpose of indicating that
+material is shared under a Creative Commons public license or as
+otherwise permitted by the Creative Commons policies published at
+creativecommons.org/policies, Creative Commons does not authorize the
+use of the trademark "Creative Commons" or any other trademark or logo
+of Creative Commons without its prior written consent including,
+without limitation, in connection with any unauthorized modifications
+to any of its public licenses or any other arrangements,
+understandings, or agreements concerning use of licensed material. For
+the avoidance of doubt, this paragraph does not form part of the
+public licenses.
+
+Creative Commons may be contacted at creativecommons.org.
+

README.md

@@ -0,0 +1,104 @@
+---
+license: cc-by-nc-4.0
+library_name: transformers
+pipeline_tag: text-classification
+base_model: OpenAssistant/reward-model-deberta-v3-base
+base_model_relation: finetune
+datasets:
+- HannahRoseKirk/prism-alignment
+language:
+- en
+tags:
+- reward-model
+- scpo
+- cultural-alignment
+- canada
+- arxiv:2606.18606
+---
+# SCPO DeBERTa-v3-base — Canada
+
+A scalar reward model fine-tuned on PRISM preference pairs from participants
+residing in Canada. Higher scores indicate a preferred response for the
+same prompt. Scores are not probabilities and are not calibrated across models.
+
+[Paper](https://arxiv.org/abs/2606.18606) ·
+[Code](https://github.com/minsik-ai/Steerable-Cultural-Preference)
+
+## Recorded experiment results
+
+| Measurement | Accuracy |
+|---|---:|
+| Starting global model, overall | 60.4046% |
+| Selected checkpoint, overall | 66.1850% |
+| Selected checkpoint, reference-misranked subset | 27.7372% |
+
+These are original in-memory post-training experiment scores, with no model-mode
+or random-state reset before scoring. A fresh `eval()` inference session disables
+dropout and may produce different accuracy. `validation.json` checks loading and
+finite, repeatable inference on generic probes; it is not a benchmark result.
+
+The country configuration was selected through exploratory trials on a fixed
+test split. It is not an unbiased held-out estimate or the paper's three-run mean.
+The independent execution rerun reproduced the selected recorded accuracy.
+The reference-misranked subset consists of pairs strictly misranked by this base
+model, excluding tied reference scores. Different backbones have different subsets.
+
+## Training and reproduction
+
+- Full-model fine-tuning from `OpenAssistant/reward-model-deberta-v3-base`.
+- Base revision: `7dab4156e63944834c3aae169c3fe83955672cff`.
+- PRISM revision: `18ab5cfb37456f4ec8cbc00212ce54cf7b1239f6`.
+- Learning rate: `1e-06`; batch size: 8; optimizer steps: 1024.
+- Filtering tau: 0.7; weighting beta: 1.1.
+- Data shuffle seed: 0; survey split/model seed: 0;
+  Trainer seed: 42; Trainer data seed: unset.
+- Survey users split 85/15; human-rating ties excluded from preference pairs.
+- Original training inputs use literal `prompt + " [SEP] " + response`, capped
+  and padded to 512 tokens. Original scoring uses tokenizer prompt/response pairs
+  in interaction batches of at most eight with no explicit truncation.
+
+See `release-settings.json`, `provenance.json`, and `requirements-pw-lock.txt`.
+The exact country script and unchanged trainer/collator are included under
+`reproduction/`. With the recorded environment installed, run:
+
+```bash
+python reproduction/preferences.py ./reproduction-output
+```
+
+Original loaders use upstream `main`; verify the recorded model/data revisions
+before rerunning. No private dataset records are bundled.
+
+## Inference
+
+The following uses standard evaluation-mode inference.
+
+```python
+import torch
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+model_id = "MulticulturalAlignment/scpo-deberta-v3-base-canada"
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = AutoModelForSequenceClassification.from_pretrained(model_id).eval()
+inputs = tokenizer("How can I disagree respectfully?",
+                   "Listen carefully and explain your reasons calmly.",
+                   return_tensors="pt")
+with torch.inference_mode():
+    reward = model(**inputs).logits.item()
+print(reward)
+```
+
+## Intended use and limitations
+
+Research on preference modeling and pluralistic alignment. Country labels reflect
+participants' residence, not a representative account of a country's culture or
+every resident's preferences. The English-dominant sample and small country test
+sets limit generalization. This reward model does not generate text and is not
+validated for consequential decisions about people.
+
+## Attribution and release license
+
+These fine-tuned weights are released under [CC-BY-NC-4.0](https://creativecommons.org/licenses/by-nc/4.0/).
+The [OpenAssistant base model](https://huggingface.co/OpenAssistant/reward-model-deberta-v3-base) is
+marked MIT. The [PRISM dataset card](https://huggingface.co/datasets/HannahRoseKirk/prism-alignment)
+lists human text as CC-BY-4.0 and model responses as CC-BY-NC-4.0, subject to
+original provider terms. This package contains no PRISM records.

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "[MASK]": 128000
+}

checksums.json

@@ -0,0 +1,19 @@
+{
+  "LICENSE": "41003d4a74749c0220e33dd415042164b5a1093ed401f36277234f772d22d3d0",
+  "README.md": "b05ba3cf9bb2cd44f29a166926cc98ff88d550e779109e1ca78ea2da847c566f",
+  "added_tokens.json": "dc046d04c9b0ada7ae6f1dc89c465801799acdf0c9a6aab8c15a1b2d5ca4e91f",
+  "config.json": "d88248219ef56b7227b25161fe83bfc1f949d77032367525b372270ef80bef17",
+  "model.safetensors": "aa23ee2ae72319bcc6351cd800c413a320856483208fc09f7e447351ba868583",
+  "provenance.json": "fd6291ee5bc555a8abbe283f6e81a7fcfe153cb5fbc18ccbc1a0d8b4e7c2a2c8",
+  "release-settings.json": "df34196ae4709cef88574e10e06b2bd6e940a7881d4ee2428219ac75c75ed4f6",
+  "requirements-pw-lock.txt": "abe2e55884496b23fc4206efbf563edc7cfc615ef37189851f6f442454e3e8bc",
+  "special_tokens_map.json": "63cd5b53789b3b5ae2c3d7aa377bcba1557ff0b41dc69ce07b94350ea32291c2",
+  "spm.model": "c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd",
+  "tokenizer.json": "b8d055f68ab07e04ecdeacf0bf506d578d826d2682b4709c94899a2f3556f583",
+  "tokenizer_config.json": "2c9737294d56f67d6122507ce309dfb8dd98221a73ed8315db8f9b07cf1988fa",
+  "validation.json": "f8ec56e8a9fa315675301294d15e14f753aa22480629349f4c31d2cff427edde",
+  "reproduction/our_data_collator.py": "67c1b081e7886ed8a5747abe7612ff255f0043d8c3d59ef64bb774db12518c79",
+  "reproduction/our_inverse_reward_trainer.py": "b41fc006ae62dfdd5553217cdb231c35d45c80dd5437f3f7c70ce006f59835a1",
+  "reproduction/our_reward_trainer.py": "35aea8a2f4b30f51e5427b61418698d69528b868fd7b35abad7132508d092ab3",
+  "reproduction/preferences.py": "16642fdfa0c817ca2b0ce33600d6478535f2994bef5c4f00d0d7b7227026b72e"
+}

config.json

@@ -0,0 +1,42 @@
+{
+  "_name_or_path": "OpenAssistant/reward-model-deberta-v3-base",
+  "architectures": [
+    "DebertaV2ForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "REWARD"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "REWARD": 0
+  },
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "problem_type": "regression",
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.45.2",
+  "type_vocab_size": 0,
+  "vocab_size": 128100
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa23ee2ae72319bcc6351cd800c413a320856483208fc09f7e447351ba868583
+size 737716196

provenance.json

@@ -0,0 +1,22 @@
+{
+  "country": "Canada",
+  "protocol": "original-source-v1",
+  "source_revision": "94f8430054835aa919b226b4344b8f0e8424f29b",
+  "original_sha256": {
+    "preferences.py": "df4c9e3991b8238c5a50a3d672d67ca3d6350f06a28a99bc6959e8edd817cd8e",
+    "our_reward_trainer.py": "35aea8a2f4b30f51e5427b61418698d69528b868fd7b35abad7132508d092ab3",
+    "our_inverse_reward_trainer.py": "b41fc006ae62dfdd5553217cdb231c35d45c80dd5437f3f7c70ce006f59835a1",
+    "our_data_collator.py": "67c1b081e7886ed8a5747abe7612ff255f0043d8c3d59ef64bb774db12518c79"
+  },
+  "generated_sha256": "16642fdfa0c817ca2b0ce33600d6478535f2994bef5c4f00d0d7b7227026b72e",
+  "base_model_revision": "7dab4156e63944834c3aae169c3fe83955672cff",
+  "dataset_revision": "18ab5cfb37456f4ec8cbc00212ce54cf7b1239f6",
+  "experiment_metrics": {
+    "overall": 66.1849710982659,
+    "global_overall": 60.40462427745664,
+    "misranked": 27.73722627737226,
+    "global_misranked": 0.0
+  },
+  "weight_sha256": "aa23ee2ae72319bcc6351cd800c413a320856483208fc09f7e447351ba868583",
+  "selection": "Per-country exploratory fixed-test selections; Israel accepted by user below prior matching threshold"
+}

release-settings.json

@@ -0,0 +1,15 @@
+{
+  "country": "Canada",
+  "shuffle_seed": 0,
+  "learning_rate": 1e-06,
+  "tau": 0.7,
+  "beta": 1.1,
+  "batch_size": 8,
+  "max_steps": 1024,
+  "method": "filtered_weighted",
+  "model_name": "OpenAssistant/reward-model-deberta-v3-base",
+  "split_seed": 0,
+  "model_seed": 0,
+  "trainer_seed": 42,
+  "trainer_data_seed": null
+}

reproduction/our_data_collator.py

@@ -0,0 +1,90 @@
+from dataclasses import dataclass
+from typing import Union, Optional, List, Dict, Any
+
+import torch
+from transformers import PreTrainedTokenizerBase
+
+
+@dataclass
+class OurRewardDataCollatorWithPadding:
+    r"""
+    Reward DataCollator class that pads the inputs to the maximum length of the batch.
+
+    Args:
+        tokenizer (`PreTrainedTokenizerBase`):
+            The tokenizer used for encoding the data.
+        padding (`Union[bool, str, `PaddingStrategy`]`, `optional`, defaults to `True`):
+            padding_strategy to pass to the tokenizer.
+        pad_to_multiple_of (`Optional[int]`, `optional`, defaults to `None`):
+            If set will pad the sequence to a multiple of the provided value.
+        return_tensors (`str`, `optional`, defaults to `"pt"`):
+            The tensor type to use.
+    """
+
+    tokenizer: PreTrainedTokenizerBase
+    padding: Union[bool, str] = True
+    max_length: Optional[int] = None
+    pad_to_multiple_of: Optional[int] = None
+    return_tensors: str = "pt"
+
+    def __call__(self, features: List[Dict[str, Any]]) -> Dict[str, Any]:
+        features_chosen = []
+        features_rejected = []
+        margin = []
+        # check if we have a margin. If we do, we need to batch it as well
+        has_margin = "margin" in features[0]
+        for feature in features:
+            # check if the keys are named as expected
+            if (
+                    "input_ids_chosen" not in feature
+                    or "input_ids_rejected" not in feature
+                    or "attention_mask_chosen" not in feature
+                    or "attention_mask_rejected" not in feature
+            ):
+                raise ValueError(
+                    "The features should include `input_ids_chosen`, `attention_mask_chosen`, `input_ids_rejected` and `attention_mask_rejected`"
+                )
+
+            features_chosen.append(
+                {
+                    "input_ids": feature["input_ids_chosen"],
+                    "attention_mask": feature["attention_mask_chosen"],
+                    "ref_rewards": feature["ref_rewards_chosen"]
+                }
+            )
+            features_rejected.append(
+                {
+                    "input_ids": feature["input_ids_rejected"],
+                    "attention_mask": feature["attention_mask_rejected"],
+                    "ref_rewards": feature["ref_rewards_rejected"]
+                }
+            )
+            if has_margin:
+                margin.append(feature["margin"])
+        batch_chosen = self.tokenizer.pad(
+            features_chosen,
+            padding=self.padding,
+            max_length=self.max_length,
+            pad_to_multiple_of=self.pad_to_multiple_of,
+            return_tensors=self.return_tensors,
+        )
+        batch_rejected = self.tokenizer.pad(
+            features_rejected,
+            padding=self.padding,
+            max_length=self.max_length,
+            pad_to_multiple_of=self.pad_to_multiple_of,
+            return_tensors=self.return_tensors,
+        )
+        batch = {
+            "input_ids_chosen": batch_chosen["input_ids"],
+            "attention_mask_chosen": batch_chosen["attention_mask"],
+            "ref_rewards_chosen": batch_chosen["ref_rewards"],
+            "input_ids_rejected": batch_rejected["input_ids"],
+            "attention_mask_rejected": batch_rejected["attention_mask"],
+            "ref_rewards_rejected": batch_rejected["ref_rewards"],
+            "return_loss": True,
+        }
+        if has_margin:
+            margin = torch.tensor(margin, dtype=torch.float)
+            batch["margin"] = margin
+        return batch

reproduction/our_inverse_reward_trainer.py

@@ -0,0 +1,57 @@
+import warnings
+from typing import Union, Dict, Any, Tuple, List, Mapping, Optional
+
+import torch
+from torch import nn
+from transformers import PreTrainedModel
+from trl import RewardTrainer
+
+
+class OurInverseRewardTrainer(RewardTrainer):
+    _tag_names = ["trl", "reward-trainer"]
+
+
+    def compute_loss(
+            self,
+            model: Union[PreTrainedModel, nn.Module],
+            inputs: Dict[str, Union[torch.Tensor, Any]],
+            return_outputs=False,
+            num_items_in_batch=None,
+    ) -> Union[torch.Tensor, Tuple[torch.Tensor, Dict[str, torch.Tensor]]]:
+        if not self.use_reward_data_collator:
+            warnings.warn(
+                "The current compute_loss is implemented for RewardDataCollatorWithPadding,"
+                " if you are using a custom data collator make sure you know what you are doing or"
+                " implement your own compute_loss method."
+            )
+        rewards_chosen = model(
+            input_ids=inputs["input_ids_chosen"],
+            attention_mask=inputs["attention_mask_chosen"],
+            return_dict=True,
+        )["logits"]
+        ref_rewards_chosen = inputs["ref_rewards_chosen"]
+        rewards_rejected = model(
+            input_ids=inputs["input_ids_rejected"],
+            attention_mask=inputs["attention_mask_rejected"],
+            return_dict=True,
+        )["logits"]
+        ref_rewards_rejected = inputs["ref_rewards_rejected"]
+        # calculate loss, optionally modulate with margin
+        # ratio = 1. + torch.exp(ref_rewards_chosen) / torch.exp(ref_rewards_rejected)
+        ratio = torch.maximum(torch.exp(ref_rewards_rejected) / torch.exp(ref_rewards_chosen), torch.ones_like(ref_rewards_chosen))
+        # ratio = torch.exp(ref_rewards_chosen) / torch.exp(ref_rewards_rejected)
+        # print(ratio)
+        if "margin" in inputs:
+            loss = -(ratio * nn.functional.logsigmoid(rewards_chosen - rewards_rejected - inputs["margin"])).mean()
+        else:
+            loss = -(ratio * nn.functional.logsigmoid(rewards_chosen - rewards_rejected)).mean()
+
+        if self.args.center_rewards_coefficient is not None:
+            loss += self.args.center_rewards_coefficient * torch.mean((rewards_chosen + rewards_rejected) ** 2)
+
+        if return_outputs:
+            return loss, {
+                "rewards_chosen": rewards_chosen,
+                "rewards_rejected": rewards_rejected,
+            }
+        return loss

reproduction/our_reward_trainer.py

@@ -0,0 +1,60 @@
+import warnings
+from typing import Union, Dict, Any, Tuple, List, Mapping, Optional
+
+import torch
+from torch import nn
+from transformers import PreTrainedModel
+from trl import RewardTrainer
+
+
+class OurRewardTrainer(RewardTrainer):
+    _tag_names = ["trl", "reward-trainer"]
+
+    def __init__(self, *args, beta: float = 1.0, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.beta = beta
+
+    def compute_loss(
+            self,
+            model: Union[PreTrainedModel, nn.Module],
+            inputs: Dict[str, Union[torch.Tensor, Any]],
+            return_outputs=False,
+            num_items_in_batch=None,
+    ) -> Union[torch.Tensor, Tuple[torch.Tensor, Dict[str, torch.Tensor]]]:
+        if not self.use_reward_data_collator:
+            warnings.warn(
+                "The current compute_loss is implemented for RewardDataCollatorWithPadding,"
+                " if you are using a custom data collator make sure you know what you are doing or"
+                " implement your own compute_loss method."
+            )
+        rewards_chosen = model(
+            input_ids=inputs["input_ids_chosen"],
+            attention_mask=inputs["attention_mask_chosen"],
+            return_dict=True,
+        )["logits"]
+        ref_rewards_chosen = inputs["ref_rewards_chosen"]
+        rewards_rejected = model(
+            input_ids=inputs["input_ids_rejected"],
+            attention_mask=inputs["attention_mask_rejected"],
+            return_dict=True,
+        )["logits"]
+        ref_rewards_rejected = inputs["ref_rewards_rejected"]
+        # calculate loss, optionally modulate with margin
+        # ratio = 1. + torch.exp(ref_rewards_chosen) / torch.exp(ref_rewards_rejected)
+        ratio = torch.minimum(torch.exp((ref_rewards_chosen - ref_rewards_rejected) / self.beta), torch.ones_like(ref_rewards_chosen))
+        # ratio = torch.exp(ref_rewards_chosen) / torch.exp(ref_rewards_rejected)
+        # print(ratio)
+        if "margin" in inputs:
+            loss = -(ratio * nn.functional.logsigmoid(rewards_chosen - rewards_rejected - inputs["margin"])).mean()
+        else:
+            loss = -(ratio * nn.functional.logsigmoid(rewards_chosen - rewards_rejected)).mean()
+
+        if self.args.center_rewards_coefficient is not None:
+            loss += self.args.center_rewards_coefficient * torch.mean((rewards_chosen + rewards_rejected) ** 2)
+
+        if return_outputs:
+            return loss, {
+                "rewards_chosen": rewards_chosen,
+                "rewards_rejected": rewards_rejected,
+            }
+        return loss

reproduction/preferences.py

@@ -0,0 +1,626 @@
+# from transformers.utils import logging
+# logging.set_verbosity_error()
+import csv
+import warnings
+
+from peft import LoraConfig, TaskType
+
+warnings.filterwarnings('ignore')
+
+import torch
+from datasets import Dataset
+from trl import RewardConfig, RewardTrainer
+import collections
+from tqdm import tqdm
+import argparse
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import os
+import numpy as np
+
+from our_data_collator import OurRewardDataCollatorWithPadding
+from our_reward_trainer import OurRewardTrainer
+# from our_inverse_reward_trainer import OurInverseRewardTrainer
+
+BATCH_SIZE = 8
+# EPOCHS = 2
+EPOCHS = 1
+LR = 1e-6
+LR_OVERRIDE = {
+}
+# LR = 1e-5
+EDITED_EPOCHS = 3
+EDITED_LR = 1e-06
+EDITED_LR_OVERRIDE = {
+}
+
+SKIP_ORIG_PARTIAL = ["South Africa", "Mexico", "Australia", "New Zealand", "Chile", "Canada", "Israel"]
+
+REWARD_THRES = 0.7
+REWARD_THRES_OVERRIDE = {
+}
+WEIGHTING_BETA = 1.1
+
+MAX_STEPS = {
+    # "United States": 1200,
+    "New Zealand": 1024,
+    "Mexico": 1024,
+    "Chile": 1024,
+    "South Africa": 1024,
+    "Israel": 1024,
+    "Canada": 1024,
+    "Australia": 1024
+}
+
+REPORT = 1000000
+
+RANDOM_DATA_SPLIT_SEED = 0
+RANDOM_DATA_SHUFFLE_SEED = 0
+RANDOM_MODEL_SEED = 0
+
+from transformers import set_seed
+set_seed(RANDOM_MODEL_SEED)
+MODEL_NAME = "OpenAssistant/reward-model-deberta-v3-base"
+# MODEL_NAME = "allenai/Llama-3.1-Tulu-3-8B-RM"
+
+
+def main(out_path):
+    print("script starting")
+    # Use reside_country
+
+    # GPO list
+    countries = ["United States", "Canada",
+                 "Germany", "France", "Spain",
+                 "Nigeria", "Egypt",
+                 "India", "China", "Japan",
+                 "Brazil", "Argentina",
+                 "Australia", "New Zealand"]
+    # Our list
+    # Max 3 countries per continent, more than 10 users.
+    # Must be also in GlobalOpinionsQA - South Africa, Korea not in the Global Opinions QA
+    # North America : United States, Canada, Mexico
+    # South America : Chile
+    # Europe: United Kingdom, Spain, Belgium
+    # Asia: Israel, Japan
+    # Australia: Australia
+    # countries = ["United States", "Mexico", "Canada",
+    #              "Chile",
+    #              "United Kingdom", "Spain", "Belgium",
+    #              "Israel", "Japan",
+    #              "Australia"]
+    # Counter({'United States': 386, 'United Kingdom': 340, 'South Africa': 86, 'New Zealand': 77, 'Australia': 72,
+    # 'Mexico': 67, 'Chile': 65, 'Israel': 61, 'Canada': 54, 'Spain': 18, 'Belgium': 17, 'Hungary': 16, 'Denmark': 15,
+    # 'Norway': 15, 'Ireland': 15, 'Poland': 14, 'Czechia': 14, 'Switzerland': 14, 'Latvia': 14, 'Germany': 13,
+    # 'Greece': 13, 'Finland': 13, 'Italy': 13, 'France': 12, 'Japan': 11, 'Slovenia': 10, 'Austria': 10, 'Estonia': 10,
+    # 'Netherlands': 8, 'Portugal': 7, 'Korea, Republic of': 7, 'Sweden': 6, 'Luxembourg': 2,
+    # 'Tanzania, United Republic of': 1, 'Uruguay': 1, 'Lesotho': 1, 'Iceland': 1, 'Prefer not to say': 1})
+
+    # Or, just top-10 responding countries with more than 20 users
+    # countries = ["United States", "United Kingdom", "South Africa", "New Zealand", "Australia", "Mexico", "Chile",
+    #              "Israel", "Canada"]
+    # countries = ["United States"]
+
+    # countries = ["New Zealand", "Mexico", "South Africa"]
+    # countries = ["South Africa"]
+    # countries = ["Australia"]
+
+    # countries = ["Japan"]
+    countries = ['Canada']
+
+    print("dataset starting")
+    from datasets import load_dataset
+    # Load datasets
+    utts = load_dataset("HannahRoseKirk/prism-alignment", "utterances", split="train")
+    infos = load_dataset("HannahRoseKirk/prism-alignment", "survey", split="train")
+
+    info_split = infos.train_test_split(test_size=0.15, seed=RANDOM_DATA_SPLIT_SEED)
+    info_split.save_to_disk(f"{out_path}/dataset/PRISM/seed/{RANDOM_DATA_SPLIT_SEED}/")
+
+    info_train = info_split["train"]
+    info_test = info_split["test"]
+
+    reside_countries = collections.Counter()
+    birth_countries = collections.Counter()
+
+    info_train_ids = collections.defaultdict(set)
+
+    # map users to countries for the training set
+    # count number of users that reside/were born in each country
+    for info in info_train:
+        country = info["location"]["reside_country"]
+        b_country = info["location"]["birth_country"]
+        reside_countries[country] += 1
+        birth_countries[b_country] += 1
+        if country in countries:
+            info_train_ids[country].add(info["user_id"])
+
+    # map users to countries for the test set
+    # count number of users that reside or were born in each country
+    info_test_ids = collections.defaultdict(set)
+    for info in info_test:
+        country = info["location"]["reside_country"]
+        b_country = info["location"]["birth_country"]
+        reside_countries[country] += 1
+        birth_countries[b_country] += 1
+        if country in countries:
+            info_test_ids[country].add(info["user_id"])
+
+    print("**** RESIDE COUNTRIES ****")
+    print(reside_countries)
+    # print(birth_countries)
+
+    full_perfs = {count: dict() for count in countries}
+    # Different from average
+    full_perfs["avg"] = dict()
+
+    partial_perfs = {count: dict() for count in countries}
+    # Different from average
+    partial_perfs["avg"] = dict()
+
+    print("**** USERS ****")
+    for country in countries:
+        print(f"{country}: train {len(info_train_ids[country])} test {len(info_test_ids[country])}")
+
+    # Keyed per user & turn
+    utt_train = {country: collections.defaultdict(list) for country in countries}
+    utt_test = {country: collections.defaultdict(list) for country in countries}
+    for utt in utts:
+        user_id = utt["user_id"]
+        turn = utt["interaction_id"]
+        for country in countries:
+            key = f"{user_id}_{turn}"
+            if user_id in info_train_ids[country]:
+                utt_train[country][key].append(utt)
+            elif user_id in info_test_ids[country]:
+                utt_test[country][key].append(utt)
+            else:
+                continue
+    print("**** Raw Utterances ****")
+    print("**** Train ****")
+    print({count: len(items) for count, items in utt_train.items()})
+    print("**** Test ****")
+    print({count: len(items) for count, items in utt_test.items()})
+
+    utt_score_train = collections.defaultdict(list)
+    utt_score_test = collections.defaultdict(list)
+
+    def format_text(chosen, rejected):
+        assert chosen["interaction_id"] == rejected["interaction_id"]
+        return {
+            "chosen": [{"content": chosen["user_prompt"], "role": "user"},
+                       {"content": chosen["model_response"], "role": "assistant"}],
+            "rejected": [{"content": rejected["user_prompt"], "role": "user"},
+                         {"content": rejected["model_response"], "role": "assistant"}],
+            "chosen_id": chosen["utterance_id"],
+            "rejected_id": rejected["utterance_id"],
+            "interaction_id": chosen["interaction_id"]
+        }
+
+    # import model and tokenizer
+    # move to GPU
+    device = "cuda:0" if torch.cuda.is_available() else "cpu"
+
+    # model predicts which generated answer is better judged by a human
+    # returns a score of how good a response is given a question (?)
+    model, tokenizer = AutoModelForSequenceClassification.from_pretrained(MODEL_NAME), \
+                       AutoTokenizer.from_pretrained(MODEL_NAME)
+
+    model = model.to(device)
+
+    ### BASELINE ###
+
+    # Batched utterance score processing
+    # get scores/rewards for each utterance in the train set
+    # utterance consists of both a user's prompt and the model's response
+    utt_train_rewards = {}
+    for country, keyed_per_turn in utt_train.items():
+        print(f"Processing {country} - train")
+        for raw_utts in tqdm(keyed_per_turn.values()):
+            for i in range(0, len(raw_utts), BATCH_SIZE):
+                utt_batch = raw_utts[i:min(len(raw_utts), i + BATCH_SIZE)]
+
+                # questions are prompts from the users
+                q_s = []
+                # answers are how the model responds
+                a_s = []
+
+                # create list of all user prompts
+                # create list of all model responses
+                for utt in utt_batch:
+                    q = utt["user_prompt"]
+                    a = utt["model_response"]
+                    q_s.append(q)
+                    a_s.append(a)
+
+                # tokenize the inputs
+                inputs = tokenizer(q_s, a_s, padding=True, return_tensors='pt').to(device)
+                # given question, answer
+                # get a score for how good the answer is
+                # save the score for the given utterance
+                scores = model(**inputs).logits.cpu().detach()
+                for j in range(len(utt_batch)):
+                    utt_id = utt_batch[j]["utterance_id"]
+                    score = scores[j]
+                    utt_train_rewards[utt_id] = score
+
+    # get scores/rewards for each utterance in the test set
+    # this scoring loop could probably be decomposed with the one starting on line 161
+
+    utt_test_rewards = {}
+    for country, keyed_per_turn in utt_test.items():
+        print(f"Processing {country} - test")
+        for raw_utts in tqdm(keyed_per_turn.values()):
+            for i in range(0, len(raw_utts), BATCH_SIZE):
+                utt_batch = raw_utts[i:min(len(raw_utts), i + BATCH_SIZE)]
+                q_s = []
+                a_s = []
+                for utt in utt_batch:
+                    q = utt["user_prompt"]
+                    a = utt["model_response"]
+                    q_s.append(q)
+                    a_s.append(a)
+                inputs = tokenizer(q_s, a_s, padding=True, return_tensors='pt').to(device)
+
+                scores = model(**inputs).logits.cpu().detach()
+                for j in range(len(utt_batch)):
+                    utt = utt_batch[j]
+                    utt_id = utt["utterance_id"]
+                    score = scores[j]
+                    utt_test_rewards[utt_id] = score
+
+
+    utt_pref_train = collections.defaultdict(list)
+    utt_pref_test = collections.defaultdict(list)
+
+    utt_pref_edited_train = collections.defaultdict(list)
+    utt_pref_edited_test = collections.defaultdict(list)
+
+    print("Processing edits")
+    score_same_cnt = 0
+
+    # utt_train is keyed per user and turn
+    # {country: {key: [utterances]}}
+    # key = f"{user_id}_{turn}"
+    # turn is not "turn" in the dataset, but rather interaction_id
+    # see line 117: turn = utt["interaction_id"]
+    # what is interaction_id?
+
+    def score_thres(bigger, smaller, country):
+        return (torch.exp(bigger) / (torch.exp(bigger) + torch.exp(smaller))) < (REWARD_THRES if country not in REWARD_THRES_OVERRIDE else REWARD_THRES_OVERRIDE[country])
+
+    # utt_pref_train = {country: [(pref_utt_id, dispref_utt_id)...]}
+    for country, keyed_per_turn in utt_train.items():
+
+        # prefs is a reference to utt_pref_train
+        # when we modify prefs in the loop below, this also populates utt_pref_train
+        prefs = utt_pref_train[country]
+        edited_prefs = utt_pref_edited_train[country]
+        scores = utt_score_train[country]
+        # Ignoring key which is user / turn info.
+        for raw_utts in keyed_per_turn.values():
+            for i, utt1 in enumerate(raw_utts):
+                utt1r = utt_train_rewards[utt1["utterance_id"]]
+                utt1s = utt1["score"]
+                scores.append([utt1, utt1s])
+                for j in range(i + 1, len(raw_utts)):
+                    utt2 = raw_utts[j]
+                    utt2r = utt_train_rewards[utt2["utterance_id"]]
+                    utt2s = utt2["score"]
+                    if utt1s > utt2s:
+                        prefs.append(format_text(utt1, utt2))
+                        if score_thres(utt1r, utt2r, country):
+                            edited_prefs.append(format_text(utt1, utt2))
+                    elif utt2s > utt1s:
+                        prefs.append(format_text(utt2, utt1))
+                        if score_thres(utt2r, utt1r, country):
+                            edited_prefs.append(format_text(utt2, utt1))
+                    else:
+                        score_same_cnt += 1
+
+    print("Evaluating test results")
+    for country, keyed_per_turn in utt_test.items():
+        prefs = utt_pref_test[country]
+        edited_prefs = utt_pref_edited_test[country]
+        scores = utt_score_test[country]
+        # Ignoring key which is user / turn info.
+
+        # for each pair of utterances
+        for raw_utts in keyed_per_turn.values():
+            for i, utt1 in enumerate(raw_utts):
+                # utt1s stands for utt1_score
+                utt1r = utt_test_rewards[utt1["utterance_id"]]
+                utt1s = utt1["score"]
+
+                # scores = [[utt, utt_score], ...]
+                scores.append([utt1, utt1s])
+
+                for j in range(i + 1, len(raw_utts)):
+                    utt2 = raw_utts[j]
+                    utt2r = utt_test_rewards[utt2["utterance_id"]]
+                    utt2s = utt2["score"]
+
+                    # prefs stores all pairs of utterances
+                    # with the higher scored one at idx 0
+                    if utt1s > utt2s:
+                        prefs.append(format_text(utt1, utt2))
+                        if utt1r < utt2r:
+                            edited_prefs.append(format_text(utt1, utt2))
+                    elif utt2s > utt1s:
+                        prefs.append(format_text(utt2, utt1))
+                        if utt2r < utt1r:
+                            edited_prefs.append(format_text(utt2, utt1))
+                    else:
+                        # if scored the same, count it, but leave out of prefs
+                        # the count of how many have been "removed"
+                        score_same_cnt += 1
+
+    print("**** PAIRS ****")
+    print("**** TRAIN ****")
+    print({count: len(pairs) for count, pairs in utt_pref_train.items()})
+    print("**** TEST ****")
+    print({count: len(pairs) for count, pairs in utt_pref_test.items()})
+    print("**** EDITED TRAIN ****")
+    print({count: len(pairs) for count, pairs in utt_pref_edited_train.items()})
+    print("**** EDITED TEST ****")
+    print({count: len(pairs) for count, pairs in utt_pref_edited_test.items()})
+    print(f"Scored same : {score_same_cnt}")
+
+    # Compute edited pairs ratio (edited / total) for train and test
+    edited_ratios = {"train": {}, "test": {}}
+    total_train_pairs = 0
+    total_train_edited = 0
+    total_test_pairs = 0
+    total_test_edited = 0
+    for count in countries:
+        train_total = len(utt_pref_train[count])
+        train_edited = len(utt_pref_edited_train[count])
+        test_total = len(utt_pref_test[count])
+        test_edited = len(utt_pref_edited_test[count])
+        edited_ratios["train"][count] = train_edited / train_total if train_total > 0 else 0.0
+        edited_ratios["test"][count] = test_edited / test_total if test_total > 0 else 0.0
+        total_train_pairs += train_total
+        total_train_edited += train_edited
+        total_test_pairs += test_total
+        total_test_edited += test_edited
+    # Compute average ratios based on total counts
+    edited_ratios["train"]["avg"] = total_train_edited / total_train_pairs if total_train_pairs > 0 else 0.0
+    edited_ratios["test"]["avg"] = total_test_edited / total_test_pairs if total_test_pairs > 0 else 0.0
+
+    print("**** EDITED PAIRS RATIO ****")
+    print("**** TRAIN ****")
+    for count in countries:
+        print(f"{count}: {edited_ratios['train'][count]:.4f}")
+    print(f"Average: {edited_ratios['train']['avg']:.4f}")
+    print("**** TEST ****")
+    for count in countries:
+        print(f"{count}: {edited_ratios['test'][count]:.4f}")
+    print(f"Average: {edited_ratios['test']['avg']:.4f}")
+
+    print("**** Starting reward model Evals ****")
+    print("**** Percentage of pairs correctly evaluated ****")
+    avg_perf = 0.
+    for count, pairs in utt_pref_test.items():
+        correct = 0
+        wrong = 0
+        for pair in pairs:
+            chosen_score = utt_test_rewards[pair["chosen_id"]]
+            rejected_score = utt_test_rewards[pair["rejected_id"]]
+
+            # model (deberta) gave higher score to preferred response
+            if chosen_score > rejected_score:
+                correct += 1
+            # model (deberta) gave higher score to rejected response
+            else:
+                wrong += 1
+        perf = correct * 1.0 / (correct + wrong) * 100
+        print(f"{count}: {perf:.2f}%")
+        full_perfs[count]["start"] = perf
+        avg_perf += perf
+    avg_perf /= len(countries)
+    full_perfs["avg"]["start"] = avg_perf
+    print(f"Starting Performance: {avg_perf:.2f}%")
+
+    avg_perf = 0.
+    for count, pairs in utt_pref_edited_test.items():
+        correct = 0
+        wrong = 0
+        for pair in pairs:
+            chosen_score = utt_test_rewards[pair["chosen_id"]]
+            rejected_score = utt_test_rewards[pair["rejected_id"]]
+
+            # model (deberta) gave higher score to preferred response
+            if chosen_score > rejected_score:
+                correct += 1
+            # model (deberta) gave higher score to rejected response
+            else:
+                wrong += 1
+        perf = correct * 1.0 / (correct + wrong) * 100
+        print(f"{count}: {perf:.2f}%")
+        partial_perfs[count]["start"] = perf
+        avg_perf += perf
+    avg_perf /= len(countries)
+    partial_perfs["avg"]["start"] = avg_perf
+    print(f"Starting Partial Performance: {avg_perf:.2f}%")
+
+    print("**** Training Country-Specific Models ****")
+
+    ### TRAIN THE REWARD MODEL (DEBERTA) WITH EDITED PREFS ###
+    # this training loop can probably be decomposed with the one starting on line 328
+    print("**** Ablation reward model training ****")
+
+    avg_perf = 0.
+    avg_partial_perf = 0.
+    test_baseline_rewards = collections.defaultdict(dict)
+    # Train reward model
+    for count in countries:
+        model = AutoModelForSequenceClassification.from_pretrained(MODEL_NAME)
+        tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+
+        def format_func_train(examples):
+            kwargs = {"padding": "max_length", "truncation": True, "max_length": 512, "return_tensors": "pt"}
+            chosen_chat = examples["chosen"][0]["content"] + " [SEP] " + examples["chosen"][1]["content"]
+            rejected_chat = examples["rejected"][0]["content"] + " [SEP] " + examples["rejected"][1]["content"]
+            chosen_tokens = tokenizer.encode_plus(chosen_chat, **kwargs)
+            rejected_tokens = tokenizer.encode_plus(rejected_chat, **kwargs)
+            ref_rewards_chosen = utt_train_rewards[examples["chosen_id"]]
+            ref_rewards_rejected = utt_train_rewards[examples["rejected_id"]]
+            return {
+                "input_ids_chosen": chosen_tokens["input_ids"][0],
+                "attention_mask_chosen": chosen_tokens["attention_mask"][0],
+                "input_ids_rejected": rejected_tokens["input_ids"][0],
+                "attention_mask_rejected": rejected_tokens["attention_mask"][0],
+                "ref_rewards_chosen": ref_rewards_chosen,
+                "ref_rewards_rejected": ref_rewards_rejected,
+            }
+
+        def format_func_test(examples):
+            kwargs = {"padding": "max_length", "truncation": True, "max_length": 512, "return_tensors": "pt"}
+            chosen_chat = examples["chosen"][0]["content"] + " [SEP] " + examples["chosen"][1]["content"]
+            rejected_chat = examples["rejected"][0]["content"] + " [SEP] " + examples["rejected"][1]["content"]
+            chosen_tokens = tokenizer.encode_plus(chosen_chat, **kwargs)
+            rejected_tokens = tokenizer.encode_plus(rejected_chat, **kwargs)
+            ref_rewards_chosen = utt_test_rewards[examples["chosen_id"]]
+            ref_rewards_rejected = utt_test_rewards[examples["rejected_id"]]
+            return {
+                "input_ids_chosen": chosen_tokens["input_ids"][0],
+                "attention_mask_chosen": chosen_tokens["attention_mask"][0],
+                "input_ids_rejected": rejected_tokens["input_ids"][0],
+                "attention_mask_rejected": rejected_tokens["attention_mask"][0],
+                "ref_rewards_chosen": ref_rewards_chosen,
+                "ref_rewards_rejected": ref_rewards_rejected,
+            }
+
+        train_reward_dataset = Dataset.from_list(utt_pref_edited_train[count]).map(format_func_train)
+        train_reward_dataset = train_reward_dataset.shuffle(seed=RANDOM_DATA_SHUFFLE_SEED)
+        test_reward_dataset = Dataset.from_list(utt_pref_test[count]).map(format_func_test)
+        test_reward_dataset = test_reward_dataset.shuffle(seed=RANDOM_DATA_SHUFFLE_SEED)
+
+        training_args = RewardConfig(
+            output_dir=f"{out_path}/trunc/{count}/ours_{EDITED_EPOCHS}epoch/",
+            per_device_train_batch_size=BATCH_SIZE,
+            num_train_epochs=EDITED_EPOCHS,
+            max_steps=MAX_STEPS[count],
+            learning_rate=EDITED_LR,
+            logging_steps=25,
+            eval_strategy="steps",
+            eval_steps=REPORT,
+            save_strategy="no",
+            max_length=2048,
+            report_to="none",
+            remove_unused_columns=False
+        )
+        data_collator = OurRewardDataCollatorWithPadding(
+            tokenizer=tokenizer, max_length=training_args.max_length
+        )
+        trainer = OurRewardTrainer(
+            model=model,
+            args=training_args,
+            beta=WEIGHTING_BETA,
+            data_collator=data_collator,
+            train_dataset=train_reward_dataset,
+            eval_dataset=test_reward_dataset
+        )
+
+        trainer.train()
+
+        trainer.save_model(training_args.output_dir)
+
+        rewards = test_baseline_rewards[count]
+        print("Scoring the test set!")
+        keyed_per_turn = utt_test[count]
+        for raw_utts in tqdm(keyed_per_turn.values()):
+            for i in range(0, len(raw_utts), BATCH_SIZE):
+                utt_batch = raw_utts[i:min(len(raw_utts), i + BATCH_SIZE)]
+                q_s = []
+                a_s = []
+                for utt in utt_batch:
+                    q = utt["user_prompt"]
+                    a = utt["model_response"]
+                    q_s.append(q)
+                    a_s.append(a)
+                inputs = tokenizer(q_s, a_s, padding=True, return_tensors='pt').to(device)
+                scores = model(**inputs).logits.cpu().detach()
+                for j in range(len(utt_batch)):
+                    utt_id = utt_batch[j]["utterance_id"]
+                    score = scores[j]
+                    rewards[utt_id] = score
+
+        print("Evaluating the test set!")
+
+        # this evaluation loop can probably be decomposed with the one starting on line 300
+        # currently train on the edited set, test on the full set
+        pairs = utt_pref_test[count]
+
+        correct = 0
+        wrong = 0
+        for pair in pairs:
+            chosen_score = rewards[pair["chosen_id"]]
+            rejected_score = rewards[pair["rejected_id"]]
+            if chosen_score > rejected_score:
+                correct += 1
+            else:
+                wrong += 1
+        perf = correct * 1.0 / (correct + wrong) * 100
+        print(f"{count}: {perf:.2f}%")
+        full_perfs[count]["ours_partial"] = perf
+        avg_perf += perf
+
+        pairs = utt_pref_edited_test[count]
+
+        correct = 0
+        wrong = 0
+        for pair in pairs:
+            chosen_score = rewards[pair["chosen_id"]]
+            rejected_score = rewards[pair["rejected_id"]]
+
+            # model (deberta) gave higher score to preferred response
+            if chosen_score > rejected_score:
+                correct += 1
+            # model (deberta) gave higher score to rejected response
+            else:
+                wrong += 1
+        perf = correct * 1.0 / (correct + wrong) * 100
+        print(f"{count}: {perf:.2f}%")
+        partial_perfs[count]["ours_partial"] = perf
+        avg_partial_perf += perf
+
+    print("**** Ablation reward model Evals ****")
+    print("**** Percentage of pairs correctly evaluated ****")
+    avg_perf /= len(countries)
+    full_perfs["avg"]["ours_partial"] = avg_perf
+    print(f"Ablation Model Performance: {avg_perf:.2f}%")
+
+    avg_partial_perf /= len(countries)
+    partial_perfs["avg"]["ours_partial"] = avg_partial_perf
+    print(f"Ablation Model Partial Performance: {avg_partial_perf:.2f}%")
+
+    import json
+    path = f"{out_path}/experiment/seed/{RANDOM_MODEL_SEED}/"
+    if not os.path.exists(path):
+        os.makedirs(path)
+
+    print("************* OVERALL PERFORMANCE *****************")
+    print(json.dumps(full_perfs, indent=4))
+    with open(f"{out_path}/experiment/seed/{RANDOM_MODEL_SEED}/full_report.json", 'w') as f:
+        json.dump(full_perfs, f, ensure_ascii=False, indent=4)
+
+    print("************* OVERALL Partial PERFORMANCE *****************")
+    print(json.dumps(partial_perfs, indent=4))
+    with open(f"{out_path}/experiment/seed/{RANDOM_MODEL_SEED}/partial_report.json", 'w') as f:
+        json.dump(partial_perfs, f, ensure_ascii=False, indent=4)
+
+    print("************* EDITED PAIRS RATIO *****************")
+    print(json.dumps(edited_ratios, indent=4))
+    with open(f"{out_path}/experiment/seed/{RANDOM_MODEL_SEED}/edited_ratios.json", 'w') as f:
+        json.dump(edited_ratios, f, ensure_ascii=False, indent=4)
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("out_path")
+args = parser.parse_args()
+
+os.environ["TOKENIZERS_PARALLELISM"] = "true"
+# Press the green button in the gutter to run the script.
+if __name__ == '__main__':
+    main(args.out_path)
+
+# See PyCharm help at https://www.jetbrains.com/help/pycharm/

requirements-pw-lock.txt

@@ -0,0 +1,70 @@
+# Exact observed PW environment: Linux x86_64, Python 3.11.16, CUDA 12.1 wheels.
+Jinja2==3.1.6
+MarkupSafe==3.0.4
+PyYAML==6.0.3
+Pygments==2.21.0
+accelerate==1.0.1
+aiohappyeyeballs==2.7.1
+aiohttp==3.14.4
+aiosignal==1.4.0
+attrs==26.1.0
+certifi==2026.7.22
+charset-normalizer==3.5.2
+datasets==3.0.1
+dill==0.3.8
+docstring_parser==0.18.0
+filelock==4.0.12
+frozenlist==1.8.0
+fsspec==2024.6.1
+huggingface-hub==0.25.2
+idna==3.20
+markdown-it-py==4.2.0
+mdurl==0.1.2
+mpmath==1.3.0
+multidict==6.9.1
+multiprocess==0.70.16
+networkx==3.6.1
+numpy==1.24.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.9.86
+nvidia-nvtx-cu12==12.1.105
+packaging==26.3
+pandas==2.0.3
+peft==0.13.2
+propcache==0.5.4
+protobuf==3.20.3
+psutil==7.2.2
+pyarrow==17.0.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+pytz==2026.5
+regex==2026.9.29
+requests==2.34.2
+rich==13.9.2
+safetensors==0.4.5
+sentencepiece==0.2.0
+shtab==1.12.1
+six==1.17.0
+sympy==1.14.0
+tokenizers==0.20.1
+torch==2.4.1
+tqdm==4.66.5
+transformers==4.45.2
+triton==3.0.0
+trl==0.11.4
+typeguard==4.6.0
+typing_extensions==4.16.0
+tyro==0.8.13
+tzdata==2026.5
+urllib3==2.8.0
+xxhash==4.0.1
+yarl==1.25.1

special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
+size 2464616

tokenizer_config.json

@@ -0,0 +1,62 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128000": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "[CLS]",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "max_length": 512,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "split_by_punct": false,
+  "stride": 0,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "[UNK]",
+  "vocab_type": "spm"
+}

validation.json

@@ -0,0 +1,17 @@
+{
+  "checkpoint": "scpo-deberta-v3-base-canada",
+  "device": "cpu",
+  "scores": [
+    [
+      -2.664663314819336
+    ],
+    [
+      0.41405123472213745
+    ]
+  ],
+  "deterministic": true,
+  "sample_pair_vs_concat_ids_equal": [
+    true,
+    true
+  ]
+}