npedrazzini commited on
Commit
2053c47
·
verified ·
1 Parent(s): ebd9c90

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +84 -6
README.md CHANGED
@@ -58,13 +58,42 @@ QMUL Research-IT. http://doi.org/10.5281/zenodo.438045
58
  ## Usage
59
 
60
  ```python
61
- from continuous_time_embedding import load_continuous_time_model
62
  import torch.nn.functional as F
63
  import math
 
 
 
 
64
 
65
- tokenizer, model = load_time_aware_model("npedrazzini/NewsBERT_1800-1920-Temporal")
 
 
 
 
 
 
66
  device = next(model.parameters()).device
67
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
68
  def get_target_word_probability(sentence, target_word, year):
69
  target_ids = tokenizer.encode(target_word, add_special_tokens=False)
70
  mask_str = " ".join([tokenizer.mask_token] * len(target_ids))
@@ -85,13 +114,62 @@ def get_target_word_probability(sentence, target_word, year):
85
  log_probs = F.log_softmax(out.logits[0, pos].float(), dim=-1)
86
  logprob_sum += log_probs[tgt_id].item()
87
 
88
- return math.exp(logprob_sum) if len(target_ids) == 1 else None, logprob_sum
 
 
 
 
 
 
 
 
 
89
 
90
- # Example: how does P("train") in this sentence change across years?
 
 
 
91
  sentence = "The train arrived at the station on time"
92
- for year in [1800, 1830, 1860, 1890, 1920]:
93
  prob, logprob_sum = get_target_word_probability(sentence, "train", year)
94
- print(f"year={year}: P(train)={prob:.4f}" if prob else f"year={year}: logprob_sum={logprob_sum:.4f}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
95
  ```
96
 
97
  ## Known limitations
 
58
  ## Usage
59
 
60
  ```python
61
+ import torch
62
  import torch.nn.functional as F
63
  import math
64
+ from huggingface_hub import hf_hub_download
65
+ import shutil, os
66
+
67
+ REPO_ID = "npedrazzini/NewsBERT_1800-1920-Temporal"
68
 
69
+ # Download the companion module from the model repo and make it importable
70
+ py_path = hf_hub_download(repo_id=REPO_ID, filename="continuous_time_embedding.py")
71
+ shutil.copy(py_path, os.path.basename(py_path))
72
+
73
+ from continuous_time_embedding import load_continuous_time_model
74
+
75
+ tokenizer, model = load_continuous_time_model(REPO_ID)
76
  device = next(model.parameters()).device
77
 
78
+
79
+ def predict_masked(sentence, year, top_k=5):
80
+ enc = tokenizer(sentence, return_tensors="pt").to(device)
81
+ mask_pos = (enc["input_ids"][0] == tokenizer.mask_token_id).nonzero(as_tuple=True)[0]
82
+ if len(mask_pos) == 0:
83
+ raise ValueError(f"No [MASK] found in: {sentence!r}")
84
+ mask_pos = mask_pos[0].item()
85
+
86
+ embeddings_module = model.get_input_embeddings_module()
87
+ tok_embeds = embeddings_module(enc["input_ids"])
88
+ time_vec = model.time_embed(torch.tensor([float(year)]).to(device)).unsqueeze(1)
89
+ tok_embeds = model.post_inject_norm(tok_embeds + time_vec)
90
+
91
+ out = model.model(inputs_embeds=tok_embeds, attention_mask=enc["attention_mask"])
92
+ probs = F.softmax(out.logits[0, mask_pos].float(), dim=-1)
93
+ top = torch.topk(probs, top_k)
94
+ return [(tokenizer.convert_ids_to_tokens([tid.item()])[0], p.item()) for tid, p in zip(top.indices, top.values)]
95
+
96
+
97
  def get_target_word_probability(sentence, target_word, year):
98
  target_ids = tokenizer.encode(target_word, add_special_tokens=False)
99
  mask_str = " ".join([tokenizer.mask_token] * len(target_ids))
 
114
  log_probs = F.log_softmax(out.logits[0, pos].float(), dim=-1)
115
  logprob_sum += log_probs[tgt_id].item()
116
 
117
+ prob = math.exp(logprob_sum) if len(target_ids) == 1 else None
118
+ return prob, logprob_sum
119
+
120
+
121
+ # Example 1: fill-mask, top predictions at different years
122
+ sentence = "The [MASK] arrived on time."
123
+ for year in [1800, 1810, 1820, 1830, 1840, 1850, 1860, 1870, 1880, 1890, 1900, 1910, 1920]:
124
+ preds = predict_masked(sentence, year, top_k=5)
125
+ pred_str = ", ".join(f"{tok}({p:.3f})" for tok, p in preds)
126
+ print(f"year={year}: {pred_str}")
127
 
128
+
129
+ print()
130
+
131
+ # Example 2: probability of a specific word, across years
132
  sentence = "The train arrived at the station on time"
133
+ for year in [1800, 1810, 1820, 1830, 1840, 1850, 1860, 1870, 1880, 1890, 1900, 1910, 1920]:
134
  prob, logprob_sum = get_target_word_probability(sentence, "train", year)
135
+ if prob is not None:
136
+ print(f"year={year}: P(train)={prob:.10f} (logprob_sum={logprob_sum:.4f})")
137
+ else:
138
+ print(f"year={year}: logprob_sum={logprob_sum:.4f} (multi-subtoken word, no single probability)")
139
+ ```
140
+
141
+
142
+ Example outputs:
143
+ ```
144
+ year=1800: express(0.048), steamer(0.043), mail(0.031), train(0.018), news(0.014)
145
+ year=1810: express(0.064), steamer(0.044), mail(0.035), train(0.026), news(0.015)
146
+ year=1820: express(0.068), steamer(0.044), mail(0.035), train(0.032), news(0.017)
147
+ year=1830: express(0.078), steamer(0.043), mail(0.038), train(0.033), news(0.019)
148
+ year=1840: express(0.087), steamer(0.040), train(0.039), mail(0.038), news(0.023)
149
+ year=1850: express(0.081), train(0.042), steamer(0.039), mail(0.038), news(0.024)
150
+ year=1860: express(0.079), train(0.042), mail(0.038), steamer(0.035), news(0.029)
151
+ year=1870: express(0.080), train(0.050), mail(0.037), news(0.034), telegram(0.032)
152
+ year=1880: express(0.069), train(0.051), news(0.035), mail(0.035), telegram(0.032)
153
+ year=1890: express(0.063), train(0.052), news(0.039), telegram(0.034), mail(0.034)
154
+ year=1900: express(0.059), train(0.048), news(0.046), telegram(0.038), mail(0.033)
155
+ year=1910: express(0.054), news(0.052), train(0.047), telegram(0.039), mail(0.031)
156
+ year=1920: news(0.057), express(0.049), train(0.044), telegram(0.040), mail(0.030)
157
+ ```
158
+
159
+ ```
160
+ year=1800: P(train)=0.2596983756 (logprob_sum=-1.3482)
161
+ year=1810: P(train)=0.3239273912 (logprob_sum=-1.1272)
162
+ year=1820: P(train)=0.3619919056 (logprob_sum=-1.0161)
163
+ year=1830: P(train)=0.3642843769 (logprob_sum=-1.0098)
164
+ year=1840: P(train)=0.3824294963 (logprob_sum=-0.9612)
165
+ year=1850: P(train)=0.3897419142 (logprob_sum=-0.9423)
166
+ year=1860: P(train)=0.3831270530 (logprob_sum=-0.9594)
167
+ year=1870: P(train)=0.3918517235 (logprob_sum=-0.9369)
168
+ year=1880: P(train)=0.3810631289 (logprob_sum=-0.9648)
169
+ year=1890: P(train)=0.3701365410 (logprob_sum=-0.9939)
170
+ year=1900: P(train)=0.3599881482 (logprob_sum=-1.0217)
171
+ year=1910: P(train)=0.3460587446 (logprob_sum=-1.0611)
172
+ year=1920: P(train)=0.3430851355 (logprob_sum=-1.0698)
173
  ```
174
 
175
  ## Known limitations