npedrazzini commited on
Commit
481e612
·
verified ·
1 Parent(s): 984416b

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +57 -5
README.md CHANGED
@@ -55,8 +55,9 @@ QMUL Research-IT. http://doi.org/10.5281/zenodo.438045
55
  - `torch` 2.11.0+cu126
56
  - `transformers` 5.12.1
57
 
58
- ## Usage
59
 
 
60
  ```python
61
  import torch
62
  import torch.nn.functional as F
@@ -74,8 +75,12 @@ from continuous_time_embedding import load_continuous_time_model
74
 
75
  tokenizer, model = load_continuous_time_model(REPO_ID)
76
  device = next(model.parameters()).device
 
 
 
77
 
78
 
 
79
  def predict_masked(sentence, year, top_k=5):
80
  enc = tokenizer(sentence, return_tensors="pt").to(device)
81
  mask_pos = (enc["input_ids"][0] == tokenizer.mask_token_id).nonzero(as_tuple=True)[0]
@@ -117,8 +122,7 @@ def get_target_word_probability(sentence, target_word, year):
117
  prob = math.exp(logprob_sum) if len(target_ids) == 1 else None
118
  return prob, logprob_sum
119
 
120
-
121
- # Example 1: fill-mask, top predictions at different years
122
  sentence = "The [MASK] arrived on time."
123
  for year in [1800, 1810, 1820, 1830, 1840, 1850, 1860, 1870, 1880, 1890, 1900, 1910, 1920]:
124
  preds = predict_masked(sentence, year, top_k=5)
@@ -126,8 +130,6 @@ for year in [1800, 1810, 1820, 1830, 1840, 1850, 1860, 1870, 1880, 1890, 1900, 1
126
  print(f"year={year}: {pred_str}")
127
 
128
 
129
- print()
130
-
131
  # Example 2: probability of a specific word, across years
132
  sentence = "The train arrived at the station on time"
133
  for year in [1800, 1810, 1820, 1830, 1840, 1850, 1860, 1870, 1880, 1890, 1900, 1910, 1920]:
@@ -172,6 +174,56 @@ year=1910: P(train)=0.3460587446 (logprob_sum=-1.0611)
172
  year=1920: P(train)=0.3430851355 (logprob_sum=-1.0698)
173
  ```
174
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
175
  ## Known limitations
176
 
177
  - Trained for 1 epoch only. Treat results as preliminary.
 
55
  - `torch` 2.11.0+cu126
56
  - `transformers` 5.12.1
57
 
58
+ ## Example usage
59
 
60
+ ### Load the model and the companion module
61
  ```python
62
  import torch
63
  import torch.nn.functional as F
 
75
 
76
  tokenizer, model = load_continuous_time_model(REPO_ID)
77
  device = next(model.parameters()).device
78
+ ```
79
+
80
+ ### Check changes in top mask prediction or in token probability
81
 
82
 
83
+ ```python
84
  def predict_masked(sentence, year, top_k=5):
85
  enc = tokenizer(sentence, return_tensors="pt").to(device)
86
  mask_pos = (enc["input_ids"][0] == tokenizer.mask_token_id).nonzero(as_tuple=True)[0]
 
122
  prob = math.exp(logprob_sum) if len(target_ids) == 1 else None
123
  return prob, logprob_sum
124
 
125
+ ### Example 1: fill-mask, top predictions at different years
 
126
  sentence = "The [MASK] arrived on time."
127
  for year in [1800, 1810, 1820, 1830, 1840, 1850, 1860, 1870, 1880, 1890, 1900, 1910, 1920]:
128
  preds = predict_masked(sentence, year, top_k=5)
 
130
  print(f"year={year}: {pred_str}")
131
 
132
 
 
 
133
  # Example 2: probability of a specific word, across years
134
  sentence = "The train arrived at the station on time"
135
  for year in [1800, 1810, 1820, 1830, 1840, 1850, 1860, 1870, 1880, 1890, 1900, 1910, 1920]:
 
174
  year=1920: P(train)=0.3430851355 (logprob_sum=-1.0698)
175
  ```
176
 
177
+ ### Check against known historical events
178
+
179
+ Visualize changes in probability of particular tokens and compare against historical events
180
+ which should correspond to peak in probabilities (either one-off for a specific year or as the start of
181
+ a continuous rise in probability thereafter, e.g. due to increased discourse around it in the newspapers).
182
+
183
+ ```python
184
+ event_tests = [
185
+ ("The garrison at lucknow was relieved after a long siege.", "lucknow", 1857),
186
+ ("President lincoln was assassinated.", "lincoln", 1865),
187
+ ("After a long siege, the fortress of sebastopol finally fell.", "sebastopol", 1855),
188
+ ("President mckinley was assassinated.", "mckinley", 1901),
189
+ ("The queen has died.", "died", 1901)]
190
+
191
+ test_years = list(range(1800, 1921, 5))
192
+ fig, axes = plt.subplots(len(event_tests), 1, figsize=(11, 4.5 * len(EVENT_TESTS)))
193
+
194
+ for ax, (sentence, target_word, expected_year) in zip(axes, event_tests):
195
+ masked_sentence = sentence.replace(target_word, tokenizer.mask_token, 1)
196
+ n_subtokens = len(tokenizer.encode(target_word, add_special_tokens=False))
197
+ is_single_token = n_subtokens == 1
198
+
199
+ results = [get_target_word_probability(sentence, target_word, year) for year in test_years]
200
+ plot_vals = [prob if is_single_token else logprob_sum for prob, logprob_sum in results]
201
+ for year, (prob, logprob_sum) in zip(test_years, results):
202
+ print(f"{sentence} | year={year}: " + (f"P({target_word})={prob:.6f}" if is_single_token else f"logprob_sum={logprob_sum:.4f}"))
203
+
204
+ event_prob, event_logprob = get_target_word_probability(sentence, target_word, expected_year)
205
+ event_val = event_prob if is_single_token else event_logprob
206
+
207
+ ax.plot(test_years, plot_vals, marker="o")
208
+ ax.scatter([expected_year], [event_val], marker="*", s=120, color="red", zorder=5, label=f"at {expected_year}")
209
+ ax.axvline(expected_year, color="red", linestyle="--", alpha=0.4)
210
+ ax.set_title(f"{'P' if is_single_token else 'log P'}({target_word!r}) across years\n\"{sentence}\"")
211
+ ax.set_xlabel("queried year")
212
+ ax.set_ylabel("probability" if is_single_token else "log probability")
213
+ xticks = sorted(set(test_years) | {expected_year})
214
+ ax.set_xticks(xticks)
215
+ ax.set_xticklabels(xticks, rotation=45, ha="right")
216
+ ax.legend()
217
+ ax.grid(True, alpha=0.3)
218
+
219
+ plt.tight_layout()
220
+ plt.show()
221
+ ```
222
+
223
+ Example output:
224
+
225
+ ![Event spike test](event_spike_test.png)
226
+
227
  ## Known limitations
228
 
229
  - Trained for 1 epoch only. Treat results as preliminary.