Fill-Mask
Transformers
Safetensors
bert
masked-language-modeling
historical-nlp
temporal-language-model
Instructions to use TextMachineProject/NewsBERT_1800-1920-Temporal with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use TextMachineProject/NewsBERT_1800-1920-Temporal with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("fill-mask", model="TextMachineProject/NewsBERT_1800-1920-Temporal")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForMaskedLM tokenizer = AutoTokenizer.from_pretrained("TextMachineProject/NewsBERT_1800-1920-Temporal") model = AutoModelForMaskedLM.from_pretrained("TextMachineProject/NewsBERT_1800-1920-Temporal", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Update README.md
Browse files
README.md
CHANGED
|
@@ -55,8 +55,9 @@ QMUL Research-IT. http://doi.org/10.5281/zenodo.438045
|
|
| 55 |
- `torch` 2.11.0+cu126
|
| 56 |
- `transformers` 5.12.1
|
| 57 |
|
| 58 |
-
##
|
| 59 |
|
|
|
|
| 60 |
```python
|
| 61 |
import torch
|
| 62 |
import torch.nn.functional as F
|
|
@@ -74,8 +75,12 @@ from continuous_time_embedding import load_continuous_time_model
|
|
| 74 |
|
| 75 |
tokenizer, model = load_continuous_time_model(REPO_ID)
|
| 76 |
device = next(model.parameters()).device
|
|
|
|
|
|
|
|
|
|
| 77 |
|
| 78 |
|
|
|
|
| 79 |
def predict_masked(sentence, year, top_k=5):
|
| 80 |
enc = tokenizer(sentence, return_tensors="pt").to(device)
|
| 81 |
mask_pos = (enc["input_ids"][0] == tokenizer.mask_token_id).nonzero(as_tuple=True)[0]
|
|
@@ -117,8 +122,7 @@ def get_target_word_probability(sentence, target_word, year):
|
|
| 117 |
prob = math.exp(logprob_sum) if len(target_ids) == 1 else None
|
| 118 |
return prob, logprob_sum
|
| 119 |
|
| 120 |
-
|
| 121 |
-
# Example 1: fill-mask, top predictions at different years
|
| 122 |
sentence = "The [MASK] arrived on time."
|
| 123 |
for year in [1800, 1810, 1820, 1830, 1840, 1850, 1860, 1870, 1880, 1890, 1900, 1910, 1920]:
|
| 124 |
preds = predict_masked(sentence, year, top_k=5)
|
|
@@ -126,8 +130,6 @@ for year in [1800, 1810, 1820, 1830, 1840, 1850, 1860, 1870, 1880, 1890, 1900, 1
|
|
| 126 |
print(f"year={year}: {pred_str}")
|
| 127 |
|
| 128 |
|
| 129 |
-
print()
|
| 130 |
-
|
| 131 |
# Example 2: probability of a specific word, across years
|
| 132 |
sentence = "The train arrived at the station on time"
|
| 133 |
for year in [1800, 1810, 1820, 1830, 1840, 1850, 1860, 1870, 1880, 1890, 1900, 1910, 1920]:
|
|
@@ -172,6 +174,56 @@ year=1910: P(train)=0.3460587446 (logprob_sum=-1.0611)
|
|
| 172 |
year=1920: P(train)=0.3430851355 (logprob_sum=-1.0698)
|
| 173 |
```
|
| 174 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 175 |
## Known limitations
|
| 176 |
|
| 177 |
- Trained for 1 epoch only. Treat results as preliminary.
|
|
|
|
| 55 |
- `torch` 2.11.0+cu126
|
| 56 |
- `transformers` 5.12.1
|
| 57 |
|
| 58 |
+
## Example usage
|
| 59 |
|
| 60 |
+
### Load the model and the companion module
|
| 61 |
```python
|
| 62 |
import torch
|
| 63 |
import torch.nn.functional as F
|
|
|
|
| 75 |
|
| 76 |
tokenizer, model = load_continuous_time_model(REPO_ID)
|
| 77 |
device = next(model.parameters()).device
|
| 78 |
+
```
|
| 79 |
+
|
| 80 |
+
### Check changes in top mask prediction or in token probability
|
| 81 |
|
| 82 |
|
| 83 |
+
```python
|
| 84 |
def predict_masked(sentence, year, top_k=5):
|
| 85 |
enc = tokenizer(sentence, return_tensors="pt").to(device)
|
| 86 |
mask_pos = (enc["input_ids"][0] == tokenizer.mask_token_id).nonzero(as_tuple=True)[0]
|
|
|
|
| 122 |
prob = math.exp(logprob_sum) if len(target_ids) == 1 else None
|
| 123 |
return prob, logprob_sum
|
| 124 |
|
| 125 |
+
### Example 1: fill-mask, top predictions at different years
|
|
|
|
| 126 |
sentence = "The [MASK] arrived on time."
|
| 127 |
for year in [1800, 1810, 1820, 1830, 1840, 1850, 1860, 1870, 1880, 1890, 1900, 1910, 1920]:
|
| 128 |
preds = predict_masked(sentence, year, top_k=5)
|
|
|
|
| 130 |
print(f"year={year}: {pred_str}")
|
| 131 |
|
| 132 |
|
|
|
|
|
|
|
| 133 |
# Example 2: probability of a specific word, across years
|
| 134 |
sentence = "The train arrived at the station on time"
|
| 135 |
for year in [1800, 1810, 1820, 1830, 1840, 1850, 1860, 1870, 1880, 1890, 1900, 1910, 1920]:
|
|
|
|
| 174 |
year=1920: P(train)=0.3430851355 (logprob_sum=-1.0698)
|
| 175 |
```
|
| 176 |
|
| 177 |
+
### Check against known historical events
|
| 178 |
+
|
| 179 |
+
Visualize changes in probability of particular tokens and compare against historical events
|
| 180 |
+
which should correspond to peak in probabilities (either one-off for a specific year or as the start of
|
| 181 |
+
a continuous rise in probability thereafter, e.g. due to increased discourse around it in the newspapers).
|
| 182 |
+
|
| 183 |
+
```python
|
| 184 |
+
event_tests = [
|
| 185 |
+
("The garrison at lucknow was relieved after a long siege.", "lucknow", 1857),
|
| 186 |
+
("President lincoln was assassinated.", "lincoln", 1865),
|
| 187 |
+
("After a long siege, the fortress of sebastopol finally fell.", "sebastopol", 1855),
|
| 188 |
+
("President mckinley was assassinated.", "mckinley", 1901),
|
| 189 |
+
("The queen has died.", "died", 1901)]
|
| 190 |
+
|
| 191 |
+
test_years = list(range(1800, 1921, 5))
|
| 192 |
+
fig, axes = plt.subplots(len(event_tests), 1, figsize=(11, 4.5 * len(EVENT_TESTS)))
|
| 193 |
+
|
| 194 |
+
for ax, (sentence, target_word, expected_year) in zip(axes, event_tests):
|
| 195 |
+
masked_sentence = sentence.replace(target_word, tokenizer.mask_token, 1)
|
| 196 |
+
n_subtokens = len(tokenizer.encode(target_word, add_special_tokens=False))
|
| 197 |
+
is_single_token = n_subtokens == 1
|
| 198 |
+
|
| 199 |
+
results = [get_target_word_probability(sentence, target_word, year) for year in test_years]
|
| 200 |
+
plot_vals = [prob if is_single_token else logprob_sum for prob, logprob_sum in results]
|
| 201 |
+
for year, (prob, logprob_sum) in zip(test_years, results):
|
| 202 |
+
print(f"{sentence} | year={year}: " + (f"P({target_word})={prob:.6f}" if is_single_token else f"logprob_sum={logprob_sum:.4f}"))
|
| 203 |
+
|
| 204 |
+
event_prob, event_logprob = get_target_word_probability(sentence, target_word, expected_year)
|
| 205 |
+
event_val = event_prob if is_single_token else event_logprob
|
| 206 |
+
|
| 207 |
+
ax.plot(test_years, plot_vals, marker="o")
|
| 208 |
+
ax.scatter([expected_year], [event_val], marker="*", s=120, color="red", zorder=5, label=f"at {expected_year}")
|
| 209 |
+
ax.axvline(expected_year, color="red", linestyle="--", alpha=0.4)
|
| 210 |
+
ax.set_title(f"{'P' if is_single_token else 'log P'}({target_word!r}) across years\n\"{sentence}\"")
|
| 211 |
+
ax.set_xlabel("queried year")
|
| 212 |
+
ax.set_ylabel("probability" if is_single_token else "log probability")
|
| 213 |
+
xticks = sorted(set(test_years) | {expected_year})
|
| 214 |
+
ax.set_xticks(xticks)
|
| 215 |
+
ax.set_xticklabels(xticks, rotation=45, ha="right")
|
| 216 |
+
ax.legend()
|
| 217 |
+
ax.grid(True, alpha=0.3)
|
| 218 |
+
|
| 219 |
+
plt.tight_layout()
|
| 220 |
+
plt.show()
|
| 221 |
+
```
|
| 222 |
+
|
| 223 |
+
Example output:
|
| 224 |
+
|
| 225 |
+

|
| 226 |
+
|
| 227 |
## Known limitations
|
| 228 |
|
| 229 |
- Trained for 1 epoch only. Treat results as preliminary.
|