Fill-Mask
Transformers
Safetensors
bert
masked-language-modeling
historical-nlp
temporal-language-model
Instructions to use TextMachineProject/NewsBERT_1800-1920-Temporal with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use TextMachineProject/NewsBERT_1800-1920-Temporal with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("fill-mask", model="TextMachineProject/NewsBERT_1800-1920-Temporal")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForMaskedLM tokenizer = AutoTokenizer.from_pretrained("TextMachineProject/NewsBERT_1800-1920-Temporal") model = AutoModelForMaskedLM.from_pretrained("TextMachineProject/NewsBERT_1800-1920-Temporal", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Update README.md
Browse files
README.md
CHANGED
|
@@ -58,13 +58,42 @@ QMUL Research-IT. http://doi.org/10.5281/zenodo.438045
|
|
| 58 |
## Usage
|
| 59 |
|
| 60 |
```python
|
| 61 |
-
|
| 62 |
import torch.nn.functional as F
|
| 63 |
import math
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
|
| 65 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
device = next(model.parameters()).device
|
| 67 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
def get_target_word_probability(sentence, target_word, year):
|
| 69 |
target_ids = tokenizer.encode(target_word, add_special_tokens=False)
|
| 70 |
mask_str = " ".join([tokenizer.mask_token] * len(target_ids))
|
|
@@ -85,13 +114,62 @@ def get_target_word_probability(sentence, target_word, year):
|
|
| 85 |
log_probs = F.log_softmax(out.logits[0, pos].float(), dim=-1)
|
| 86 |
logprob_sum += log_probs[tgt_id].item()
|
| 87 |
|
| 88 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
|
| 90 |
-
|
|
|
|
|
|
|
|
|
|
| 91 |
sentence = "The train arrived at the station on time"
|
| 92 |
-
for year in [1800, 1830, 1860, 1890, 1920]:
|
| 93 |
prob, logprob_sum = get_target_word_probability(sentence, "train", year)
|
| 94 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
```
|
| 96 |
|
| 97 |
## Known limitations
|
|
|
|
| 58 |
## Usage
|
| 59 |
|
| 60 |
```python
|
| 61 |
+
import torch
|
| 62 |
import torch.nn.functional as F
|
| 63 |
import math
|
| 64 |
+
from huggingface_hub import hf_hub_download
|
| 65 |
+
import shutil, os
|
| 66 |
+
|
| 67 |
+
REPO_ID = "npedrazzini/NewsBERT_1800-1920-Temporal"
|
| 68 |
|
| 69 |
+
# Download the companion module from the model repo and make it importable
|
| 70 |
+
py_path = hf_hub_download(repo_id=REPO_ID, filename="continuous_time_embedding.py")
|
| 71 |
+
shutil.copy(py_path, os.path.basename(py_path))
|
| 72 |
+
|
| 73 |
+
from continuous_time_embedding import load_continuous_time_model
|
| 74 |
+
|
| 75 |
+
tokenizer, model = load_continuous_time_model(REPO_ID)
|
| 76 |
device = next(model.parameters()).device
|
| 77 |
|
| 78 |
+
|
| 79 |
+
def predict_masked(sentence, year, top_k=5):
|
| 80 |
+
enc = tokenizer(sentence, return_tensors="pt").to(device)
|
| 81 |
+
mask_pos = (enc["input_ids"][0] == tokenizer.mask_token_id).nonzero(as_tuple=True)[0]
|
| 82 |
+
if len(mask_pos) == 0:
|
| 83 |
+
raise ValueError(f"No [MASK] found in: {sentence!r}")
|
| 84 |
+
mask_pos = mask_pos[0].item()
|
| 85 |
+
|
| 86 |
+
embeddings_module = model.get_input_embeddings_module()
|
| 87 |
+
tok_embeds = embeddings_module(enc["input_ids"])
|
| 88 |
+
time_vec = model.time_embed(torch.tensor([float(year)]).to(device)).unsqueeze(1)
|
| 89 |
+
tok_embeds = model.post_inject_norm(tok_embeds + time_vec)
|
| 90 |
+
|
| 91 |
+
out = model.model(inputs_embeds=tok_embeds, attention_mask=enc["attention_mask"])
|
| 92 |
+
probs = F.softmax(out.logits[0, mask_pos].float(), dim=-1)
|
| 93 |
+
top = torch.topk(probs, top_k)
|
| 94 |
+
return [(tokenizer.convert_ids_to_tokens([tid.item()])[0], p.item()) for tid, p in zip(top.indices, top.values)]
|
| 95 |
+
|
| 96 |
+
|
| 97 |
def get_target_word_probability(sentence, target_word, year):
|
| 98 |
target_ids = tokenizer.encode(target_word, add_special_tokens=False)
|
| 99 |
mask_str = " ".join([tokenizer.mask_token] * len(target_ids))
|
|
|
|
| 114 |
log_probs = F.log_softmax(out.logits[0, pos].float(), dim=-1)
|
| 115 |
logprob_sum += log_probs[tgt_id].item()
|
| 116 |
|
| 117 |
+
prob = math.exp(logprob_sum) if len(target_ids) == 1 else None
|
| 118 |
+
return prob, logprob_sum
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
# Example 1: fill-mask, top predictions at different years
|
| 122 |
+
sentence = "The [MASK] arrived on time."
|
| 123 |
+
for year in [1800, 1810, 1820, 1830, 1840, 1850, 1860, 1870, 1880, 1890, 1900, 1910, 1920]:
|
| 124 |
+
preds = predict_masked(sentence, year, top_k=5)
|
| 125 |
+
pred_str = ", ".join(f"{tok}({p:.3f})" for tok, p in preds)
|
| 126 |
+
print(f"year={year}: {pred_str}")
|
| 127 |
|
| 128 |
+
|
| 129 |
+
print()
|
| 130 |
+
|
| 131 |
+
# Example 2: probability of a specific word, across years
|
| 132 |
sentence = "The train arrived at the station on time"
|
| 133 |
+
for year in [1800, 1810, 1820, 1830, 1840, 1850, 1860, 1870, 1880, 1890, 1900, 1910, 1920]:
|
| 134 |
prob, logprob_sum = get_target_word_probability(sentence, "train", year)
|
| 135 |
+
if prob is not None:
|
| 136 |
+
print(f"year={year}: P(train)={prob:.10f} (logprob_sum={logprob_sum:.4f})")
|
| 137 |
+
else:
|
| 138 |
+
print(f"year={year}: logprob_sum={logprob_sum:.4f} (multi-subtoken word, no single probability)")
|
| 139 |
+
```
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
Example outputs:
|
| 143 |
+
```
|
| 144 |
+
year=1800: express(0.048), steamer(0.043), mail(0.031), train(0.018), news(0.014)
|
| 145 |
+
year=1810: express(0.064), steamer(0.044), mail(0.035), train(0.026), news(0.015)
|
| 146 |
+
year=1820: express(0.068), steamer(0.044), mail(0.035), train(0.032), news(0.017)
|
| 147 |
+
year=1830: express(0.078), steamer(0.043), mail(0.038), train(0.033), news(0.019)
|
| 148 |
+
year=1840: express(0.087), steamer(0.040), train(0.039), mail(0.038), news(0.023)
|
| 149 |
+
year=1850: express(0.081), train(0.042), steamer(0.039), mail(0.038), news(0.024)
|
| 150 |
+
year=1860: express(0.079), train(0.042), mail(0.038), steamer(0.035), news(0.029)
|
| 151 |
+
year=1870: express(0.080), train(0.050), mail(0.037), news(0.034), telegram(0.032)
|
| 152 |
+
year=1880: express(0.069), train(0.051), news(0.035), mail(0.035), telegram(0.032)
|
| 153 |
+
year=1890: express(0.063), train(0.052), news(0.039), telegram(0.034), mail(0.034)
|
| 154 |
+
year=1900: express(0.059), train(0.048), news(0.046), telegram(0.038), mail(0.033)
|
| 155 |
+
year=1910: express(0.054), news(0.052), train(0.047), telegram(0.039), mail(0.031)
|
| 156 |
+
year=1920: news(0.057), express(0.049), train(0.044), telegram(0.040), mail(0.030)
|
| 157 |
+
```
|
| 158 |
+
|
| 159 |
+
```
|
| 160 |
+
year=1800: P(train)=0.2596983756 (logprob_sum=-1.3482)
|
| 161 |
+
year=1810: P(train)=0.3239273912 (logprob_sum=-1.1272)
|
| 162 |
+
year=1820: P(train)=0.3619919056 (logprob_sum=-1.0161)
|
| 163 |
+
year=1830: P(train)=0.3642843769 (logprob_sum=-1.0098)
|
| 164 |
+
year=1840: P(train)=0.3824294963 (logprob_sum=-0.9612)
|
| 165 |
+
year=1850: P(train)=0.3897419142 (logprob_sum=-0.9423)
|
| 166 |
+
year=1860: P(train)=0.3831270530 (logprob_sum=-0.9594)
|
| 167 |
+
year=1870: P(train)=0.3918517235 (logprob_sum=-0.9369)
|
| 168 |
+
year=1880: P(train)=0.3810631289 (logprob_sum=-0.9648)
|
| 169 |
+
year=1890: P(train)=0.3701365410 (logprob_sum=-0.9939)
|
| 170 |
+
year=1900: P(train)=0.3599881482 (logprob_sum=-1.0217)
|
| 171 |
+
year=1910: P(train)=0.3460587446 (logprob_sum=-1.0611)
|
| 172 |
+
year=1920: P(train)=0.3430851355 (logprob_sum=-1.0698)
|
| 173 |
```
|
| 174 |
|
| 175 |
## Known limitations
|