Ribeiro87 commited on
Commit
84ffcf8
·
verified ·
1 Parent(s): acc39e9

Add model.py, README.md, config.json, training_args.json, model.safetensors, NOTES.md, sample_usage.py

Browse files
Files changed (7) hide show
  1. NOTES.md +6 -0
  2. README.md +73 -0
  3. config.json +19 -0
  4. model.py +126 -0
  5. model.safetensors +3 -0
  6. sample_usage.py +12 -0
  7. training_args.json +9 -0
NOTES.md ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ # Notes
2
+
3
+ - check whether the config round-trips through from_pretrained
4
+ - compare parameter counts against a reference implementation
5
+ - move constants into the config file
6
+ - document the expected input shapes
README.md ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ tags:
4
+ - pytorch
5
+ - beit
6
+ - multitask
7
+ - prototype
8
+ - untrained
9
+ ---
10
+
11
+ # Beit for Multitask
12
+
13
+ ## Overview
14
+
15
+ Working implementation of **Beit** for **Multitask** using a **base** configuration. The repository focuses on transparent code and repeatable smoke tests; benchmark claims are deliberately omitted.
16
+
17
+ The whole setup fits in a single module on purpose.
18
+
19
+ ## Architecture
20
+
21
+ 1. **Architecture**: Beit
22
+ 2. **Scale**: base
23
+ 3. **Attention**: grouped query
24
+ 4. **Fusion**: low rank
25
+ 5. **Activation**: gelu tanh
26
+ 6. **Normalization**: layernorm
27
+
28
+ ## Limitations
29
+
30
+ The checkpoint is untrained and unaudited. Treat the repository as an experimental starting point; anything learned from a future trained version must be documented separately.
31
+
32
+ ## Environment
33
+
34
+ 1. **Python**: >= 3.9
35
+ 2. **torch**: 2.x
36
+ 3. **safetensors**: any recent
37
+ 4. **transformers**: optional, for loaders
38
+
39
+ ## Repository status
40
+
41
+ 1. The Python file contains the model and runnable example or training entry point.
42
+ 2. `config.json` records the generated architecture settings.
43
+ 3. `training_args.json` records the default experiment recipe.
44
+ 4. `model.safetensors` is a valid initialization checkpoint for smoke tests; it is **not** presented as a trained benchmark checkpoint.
45
+ 5. No benchmark score is claimed in this repository.
46
+
47
+ ## Quick check
48
+
49
+ ```bash
50
+ python model.py --help
51
+ ```
52
+
53
+ Inspect the script's `__main__` block for its generated smoke-test example. Because this is a custom implementation, generic automatic loading APIs require an explicit adapter before use.
54
+
55
+ ## Default experiment recipe
56
+
57
+ The included configuration uses **sgd** with a **exponential** schedule. These are starting values in the script, not evidence of a completed run. For a meaningful evaluation, train all baselines with the same data exposure, tuning budget, and random seeds.
58
+
59
+ ## Files
60
+
61
+ | File | Purpose |
62
+ |---|---|
63
+ | `model.py` | primary artifact |
64
+ | `README.md` | this documentation |
65
+ | `config.json` | architecture configuration |
66
+ | `training_args.json` | default experiment settings |
67
+ | `model.safetensors` | initialization checkpoint |
68
+ | `NOTES.md` | informal to-do notes |
69
+ | `examples/sample_usage.py` | usage sketch |
70
+
71
+ ## License
72
+
73
+ Released under **apache-2.0**. Review the source-data terms separately when this repository is used with external datasets.
config.json ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "CustomResearchModel"
4
+ ],
5
+ "architecture": "beit",
6
+ "model_type": "beit",
7
+ "hidden_size": 192,
8
+ "num_hidden_layers": 6,
9
+ "num_attention_heads": 8,
10
+ "intermediate_size": 768,
11
+ "hidden_act": "gelu_tanh",
12
+ "max_position_embeddings": 512,
13
+ "layer_norm_eps": 1e-12,
14
+ "checkpoint_status": "initialization-only",
15
+ "notes": "Untrained checkpoint for smoke tests; no benchmark claim.",
16
+ "use_cache": true,
17
+ "torch_dtype": "float32",
18
+ "vocab_size": 2048
19
+ }
model.py ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ import math
4
+
5
+
6
+ class BeitModel(nn.Module):
7
+ '''
8
+ beit model with grouped_query attention.
9
+ Scale: base (dim=256, layers=6, heads=4)
10
+ Fusion: low_rank, Task: multitask
11
+ '''
12
+
13
+ def __init__(self, embed_dim=256, num_layers=6, num_heads=4, num_classes=10):
14
+ super().__init__()
15
+ self.embed_dim = embed_dim
16
+ self.num_layers = num_layers
17
+ self.num_heads = num_heads
18
+
19
+ # image patch embedding
20
+ self.patch_proj = nn.Conv2d(3, embed_dim, kernel_size=16, stride=16)
21
+ self.cls_token = nn.Parameter(torch.randn(1, 1, embed_dim) * 0.02)
22
+ self.pos_embed = nn.Parameter(torch.randn(1, 197, embed_dim) * 0.02)
23
+ self.dropout = nn.Dropout(0.1)
24
+
25
+ # image transformer blocks
26
+ self.image_blocks = nn.ModuleList([
27
+ nn.TransformerEncoderLayer(
28
+ embed_dim, num_heads, embed_dim * 4, 0.1,
29
+ activation='gelu', batch_first=True, norm_first=True
30
+ )
31
+ for _ in range(num_layers)
32
+ ])
33
+ self.image_norm = nn.LayerNorm(embed_dim)
34
+
35
+ # text embedding
36
+ self.text_embed = nn.Embedding(30522, embed_dim, padding_idx=0)
37
+ self.text_pos = nn.Parameter(torch.randn(1, 128, embed_dim) * 0.02)
38
+ self.text_blocks = nn.ModuleList([
39
+ nn.TransformerEncoderLayer(
40
+ embed_dim, num_heads, embed_dim * 4, 0.1,
41
+ activation='gelu', batch_first=True, norm_first=True
42
+ )
43
+ for _ in range(num_layers)
44
+ ])
45
+ self.text_norm = nn.LayerNorm(embed_dim)
46
+
47
+ # fusion
48
+ self.fusion_blocks = nn.ModuleList([
49
+ nn.TransformerEncoderLayer(
50
+ embed_dim, num_heads, embed_dim * 4, 0.1,
51
+ activation='gelu', batch_first=True, norm_first=True
52
+ )
53
+ for _ in range(2)
54
+ ])
55
+ self.fusion_norm = nn.LayerNorm(embed_dim)
56
+
57
+ # task head
58
+ self.classifier = nn.Sequential(
59
+ nn.Linear(embed_dim, embed_dim),
60
+ nn.GELU(approximate='tanh'),
61
+ nn.Dropout(0.1),
62
+ nn.Linear(embed_dim, num_classes),
63
+ )
64
+
65
+ self._initialize_weights()
66
+
67
+ def _initialize_weights(self):
68
+ for m in self.modules():
69
+ if isinstance(m, nn.Linear):
70
+ nn.init.kaiming_normal_(m.weight)
71
+ if m.bias is not None:
72
+ nn.init.zeros_(m.bias)
73
+ elif isinstance(m, nn.Embedding):
74
+ nn.init.trunc_normal_(m.weight, std=0.02)
75
+ if m.padding_idx is not None:
76
+ m.weight[m.padding_idx].zero_()
77
+ elif isinstance(m, nn.LayerNorm):
78
+ nn.init.ones_(m.weight)
79
+ nn.init.zeros_(m.bias)
80
+
81
+ def encode_image(self, images):
82
+ x = self.patch_proj(images)
83
+ x = x.flatten(2).transpose(1, 2)
84
+ cls = self.cls_token.expand(x.size(0), -1, -1)
85
+ x = torch.cat([cls, x], dim=1)
86
+ x = x + self.pos_embed
87
+ x = self.dropout(x)
88
+ for block in self.image_blocks:
89
+ x = block(x)
90
+ return self.image_norm(x)
91
+
92
+ def encode_text(self, input_ids, attention_mask=None):
93
+ x = self.text_embed(input_ids)
94
+ x = x + self.text_pos[:, :input_ids.size(1)]
95
+ x = self.dropout(x)
96
+ padding = (attention_mask == 0) if attention_mask is not None else None
97
+ for block in self.text_blocks:
98
+ x = block(x, src_key_padding_mask=padding)
99
+ return self.text_norm(x)
100
+
101
+ def forward(self, images, input_ids, attention_mask=None, labels=None):
102
+ image_features = self.encode_image(images)
103
+ text_features = self.encode_text(input_ids, attention_mask)
104
+
105
+ fused = text_features
106
+ for block in self.fusion_blocks:
107
+ fused = block(fused)
108
+ fused = self.fusion_norm(fused[:, 0])
109
+ logits = self.classifier(fused)
110
+
111
+ loss = None
112
+ if labels is not None:
113
+ loss = nn.functional.cross_entropy(logits, labels)
114
+
115
+ return {'logits': logits, 'loss': loss}
116
+
117
+
118
+ if __name__ == '__main__':
119
+ model = BeitModel()
120
+ total = sum(p.numel() for p in model.parameters())
121
+ print(f'BeitModel: {total:,} params ({total/1e6:.2f}M)')
122
+ img = torch.randn(2, 3, 224, 224)
123
+ ids = torch.randint(0, 30522, (2, 128))
124
+ mask = torch.ones(2, 128)
125
+ out = model(img, ids, mask, torch.tensor([0, 1]))
126
+ print(f'Output: {out["logits"].shape}, Loss: {out["loss"].item():.4f}')
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1758a04d5fd54449b4eb33bfe859f9d6418d1aa9e3b08166e84259271358c2e4
3
+ size 99808
sample_usage.py ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Minimal usage sketch. Not an executable pipeline."""
2
+
3
+ import json
4
+
5
+
6
+ def load_settings(path="config.json"):
7
+ with open(path, "r", encoding="utf-8") as f:
8
+ return json.load(f)
9
+
10
+
11
+ if __name__ == "__main__":
12
+ print(load_settings())
training_args.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "optimizer": "sgd",
3
+ "scheduler": "exponential",
4
+ "learning_rate": 5e-05,
5
+ "batch_size": 24,
6
+ "epochs": 30,
7
+ "seed": 42,
8
+ "status": "default recipe; not a completed run"
9
+ }