Buckets:
| import unittest | |
| import torch | |
| from shared.labels import canonical_autonomous_text_labels | |
| from uniar import SPECIAL_TOKEN_IDS | |
| class SharedSupervisedTest(unittest.TestCase): | |
| def test_autonomous_labels_keep_vision_start_and_mask_controller_end(self): | |
| token = SPECIAL_TOKEN_IDS | |
| ids = torch.tensor( | |
| [ | |
| 7, | |
| token["<|vision_start|>"], | |
| token["<|image_pad|>"], | |
| token["<|vision_end|>"], | |
| token["<|im_end|>"], | |
| ] | |
| ) | |
| labels = ids.clone() | |
| labels[0] = -100 | |
| result = canonical_autonomous_text_labels(ids, labels) | |
| self.assertEqual(int(result[1]), token["<|vision_start|>"]) | |
| self.assertEqual(int(result[2]), -100) | |
| self.assertEqual(int(result[3]), -100) | |
| self.assertEqual(int(result[4]), token["<|im_end|>"]) | |
| self.assertEqual(int(labels[3]), token["<|vision_end|>"]) | |
| def test_autonomous_labels_require_aligned_shapes(self): | |
| with self.assertRaisesRegex(ValueError, "identical shapes"): | |
| canonical_autonomous_text_labels( | |
| torch.zeros(2, dtype=torch.long), | |
| torch.zeros(3, dtype=torch.long), | |
| ) | |
| if __name__ == "__main__": | |
| unittest.main() | |
Xet Storage Details
- Size:
- 1.28 kB
- Xet hash:
- bc20143c94afb594e76bc372455f82c2a26c9d27bbcb549ed1955f679bc2634a
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.