| import os |
| import unittest |
| from pathlib import Path |
| from typing import Callable |
|
|
| import pytest |
|
|
| from transformers.utils.import_utils import ( |
| Backend, |
| VersionComparison, |
| define_import_structure, |
| spread_import_structure, |
| ) |
|
|
|
|
| import_structures = Path(__file__).parent / "import_structures" |
|
|
|
|
| def fetch__all__(file_content): |
| """ |
| Returns the content of the __all__ variable in the file content. |
| Returns None if not defined, otherwise returns a list of strings. |
| """ |
| lines = file_content.split("\n") |
| for line_index in range(len(lines)): |
| line = lines[line_index] |
| if line.startswith("__all__ = "): |
| |
| if line.endswith("]"): |
| return [obj.strip("\"' ") for obj in line.split("=")[1].strip(" []").split(",")] |
|
|
| |
| else: |
| _all = [] |
| for __all__line_index in range(line_index + 1, len(lines)): |
| if lines[__all__line_index].strip() == "]": |
| return _all |
| else: |
| _all.append(lines[__all__line_index].strip("\"', ")) |
|
|
|
|
| class TestImportStructures(unittest.TestCase): |
| base_transformers_path = Path(__file__).parent.parent.parent |
| models_path = base_transformers_path / "src" / "transformers" / "models" |
| models_import_structure = spread_import_structure(define_import_structure(models_path)) |
|
|
| def test_definition(self): |
| import_structure = define_import_structure(import_structures) |
| valid_frozensets: dict[frozenset | frozenset[str], dict[str, set[str]]] = { |
| frozenset(): { |
| "import_structure_raw_register": {"A0", "A4", "a0"}, |
| "import_structure_register_with_comments": {"B0", "b0"}, |
| }, |
| frozenset({"random_item_that_should_not_exist"}): {"failing_export": {"A0"}}, |
| frozenset({"torch"}): { |
| "import_structure_register_with_duplicates": {"C0", "C1", "C2", "C3", "c0", "c1", "c2", "c3"} |
| }, |
| frozenset({"tf", "torch"}): { |
| "import_structure_raw_register": {"A1", "A2", "A3", "a1", "a2", "a3"}, |
| "import_structure_register_with_comments": {"B1", "B2", "B3", "b1", "b2", "b3"}, |
| }, |
| frozenset({"torch>=2.5"}): {"import_structure_raw_register_with_versions": {"D0", "d0"}}, |
| frozenset({"torch>2.5"}): {"import_structure_raw_register_with_versions": {"D1", "d1"}}, |
| frozenset({"torch<=2.5"}): {"import_structure_raw_register_with_versions": {"D2", "d2"}}, |
| frozenset({"torch<2.5"}): {"import_structure_raw_register_with_versions": {"D3", "d3"}}, |
| frozenset({"torch==2.5"}): {"import_structure_raw_register_with_versions": {"D4", "d4"}}, |
| frozenset({"torch!=2.5"}): {"import_structure_raw_register_with_versions": {"D5", "d5"}}, |
| frozenset({"torch>=2.5", "accelerate<0.20"}): { |
| "import_structure_raw_register_with_versions": {"D6", "d6"} |
| }, |
| } |
|
|
| self.assertEqual(len(import_structure.keys()), len(valid_frozensets.keys())) |
| for _frozenset in valid_frozensets.keys(): |
| self.assertTrue(_frozenset in import_structure) |
| self.assertListEqual(list(import_structure[_frozenset].keys()), list(valid_frozensets[_frozenset].keys())) |
| for module, objects in valid_frozensets[_frozenset].items(): |
| self.assertTrue(module in import_structure[_frozenset]) |
| self.assertSetEqual(objects, import_structure[_frozenset][module]) |
|
|
| def test_transformers_specific_model_import(self): |
| """ |
| This test ensures that there is equivalence between what is written down in __all__ and what is |
| written down with register(). |
| |
| It doesn't test the backends attributed to register(). |
| """ |
| for architecture in os.listdir(self.models_path): |
| if ( |
| os.path.isfile(self.models_path / architecture) |
| or architecture.startswith("_") |
| or architecture == "deprecated" |
| ): |
| continue |
|
|
| with self.subTest(f"Testing arch {architecture}"): |
| import_structure = define_import_structure(self.models_path / architecture) |
| backend_agnostic_import_structure = {} |
| for requirement, module_object_mapping in import_structure.items(): |
| for module, objects in module_object_mapping.items(): |
| if module not in backend_agnostic_import_structure: |
| backend_agnostic_import_structure[module] = [] |
|
|
| backend_agnostic_import_structure[module].extend(objects) |
|
|
| for module, objects in backend_agnostic_import_structure.items(): |
| with open(self.models_path / architecture / f"{module}.py") as f: |
| content = f.read() |
| _all = fetch__all__(content) |
|
|
| if _all is None: |
| raise ValueError(f"{module} doesn't have __all__ defined.") |
|
|
| error_message = ( |
| f"self.models_path / architecture / f'{module}.py doesn't seem to be defined correctly:\n" |
| f"Defined in __all__: {sorted(_all)}\nDefined with register: {sorted(objects)}" |
| ) |
| self.assertListEqual(sorted(objects), sorted(_all), msg=error_message) |
|
|
| def test_import_spread(self): |
| """ |
| This test is specifically designed to test that varying levels of depth across import structures are |
| respected. |
| |
| In this instance, frozensets are at respective depths of 1, 2 and 3, for example: |
| - models.{frozensets} |
| - models.albert.{frozensets} |
| - models.deprecated.transfo_xl.{frozensets} |
| """ |
| initial_import_structure = { |
| frozenset(): {"dummy_non_model": {"DummyObject"}}, |
| "models": { |
| frozenset(): {"dummy_config": {"DummyConfig"}}, |
| "albert": { |
| frozenset(): {"configuration_albert": {"AlbertConfig", "AlbertOnnxConfig"}}, |
| frozenset({"torch"}): { |
| "modeling_albert": { |
| "AlbertForMaskedLM", |
| } |
| }, |
| }, |
| "llama": { |
| frozenset(): {"configuration_llama": {"LlamaConfig"}}, |
| frozenset({"torch"}): { |
| "modeling_llama": { |
| "LlamaForCausalLM", |
| } |
| }, |
| }, |
| "deprecated": { |
| "transfo_xl": { |
| frozenset({"torch"}): { |
| "modeling_transfo_xl": { |
| "TransfoXLModel", |
| } |
| }, |
| frozenset(): { |
| "configuration_transfo_xl": {"TransfoXLConfig"}, |
| "tokenization_transfo_xl": {"TransfoXLCorpus", "TransfoXLTokenizer"}, |
| }, |
| }, |
| "deta": { |
| frozenset({"torch"}): { |
| "modeling_deta": {"DetaForObjectDetection", "DetaModel", "DetaPreTrainedModel"} |
| }, |
| frozenset(): {"configuration_deta": {"DetaConfig"}}, |
| frozenset({"vision"}): {"image_processing_deta": {"DetaImageProcessor"}}, |
| }, |
| }, |
| }, |
| } |
|
|
| ground_truth_spread_import_structure = { |
| frozenset(): { |
| "dummy_non_model": {"DummyObject"}, |
| "models.dummy_config": {"DummyConfig"}, |
| "models.albert.configuration_albert": {"AlbertConfig", "AlbertOnnxConfig"}, |
| "models.llama.configuration_llama": {"LlamaConfig"}, |
| "models.deprecated.transfo_xl.configuration_transfo_xl": {"TransfoXLConfig"}, |
| "models.deprecated.transfo_xl.tokenization_transfo_xl": {"TransfoXLCorpus", "TransfoXLTokenizer"}, |
| "models.deprecated.deta.configuration_deta": {"DetaConfig"}, |
| }, |
| frozenset({"torch"}): { |
| "models.albert.modeling_albert": {"AlbertForMaskedLM"}, |
| "models.llama.modeling_llama": {"LlamaForCausalLM"}, |
| "models.deprecated.transfo_xl.modeling_transfo_xl": {"TransfoXLModel"}, |
| "models.deprecated.deta.modeling_deta": {"DetaForObjectDetection", "DetaModel", "DetaPreTrainedModel"}, |
| }, |
| frozenset({"vision"}): {"models.deprecated.deta.image_processing_deta": {"DetaImageProcessor"}}, |
| } |
|
|
| newly_spread_import_structure = spread_import_structure(initial_import_structure) |
|
|
| self.assertEqual(ground_truth_spread_import_structure, newly_spread_import_structure) |
|
|
|
|
| @pytest.mark.parametrize( |
| "backend,package_name,version_comparison,version", |
| [ |
| pytest.param(Backend("torch>=2.5 "), "torch", VersionComparison.GREATER_THAN_OR_EQUAL.value, "2.5"), |
| pytest.param(Backend("tf<=1"), "tf", VersionComparison.LESS_THAN_OR_EQUAL.value, "1"), |
| pytest.param(Backend("torchvision==0.19.1"), "torchvision", VersionComparison.EQUAL.value, "0.19.1"), |
| ], |
| ) |
| def test_backend_specification(backend: Backend, package_name: str, version_comparison: Callable, version: str): |
| assert backend.package_name == package_name |
| assert VersionComparison.from_string(backend.version_comparison) == version_comparison |
| assert backend.version == version |
|
|
