Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /DiamondLightSource_python-murfey /tests /util /test_config.py
| import copy | |
| from pathlib import Path | |
| from typing import Any | |
| import pytest | |
| import yaml | |
| from pytest_mock import MockerFixture | |
| from murfey.util.config import Settings, get_machine_config | |
| def mock_general_config(): | |
| # Most extra keys go in this category | |
| return { | |
| "pkg_2": { | |
| "url": "https://some-url.some.org", | |
| "token": "pneumonoultrasmicroscopicsilicovolcanoconiosis", | |
| } | |
| } | |
| def mock_tem_shared_config(): | |
| return { | |
| # Hardware and software | |
| "acquisition_software": ["epu", "tomo", "serialem"], | |
| "software_versions": {"tomo": "5.12"}, | |
| "data_required_substrings": { | |
| "epu": { | |
| ".mrc": ["fractions", "Fractions"], | |
| ".tiff": ["fractions", "Fractions"], | |
| ".eer": ["EER"], | |
| }, | |
| "tomo": { | |
| ".mrc": ["fractions", "Fractions"], | |
| ".tiff": ["fractions", "Fractions"], | |
| ".eer": ["EER"], | |
| }, | |
| }, | |
| # Client directory setup | |
| "analyse_created_directories": ["atlas"], | |
| "gain_reference_directory": "C:/ProgramData/Gatan/Reference Images/", | |
| # Data transfer keys | |
| "data_transfer_enabled": True, | |
| "substrings_blacklist": { | |
| "directories": ["some_str"], | |
| "files": ["some_str"], | |
| }, | |
| "rsync_module": "rsync", | |
| "allow_removal": True, | |
| "upstream_data_directories": { | |
| "upstream_instrument": "/path/to/upstream_instrument", | |
| }, | |
| "upstream_data_download_directory": "/path/to/download/directory", | |
| "upstream_data_search_strings": { | |
| "upstream_instrument": ["some_string"], | |
| }, | |
| # Data processing keys | |
| "processing_enabled": True, | |
| "gain_directory_name": "some_directory", | |
| "processed_directory_name": "some_directory", | |
| "processed_extra_directory": "some_directory", | |
| "recipes": { | |
| "recipe_1": "recipe_1", | |
| "recipe_2": "recipe_2", | |
| }, | |
| "default_model": "some_file", | |
| "external_executables": { | |
| "app_1": "/path/to/app_1", | |
| "app_2": "/path/to/app_2", | |
| "app_3": "/path/to/app_3", | |
| }, | |
| "external_executables_eer": { | |
| "app_1": "/path/to/app_1", | |
| "app_2": "/path/to/app_2", | |
| "app_3": "/path/to/app_3", | |
| }, | |
| "external_environment": { | |
| "ENV_1": "/path/to/env_1", | |
| "ENV_2": "/path/to/env_2", | |
| }, | |
| "plugin_packages": { | |
| "pkg_1": "/path/to/pkg_1", | |
| "pkg_2": "/path/to/pkg_2", | |
| }, | |
| # Extra keys | |
| "pkg_1": { | |
| "file_path": "", | |
| "command": [ | |
| "/path/to/executable", | |
| "--some_arg", | |
| "-a", | |
| "./path/to/file", | |
| ], | |
| "step_size": 100, | |
| }, | |
| } | |
| def mock_instrument_config(): | |
| return { | |
| # Extra key to point to hierarchical dictionary to use | |
| "instrument_type": "tem", | |
| # General information | |
| "display_name": "Some TEM", | |
| "image_path": "/path/to/tem.jpg", | |
| # Hardware and software | |
| "camera": "Some camera", | |
| "superres": True, | |
| "calibrations": { | |
| "magnification": { | |
| 100: 0.1, | |
| 200: 0.05, | |
| 400: 0.025, | |
| }, | |
| }, | |
| # Client directory setup | |
| "data_directories": ["C:"], | |
| # Data transfer keys | |
| "rsync_basepath": "/path/to/data", | |
| "rsync_url": "http://123.45.678.90:8000", | |
| # Server and network keys | |
| "security_configuration_path": "/path/to/security-config.yaml", | |
| "murfey_url": "https://www.murfey.com", | |
| "instrument_server_url": "http://10.123.4.5:8000", | |
| "node_creator_queue": "node_creator", | |
| # Extra keys | |
| "pkg_1": { | |
| "file_path": "/path/to/pkg_1/file.txt", | |
| }, | |
| } | |
| def mock_second_instrument_config(mock_instrument_config: dict[str, Any]): | |
| """Test the case of recursive dictionaries which are present | |
| in both the general and instrument configs""" | |
| second_instrument_config = copy.deepcopy(mock_instrument_config) | |
| second_instrument_config.update( | |
| {"pkg_1": {"file_path": "/path/to/pkg_1/file_2.txt"}} | |
| ) | |
| return second_instrument_config | |
| def mock_hierarchical_machine_config_yaml( | |
| mock_general_config: dict[str, Any], | |
| mock_tem_shared_config: dict[str, Any], | |
| mock_instrument_config: dict[str, Any], | |
| mock_second_instrument_config: dict[str, Any], | |
| tmp_path: Path, | |
| ): | |
| # Create machine config (with all currently supported keys) for the instrument | |
| hierarchical_config = { | |
| "general": mock_general_config, | |
| "tem": mock_tem_shared_config, | |
| "m01": mock_instrument_config, | |
| "m02": mock_second_instrument_config, | |
| } | |
| config_file = tmp_path / "config" / "murfey-machine-config-hierarchical.yaml" | |
| config_file.parent.mkdir(parents=True, exist_ok=True) | |
| with open(config_file, "w") as file: | |
| yaml.safe_dump(hierarchical_config, file, indent=2) | |
| return config_file | |
| def mock_standard_machine_config_yaml( | |
| mock_general_config: dict[str, Any], | |
| mock_tem_shared_config: dict[str, Any], | |
| mock_instrument_config: dict[str, Any], | |
| tmp_path: Path, | |
| ): | |
| # Compile the different dictionaries into one dictionary for the instrument | |
| machine_config = { | |
| key: value | |
| for config in ( | |
| mock_general_config, | |
| mock_tem_shared_config, | |
| mock_instrument_config, | |
| ) | |
| for key, value in config.items() | |
| } | |
| # Correct nested dicts that would have been partially overwritten | |
| machine_config["pkg_1"] = { | |
| "file_path": "/path/to/pkg_1/file.txt", | |
| "command": [ | |
| "/path/to/executable", | |
| "--some_arg", | |
| "-a", | |
| "./path/to/file", | |
| ], | |
| "step_size": 100, | |
| } | |
| # Remove 'instrument_type' value (not needed in standard config) | |
| machine_config["instrument_type"] = "" | |
| master_config = { | |
| "m01": machine_config, | |
| "m02": machine_config, | |
| } | |
| config_file = tmp_path / "config" / "murfey-machine-config-standard.yaml" | |
| config_file.parent.mkdir(parents=True, exist_ok=True) | |
| with open(config_file, "w") as file: | |
| yaml.safe_dump(master_config, file, indent=2) | |
| return config_file | |
| get_machine_config_test_matrix: tuple[tuple[str, list[str]], ...] = ( | |
| # Config to test | Instrument names to pass to function | |
| ("hierarchical", ["", "m01", "m02"]), | |
| ("standard", ["", "m01", "m02"]), | |
| ) | |
| def test_get_machine_config( | |
| mocker: MockerFixture, | |
| mock_general_config: dict[str, Any], | |
| mock_tem_shared_config: dict[str, Any], | |
| mock_instrument_config: dict[str, Any], | |
| mock_second_instrument_config: dict[str, Any], | |
| mock_hierarchical_machine_config_yaml: Path, | |
| mock_standard_machine_config_yaml: Path, | |
| test_params: tuple[str, list[str]], | |
| ): | |
| # Unpack test params | |
| config_to_test, instrument_names = test_params | |
| # Set up mocks | |
| mock_settings = mocker.patch("murfey.util.config.settings", spec=Settings) | |
| # Run 'get_machine_config' using different instrument name parameters | |
| for i in instrument_names: | |
| # Patch the 'settings' environment variable with the YAML file to test | |
| mock_settings.murfey_machine_configuration = ( | |
| str(mock_hierarchical_machine_config_yaml) | |
| if config_to_test == "hierarchical" | |
| else str(mock_standard_machine_config_yaml) | |
| ) | |
| # Run the function | |
| config = get_machine_config(i) | |
| # Validate that the config was loaded correctly | |
| assert config | |
| # Multiple configs should be returned if instrument name was "" | |
| assert len(config) == 2 if i == "" else len(config) == 1 | |
| # When getting the config for individual microscopes, validate key-by-key | |
| if i != "": | |
| # General info | |
| assert config[i].display_name == mock_instrument_config["display_name"] | |
| assert config[i].image_path == Path(mock_instrument_config["image_path"]) | |
| assert ( | |
| config[i].instrument_type == mock_instrument_config["instrument_type"] | |
| if config_to_test == "hierarchical" | |
| else not config[i].instrument_type | |
| ) | |
| # Hardware & software | |
| assert config[i].camera == mock_instrument_config["camera"] | |
| assert config[i].superres == mock_instrument_config["superres"] | |
| assert config[i].calibrations == mock_instrument_config["calibrations"] | |
| assert ( | |
| config[i].acquisition_software | |
| == mock_tem_shared_config["acquisition_software"] | |
| ) | |
| assert ( | |
| config[i].software_versions | |
| == mock_tem_shared_config["software_versions"] | |
| ) | |
| assert ( | |
| config[i].data_required_substrings | |
| == mock_tem_shared_config["data_required_substrings"] | |
| ) | |
| # Client directory setup | |
| assert config[i].data_directories == [ | |
| Path(p) for p in mock_instrument_config["data_directories"] | |
| ] | |
| assert ( | |
| config[i].analyse_created_directories | |
| == mock_tem_shared_config["analyse_created_directories"] | |
| ) | |
| assert config[i].gain_reference_directory == Path( | |
| mock_tem_shared_config["gain_reference_directory"] | |
| ) | |
| # Data transfer setup | |
| assert ( | |
| config[i].data_transfer_enabled | |
| == mock_tem_shared_config["data_transfer_enabled"] | |
| ) | |
| assert ( | |
| config[i].substrings_blacklist | |
| == mock_tem_shared_config["substrings_blacklist"] | |
| ) | |
| assert config[i].rsync_url == mock_instrument_config["rsync_url"] | |
| assert config[i].rsync_basepath == Path( | |
| mock_instrument_config["rsync_basepath"] | |
| ) | |
| assert config[i].rsync_module == mock_tem_shared_config["rsync_module"] | |
| assert config[i].allow_removal == mock_tem_shared_config["allow_removal"] | |
| assert config[i].upstream_data_directories == { | |
| key: Path(value) | |
| for key, value in mock_tem_shared_config[ | |
| "upstream_data_directories" | |
| ].items() | |
| } | |
| assert config[i].upstream_data_download_directory == Path( | |
| mock_tem_shared_config["upstream_data_download_directory"] | |
| ) | |
| assert ( | |
| config[i].upstream_data_search_strings | |
| == mock_tem_shared_config["upstream_data_search_strings"] | |
| ) | |
| # Data processing setup | |
| assert ( | |
| config[i].processing_enabled | |
| == mock_tem_shared_config["processing_enabled"] | |
| ) | |
| assert ( | |
| config[i].gain_directory_name | |
| == mock_tem_shared_config["gain_directory_name"] | |
| ) | |
| assert ( | |
| config[i].processed_directory_name | |
| == mock_tem_shared_config["processed_directory_name"] | |
| ) | |
| assert ( | |
| config[i].processed_extra_directory | |
| == mock_tem_shared_config["processed_extra_directory"] | |
| ) | |
| assert config[i].recipes == mock_tem_shared_config["recipes"] | |
| assert config[i].default_model == Path( | |
| mock_tem_shared_config["default_model"] | |
| ) | |
| assert ( | |
| config[i].external_executables | |
| == mock_tem_shared_config["external_executables"] | |
| ) | |
| assert ( | |
| config[i].external_executables_eer | |
| == mock_tem_shared_config["external_executables_eer"] | |
| ) | |
| assert ( | |
| config[i].external_environment | |
| == mock_tem_shared_config["external_environment"] | |
| ) | |
| assert config[i].plugin_packages == { | |
| key: Path(value) | |
| for key, value in mock_tem_shared_config["plugin_packages"].items() | |
| } | |
| # Server and network setup | |
| assert config[i].security_configuration_path == Path( | |
| mock_instrument_config["security_configuration_path"] | |
| ) | |
| assert config[i].murfey_url == mock_instrument_config["murfey_url"] | |
| assert ( | |
| config[i].instrument_server_url | |
| == mock_instrument_config["instrument_server_url"] | |
| ) | |
| assert ( | |
| config[i].node_creator_queue | |
| == mock_instrument_config["node_creator_queue"] | |
| ) | |
| # Extra keys | |
| if config_to_test == "standard" or i == "m01": | |
| assert config[i].pkg_1 == { | |
| "file_path": "/path/to/pkg_1/file.txt", | |
| "command": [ | |
| "/path/to/executable", | |
| "--some_arg", | |
| "-a", | |
| "./path/to/file", | |
| ], | |
| "step_size": 100, | |
| } | |
| elif i == "m02": | |
| assert config[i].pkg_1 == { | |
| "file_path": "/path/to/pkg_1/file_2.txt", | |
| "command": [ | |
| "/path/to/executable", | |
| "--some_arg", | |
| "-a", | |
| "./path/to/file", | |
| ], | |
| "step_size": 100, | |
| } | |
| assert config[i].pkg_2 == mock_general_config["pkg_2"] | |
Xet Storage Details
- Size:
- 14 kB
- Xet hash:
- 4cf538ff080d0b048dc677c113051c1956b69703c21fa1afbe3abeba8fc3658a
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.