Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /DiamondLightSource_python-murfey /tests /client /test_rsync.py
| import queue | |
| import threading | |
| from datetime import datetime | |
| from pathlib import Path | |
| from unittest import mock | |
| from unittest.mock import MagicMock | |
| import pytest | |
| from pytest_mock import MockerFixture | |
| from murfey.client.rsync import RSyncer | |
| from tests.conftest import ExampleVisit | |
| def rsync_module(): | |
| return "data" | |
| def mock_server_url(): | |
| mock_url = MagicMock() | |
| mock_url.hostname = "10.0.0.1" | |
| return mock_url | |
| # Create a dummy callback function | |
| def dummy_callback(): | |
| return None | |
| def test_rsyncer_initialises( | |
| tmp_path: Path, | |
| rsync_module: str, | |
| mock_server_url: MagicMock, | |
| is_local: bool, | |
| ): | |
| # Assign values to parameters | |
| basepath_local = tmp_path / "local" | |
| basepath_remote = tmp_path / "remote" | |
| # Create a test substrings blacklist dict | |
| substrings_blacklist = { | |
| "directories": ["1", "2", "3"], | |
| "files": ["a", "b", "c"], | |
| } | |
| # Create a timestamp | |
| timestamp = datetime.now() | |
| # Initialise the RSyncer | |
| rsyncer = RSyncer( | |
| basepath_local=basepath_local, | |
| basepath_remote=basepath_remote, | |
| rsync_module=rsync_module, | |
| server_url=mock_server_url, | |
| stop_callback=dummy_callback, | |
| local=is_local, | |
| substrings_blacklist=substrings_blacklist, | |
| end_time=timestamp, | |
| ) | |
| # Check that the attributes are as expected | |
| assert rsyncer._basepath == basepath_local.absolute() | |
| assert rsyncer._basepath_remote == basepath_remote | |
| assert rsyncer._rsync_module == rsync_module | |
| assert rsyncer._server_url == mock_server_url | |
| assert rsyncer._stop_callback == dummy_callback | |
| assert rsyncer._local == is_local | |
| assert rsyncer._do_transfer | |
| assert not rsyncer._remove_files | |
| assert rsyncer._required_substrings_for_removal == [] | |
| assert rsyncer._substrings_blacklist == substrings_blacklist | |
| assert rsyncer._notify | |
| assert rsyncer._end_time == timestamp | |
| assert rsyncer._skipped_files == [] | |
| assert ( | |
| rsyncer._remote == str(basepath_remote) | |
| if is_local | |
| else f"{mock_server_url.hostname}::{rsync_module}/{basepath_remote}" | |
| ) | |
| assert rsyncer._files_transferred == 0 | |
| assert rsyncer._bytes_transferred == 0 | |
| assert isinstance(rsyncer.queue, queue.Queue) | |
| assert isinstance(rsyncer.thread, threading.Thread) | |
| assert not rsyncer._stopping | |
| assert not rsyncer._halt_thread | |
| assert not rsyncer._finalising | |
| assert not rsyncer._finalised | |
| def test_rsyncer_status( | |
| tmp_path: Path, | |
| mock_server_url: MagicMock, | |
| test_params: tuple[bool, bool, str], | |
| ): | |
| # Unpack test params | |
| is_stopping, is_thread_alive, expected_status = test_params | |
| # Mock the thread | |
| mock_thread = MagicMock() | |
| mock_thread.is_alive.return_value = is_thread_alive | |
| # Initialise the RSyncer and patch the attributes to be tested | |
| rsyncer = RSyncer( | |
| basepath_local=tmp_path / "local", | |
| basepath_remote=tmp_path / "remote", | |
| rsync_module=mock.ANY, | |
| server_url=mock_server_url, | |
| ) | |
| rsyncer.thread = mock_thread | |
| rsyncer._stopping = is_stopping | |
| # Check that its status is correct | |
| assert rsyncer.status == expected_status | |
| # Check that its canonical representation is correct | |
| assert str(rsyncer) == f"<RSyncer ({rsyncer._basepath} → {rsyncer._remote})>" | |
| def test_rsyncer_notify( | |
| mocker: MockerFixture, | |
| tmp_path: Path, | |
| mock_server_url: MagicMock, | |
| notify: bool, | |
| ): | |
| # Patch the superclass that RSyncer stems from | |
| mock_notify = mocker.patch("murfey.client.rsync.Observer.notify") | |
| mock_notify.return_value = None | |
| # Initialise the RSyncer | |
| rsyncer = RSyncer( | |
| basepath_local=tmp_path / "local", | |
| basepath_remote=tmp_path / "remote", | |
| rsync_module=mock.ANY, | |
| server_url=mock_server_url, | |
| notify=notify, | |
| ) | |
| # Check that the 'notify' attribute is set correctly | |
| assert rsyncer._notify == notify | |
| # Run 'notify' and check that the expected calls were made | |
| rsyncer.notify("arg1", "arg2", kwarg1="kwarg1", kwarg2="kwarg2") | |
| if notify: | |
| mock_notify.assert_called_once_with( | |
| "arg1", | |
| "arg2", | |
| secondary=False, | |
| kwarg1="kwarg1", | |
| kwarg2="kwarg2", | |
| ) | |
| else: | |
| mock_notify.assert_not_called() | |
| def test_rsyncer_start( | |
| tmp_path: Path, | |
| mock_server_url: MagicMock, | |
| rsyncer_status: str, | |
| ): | |
| # Mock the thread attribute so that it doesn't start an actual Thread | |
| mock_thread = MagicMock() | |
| mock_thread.start.return_value = None | |
| mock_thread.is_alive.return_value = rsyncer_status == "is_alive" | |
| # Initialise the RSyncer and patch the attributes to be tested | |
| rsyncer = RSyncer( | |
| basepath_local=tmp_path / "local", | |
| basepath_remote=tmp_path / "remote", | |
| rsync_module=mock.ANY, | |
| server_url=mock_server_url, | |
| ) | |
| rsyncer.thread = mock_thread | |
| rsyncer._stopping = rsyncer_status == "stopping" | |
| # Start the RSyncer | |
| if rsyncer_status == "default": | |
| rsyncer.start() | |
| mock_thread.start.assert_called_once() | |
| else: | |
| with pytest.raises(RuntimeError): | |
| rsyncer.start() | |
| def test_rsyncer_restart( | |
| mocker: MockerFixture, | |
| tmp_path: Path, | |
| mock_server_url: MagicMock, | |
| ): | |
| # Patch the 'start' class method, which is called by 'restart' | |
| mock_start = mocker.patch.object(RSyncer, "start") | |
| mock_start.return_value = None | |
| # Mock the thread and the attributes used | |
| mock_thread = MagicMock() | |
| mock_thread.join.return_value = None | |
| # Initialise the RSyncer and patch the attributes to be tested | |
| rsyncer = RSyncer( | |
| basepath_local=tmp_path / "local", | |
| basepath_remote=tmp_path / "remote", | |
| rsync_module=mock.ANY, | |
| server_url=mock_server_url, | |
| ) | |
| rsyncer.thread = mock_thread | |
| # Run 'restart' | |
| rsyncer.restart() | |
| # Check that the correct calls and attributes are present | |
| mock_thread.join.assert_called_once() | |
| assert not rsyncer._halt_thread | |
| assert isinstance(rsyncer.thread, threading.Thread) | |
| mock_start.assert_called_once() | |
| def test_rsyncer_stop( | |
| tmp_path: Path, | |
| mock_server_url: MagicMock, | |
| thread_is_alive: bool, | |
| ): | |
| # Mock the thread | |
| mock_thread = MagicMock() | |
| mock_thread.is_alive.return_value = thread_is_alive | |
| mock_thread.join.return_value = None | |
| # Mock the queue | |
| mock_queue = MagicMock() | |
| mock_queue.join.return_value = None | |
| mock_queue.put.return_value = None | |
| # Initialise the RSyncer and patch the attributes to be tested | |
| rsyncer = RSyncer( | |
| basepath_local=tmp_path / "local", | |
| basepath_remote=tmp_path / "remote", | |
| rsync_module=mock.ANY, | |
| server_url=mock_server_url, | |
| ) | |
| rsyncer.thread = mock_thread | |
| rsyncer.queue = mock_queue | |
| # Check that initial attributes are as expected | |
| assert not rsyncer._stopping | |
| assert not rsyncer._halt_thread | |
| # Run 'stop' and check that the calls are as expected | |
| rsyncer.stop() | |
| assert rsyncer._stopping | |
| assert rsyncer._halt_thread | |
| if thread_is_alive: | |
| mock_queue.join.assert_called_once() | |
| mock_queue.put.assert_called_with(None) | |
| mock_thread.join.assert_called_once() | |
| else: | |
| mock_queue.join.assert_not_called() | |
| mock_queue.put.assert_not_called() | |
| mock_thread.join.assert_not_called() | |
| def test_rsyncer_request_stop( | |
| tmp_path: Path, | |
| mock_server_url: MagicMock, | |
| ): | |
| # Initialise the RSyncer | |
| rsyncer = RSyncer( | |
| basepath_local=tmp_path / "local", | |
| basepath_remote=tmp_path / "remote", | |
| rsync_module=mock.ANY, | |
| server_url=mock_server_url, | |
| ) | |
| # Check that initial attributes are as expected | |
| assert not rsyncer._stopping | |
| assert not rsyncer._halt_thread | |
| # Run 'request_stop' and check that attributes have changed | |
| rsyncer.request_stop() | |
| assert rsyncer._stopping | |
| assert rsyncer._halt_thread | |
| def clem_visit_dir(tmp_path: Path): | |
| visit_name = f"{ExampleVisit.proposal_code}{ExampleVisit.proposal_number}-{ExampleVisit.visit_number}" | |
| visit_dir = tmp_path / "local" / visit_name | |
| visit_dir.mkdir(parents=True, exist_ok=True) | |
| return visit_dir | |
| def clem_test_files(clem_visit_dir: Path): | |
| # Create test files for the DirWatcher to scan | |
| file_list: list[Path] = [] | |
| project_dir = clem_visit_dir / "images" / "test_grid" | |
| # Example atlas collection | |
| for s in range(20): | |
| file_list.append( | |
| project_dir | |
| / "Overview 1" | |
| / "Image 1" | |
| / f"Image 1--Stage{str(s).zfill(2)}.tif" | |
| ) | |
| file_list.append( | |
| project_dir / "Overview 1" / "Image 1" / "Metadata" / "Image 1.xlif" | |
| ) | |
| # Example image stack collection | |
| for c in range(3): | |
| for z in range(10): | |
| file_list.append( | |
| project_dir | |
| / "TileScan 1" | |
| / "Position 1" | |
| / f"Position 1--C{str(c).zfill(2)}--Z{str(z).zfill(2)}.tif" | |
| ) | |
| file_list.append( | |
| project_dir / "TileScan 1" / "Position 1" / "Metadata" / "Position 1.xlif" | |
| ) | |
| # Create all files and directories specified | |
| for file in file_list: | |
| if not file.parent.exists(): | |
| file.parent.mkdir(parents=True) | |
| if not file.exists(): | |
| file.touch() | |
| return sorted(file_list) | |
| def clem_junk_files(clem_visit_dir: Path): | |
| # Create junk files that are to be blacklisted from the CLEM workflow | |
| file_list: list[Path] = [] | |
| project_dir = clem_visit_dir / "images" / "test_grid" | |
| # Create junk atlas data | |
| for n in range(5): | |
| for s in range(20): | |
| file_list.append( | |
| project_dir | |
| / "Image 1" | |
| / f"Image 1_pmd_{n}" | |
| / f"Image 1_pmd_{n}--Stage{str(s).zfill(2)}.tif" | |
| ) | |
| file_list.append( | |
| project_dir / "Image 1" / f"Image 1_pmd_{n}" / "Metadata" / "Image 1.xlif" | |
| ) | |
| # Creat junk image data | |
| for n in range(5): | |
| for c in range(3): | |
| for z in range(10): | |
| file_list.append( | |
| project_dir | |
| / "Position 1" | |
| / f"Position 1_pmd_{n}" | |
| / f"Position 1_pmd_{n}--C{str(c).zfill(2)}--Z{str(z).zfill(2)}.tif" | |
| ) | |
| file_list.append( | |
| project_dir | |
| / "Position 1" | |
| / f"Position 1_pmd_{n}" | |
| / "Metadata" | |
| / "Position 1.xlif" | |
| ) | |
| # Create remaining junk files | |
| for file_path in ( | |
| "1.xlef", | |
| "Metadata/IOManagerConfiguation.xlif", | |
| "Metadata/Overview 1.xlcf", | |
| "Metadata/TileScan 1.xlcf", | |
| "Overview 1/Image 1/Image 1_histo.lof", | |
| "TileScan 1/Position 1/Position 1_histo.lof", | |
| "Overview 1/Image 1/Metadata/Image 1_histo.xlif", | |
| "TileScan 1/Position 1/Metadata/Position 1_histo.xlif", | |
| ): | |
| file_list.append(project_dir / file_path) | |
| # Create files and directoriees | |
| for file in file_list: | |
| if not file.parent.exists(): | |
| file.parent.mkdir(parents=True) | |
| if not file.exists(): | |
| file.touch() | |
| return sorted(file_list) | |
| clem_substrings_blacklist = { | |
| "directories": [ | |
| "_pmd_", | |
| ], | |
| "files": [ | |
| ".xlef", | |
| ".xlcf", | |
| "_histo.lof", | |
| "_histo.xlif", | |
| "IOManagerConfiguation.xlif", | |
| ], | |
| } | |
| def test_rsyncer_finalise( | |
| mocker: MockerFixture, | |
| rsync_module: str, | |
| mock_server_url: MagicMock, | |
| clem_visit_dir: Path, | |
| clem_test_files: list[Path], | |
| clem_junk_files: list[Path], | |
| test_params: tuple[str, bool, bool, bool], | |
| ): | |
| # Unpack test params | |
| workflow_type, use_thread, use_callback, use_blacklist = test_params | |
| # Create a test end time | |
| timestamp = datetime.now() | |
| # Mock the class functions/attributes called by the 'finalise' class function | |
| mock_queue = MagicMock() | |
| mock_queue.put.return_value = None | |
| mock_transfer = mocker.patch.object(RSyncer, "_transfer") | |
| mock_transfer.return_value = True | |
| mock_stop = mocker.patch.object(RSyncer, "stop") | |
| mock_stop.return_value = None | |
| mock_process = mocker.patch.object(RSyncer, "_process") | |
| mock_process.return_value = None | |
| mock_callback = MagicMock(return_value=None) | |
| # Initialise the RSyncer class based on the workflow type being tested | |
| if workflow_type == "clem": | |
| rsyncer = RSyncer( | |
| basepath_local=clem_visit_dir / "images", | |
| basepath_remote=Path(clem_visit_dir.name) / "images", | |
| rsync_module=rsync_module, | |
| server_url=mock_server_url, | |
| stop_callback=dummy_callback, | |
| substrings_blacklist=clem_substrings_blacklist if use_blacklist else {}, | |
| end_time=timestamp, | |
| ) | |
| # Patch the 'queue' attribute with the mocked one | |
| rsyncer.queue = mock_queue | |
| # Check the initial state of attributes that will be changed by 'finalise' | |
| assert not rsyncer._remove_files | |
| assert rsyncer._notify | |
| assert rsyncer._end_time == timestamp | |
| assert not rsyncer._finalising | |
| assert not rsyncer._finalised | |
| # Run the 'finalise' class function with the workflow-specific paths | |
| rsyncer.finalise( | |
| thread=use_thread, | |
| callback=mock_callback if use_callback else None, | |
| ) | |
| # Check that attributes are set correctly at the start of the function | |
| assert rsyncer._remove_files | |
| assert not rsyncer._notify | |
| assert rsyncer._end_time is None | |
| assert rsyncer._finalising | |
| # Check that list of files with and without using a blacklist are correct | |
| if use_thread: | |
| for file in clem_test_files: | |
| mock_queue.put.assert_any_call(file) | |
| if not use_blacklist: | |
| for file in clem_junk_files: | |
| mock_queue.put.assert_any_call(file) | |
| else: | |
| transfer_args = mock_transfer.call_args.args | |
| assert sorted(transfer_args[0]) == ( | |
| sorted(clem_test_files) | |
| if use_blacklist | |
| else sorted([*clem_test_files, *clem_junk_files]) | |
| ) | |
| # Transfer is being mocked, so check that files to transfer are all present | |
| for file in clem_test_files: | |
| assert file.exists() | |
| for file in clem_junk_files: | |
| assert not file.exists() if use_blacklist else file.exists() | |
| # Check that stop was called the correct number of times depending on the setup | |
| assert mock_stop.call_count == 2 if use_thread else 1 | |
| # Check that the RSyncer is marked as finalised at the end | |
| assert rsyncer._finalised | |
| # Check that the callback was set at the end | |
| if use_callback: | |
| mock_callback.assert_called_once() | |
| def test_rsyncer_enqueue( | |
| tmp_path: Path, | |
| mock_server_url: MagicMock, | |
| is_stopping: bool, | |
| ): | |
| # Mock the queue | |
| mock_queue = MagicMock() | |
| mock_queue.put.return_value = None | |
| # Initialise the RSyncer and patch the attributes used by the test | |
| rsyncer = RSyncer( | |
| basepath_local=tmp_path / "local", | |
| basepath_remote=tmp_path / "remote", | |
| rsync_module=mock.ANY, | |
| server_url=mock_server_url, | |
| ) | |
| rsyncer._stopping = is_stopping | |
| rsyncer.queue = mock_queue | |
| # Run enqueue with a test file and check that the expected calls were made | |
| rsyncer.enqueue(Path("test_file")) | |
| if is_stopping: | |
| mock_queue.put.assert_not_called() | |
| else: | |
| mock_queue.put.assert_called_once_with( | |
| (tmp_path / "local" / "test_file").absolute() | |
| ) | |
| def test_rsyncer_flush_skipped( | |
| tmp_path: Path, | |
| mock_server_url: MagicMock, | |
| ): | |
| # Mock the queue | |
| mock_queue = MagicMock() | |
| mock_queue.put.return_value = None | |
| # Create a list of test files | |
| skipped_files = [ | |
| tmp_path / "local" / f"file_{str(n).zfill(2)}.txt" for n in range(20) | |
| ] | |
| # Initialise the RSyncer and patch the attributes used by the test | |
| rsyncer = RSyncer( | |
| basepath_local=tmp_path / "local", | |
| basepath_remote=tmp_path / "remote", | |
| rsync_module=mock.ANY, | |
| server_url=mock_server_url, | |
| ) | |
| rsyncer.queue = mock_queue | |
| rsyncer._skipped_files = skipped_files | |
| # Run 'flush_skipped' and check that it works as intended | |
| rsyncer.flush_skipped() | |
| for f in skipped_files: | |
| mock_queue.put.assert_any_call(f) | |
| assert rsyncer._skipped_files == [] | |
Xet Storage Details
- Size:
- 17.7 kB
- Xet hash:
- 1754eee0d0bcfc3cdbb21571b0820067a0dcc3767627dab5666f240ea7f551b4
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.