Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /bluesky_tiled /tests /test_xdi_serialization.py
| """ | |
| Tests for the built-in XDI serializer (tiled/serialization/xdi.py). | |
| The serializer is registered under StructureFamily.table and is intended for use | |
| with tables tagged with Spec("xdi", ...). The primary use case is: | |
| c.write_table(df, metadata={...}, specs=[Spec("xdi")]) | |
| table.export("output.xdi") | |
| """ | |
| import collections | |
| import io | |
| import re | |
| import pandas | |
| import pytest | |
| from tiled.adapters.dataframe import DataFrameAdapter | |
| from tiled.adapters.mapping import MapAdapter | |
| from tiled.client import Context, from_context | |
| from tiled.client.utils import ClientError | |
| from tiled.server.app import build_app | |
| from tiled.structures.core import Spec | |
| # --------------------------------------------------------------------------- | |
| # Shared test data | |
| # --------------------------------------------------------------------------- | |
| XDI_METADATA = { | |
| "xdi_version": "1.0", | |
| "extra_version": "GSE/1.0", | |
| "fields": { | |
| "Element": {"symbol": "Cu", "edge": "K"}, | |
| "Mono": {"d_spacing": "3.13553", "name": "Si 111"}, | |
| "Beamline": {"name": "13ID", "collimation": "none", "focusing": "yes"}, | |
| "Facility": { | |
| "name": "APS", | |
| "energy": "7.00 GeV", | |
| "xray_source": "APS Undulator A", | |
| }, | |
| "Scan": {"start_time": "2001-06-26T22:27:31", "edge_energy": "8980.0"}, | |
| "Detector": {"I0": "10cm N2", "I1": "10cm N2"}, | |
| "Sample": {"name": "Cu", "prep": "Cu metal foil"}, | |
| "Column": {"1": "energy eV", "2": "i0", "3": "itrans", "4": "mutrans"}, | |
| # extension namespace | |
| "GSE": {"EXTRA": "config 1"}, | |
| }, | |
| "comments": "# Cu foil Room Temperature\n# measured at beamline 13-ID\n", | |
| } | |
| XDI_DF = pandas.DataFrame( | |
| { | |
| "energy": [8779.0, 8789.0, 8799.0], | |
| "i0": [149013.7, 144864.7, 132978.7], | |
| "itrans": [550643.089065, 531876.119084, 489591.10592], | |
| "mutrans": [-1.3070486, -1.3006104, -1.3033816], | |
| } | |
| ) | |
| # Metadata with raw (non-prefixed) comment lines — tests the '#' auto-prepend path | |
| XDI_METADATA_RAW_COMMENTS = { | |
| "xdi_version": "1.0", | |
| "extra_version": "", | |
| "fields": { | |
| "Element": {"symbol": "Cu", "edge": "K"}, | |
| "Mono": {"d_spacing": "3.13553"}, | |
| }, | |
| # Mix: one line without '#' (should get one prepended), one with '#' (preserved as-is) | |
| "comments": "Cu foil at room temperature\n# measured at 13-ID\n", | |
| } | |
| # Metadata without a "comments" key — tests the no-comment path | |
| XDI_METADATA_NO_COMMENTS = { | |
| "xdi_version": "1.0", | |
| "extra_version": "", | |
| "fields": { | |
| "Element": {"symbol": "Ti", "edge": "K"}, | |
| "Mono": {"d_spacing": "3.1353241"}, | |
| "Column": {"1": "energy eV"}, | |
| }, | |
| } | |
| def _build_tree(): | |
| return MapAdapter( | |
| { | |
| "xdi_table": DataFrameAdapter.from_pandas( | |
| XDI_DF.copy(), | |
| npartitions=1, | |
| metadata=XDI_METADATA, | |
| specs=[Spec("xdi", version="1.0")], | |
| ), | |
| "xdi_no_comments": DataFrameAdapter.from_pandas( | |
| XDI_DF[["energy"]].copy(), | |
| npartitions=1, | |
| metadata=XDI_METADATA_NO_COMMENTS, | |
| specs=[Spec("xdi", version="1.0")], | |
| ), | |
| "xdi_raw_comments": DataFrameAdapter.from_pandas( | |
| XDI_DF[["energy"]].copy(), | |
| npartitions=1, | |
| metadata=XDI_METADATA_RAW_COMMENTS, | |
| specs=[Spec("xdi", version="1.0")], | |
| ), | |
| # A plain table with no Spec("xdi") and no XDI metadata — | |
| # export should still succeed since we register on StructureFamily.table. | |
| "plain_table": DataFrameAdapter.from_pandas( | |
| XDI_DF.copy(), | |
| npartitions=1, | |
| ), | |
| } | |
| ) | |
| def client(): | |
| app = build_app(_build_tree()) | |
| with Context.from_app(app) as context: | |
| yield from_context(context) | |
| # --------------------------------------------------------------------------- | |
| # Helpers | |
| # --------------------------------------------------------------------------- | |
| def _export_to_string(client_node, **kwargs): | |
| """Export a node to an in-memory buffer and return the decoded string.""" | |
| buf = io.BytesIO() | |
| client_node.export(buf, **kwargs) | |
| buf.seek(0) | |
| return buf.read().decode() | |
| # --------------------------------------------------------------------------- | |
| # Tests: file export | |
| # --------------------------------------------------------------------------- | |
| def test_export_xdi_to_file(client, tmp_path): | |
| """Export to a file with a .xdi extension using the file-extension alias.""" | |
| out = tmp_path / "output.xdi" | |
| client["xdi_table"].export(out) | |
| assert out.exists() | |
| assert out.read_text().startswith("# XDI/") | |
| # --------------------------------------------------------------------------- | |
| # Tests: buffer export and output structure | |
| # --------------------------------------------------------------------------- | |
| def test_export_xdi_to_buffer_explicit_mimetype(client): | |
| """Export with explicit MIME type produces non-empty bytes output.""" | |
| content = _export_to_string(client["xdi_table"], format="application/x-xdi") | |
| assert len(content) > 0 | |
| def test_xdi_output_version_line(client): | |
| """First line must be the XDI version line.""" | |
| content = _export_to_string(client["xdi_table"], format="application/x-xdi") | |
| assert content.splitlines()[0] == "# XDI/1.0 GSE/1.0" | |
| def test_xdi_output_version_line_no_extra_version(client): | |
| """Version line with empty extra_version should have no trailing space.""" | |
| content = _export_to_string(client["xdi_no_comments"], format="application/x-xdi") | |
| first_line = content.splitlines()[0] | |
| assert first_line == "# XDI/1.0" | |
| assert not first_line.endswith(" ") | |
| def test_xdi_output_required_element_fields(client): | |
| """Element.symbol and Element.edge must be present in the header.""" | |
| content = _export_to_string(client["xdi_table"], format="application/x-xdi") | |
| assert "# Element.symbol: Cu" in content | |
| assert "# Element.edge: K" in content | |
| def test_xdi_output_required_mono_field(client): | |
| """Mono.d_spacing must be present in the header.""" | |
| content = _export_to_string(client["xdi_table"], format="application/x-xdi") | |
| assert "# Mono.d_spacing: 3.13553" in content | |
| def test_xdi_output_extension_namespace(client): | |
| """Extension namespace fields (e.g. GSE.EXTRA) must appear in the header.""" | |
| content = _export_to_string(client["xdi_table"], format="application/x-xdi") | |
| assert "# GSE.EXTRA: config 1" in content | |
| def test_xdi_output_defined_namespaces_before_extensions(client): | |
| """Defined namespaces (Element, Mono, ...) must precede extension namespaces.""" | |
| content = _export_to_string(client["xdi_table"], format="application/x-xdi") | |
| assert content.find("# Element.") < content.find( | |
| "# GSE." | |
| ), "Defined namespaces should appear before extension namespaces" | |
| def test_xdi_output_field_end_separator(client): | |
| """The field-end separator '# /////' must be present.""" | |
| content = _export_to_string(client["xdi_table"], format="application/x-xdi") | |
| assert re.search(r"^# /{3,}", content, re.MULTILINE) | |
| def test_xdi_output_header_end_separator(client): | |
| """The header-end separator '# -----' must be present.""" | |
| content = _export_to_string(client["xdi_table"], format="application/x-xdi") | |
| assert re.search(r"^# -{3,}", content, re.MULTILINE) | |
| def test_xdi_output_generated_by_tiled_comment(client): | |
| """The '# generated by tiled' provenance comment must be present.""" | |
| content = _export_to_string(client["xdi_table"], format="application/x-xdi") | |
| assert "# generated by tiled" in content | |
| def test_xdi_output_user_comments_preserved(client): | |
| """User comments from metadata must be written between the separators.""" | |
| content = _export_to_string(client["xdi_table"], format="application/x-xdi") | |
| assert "# Cu foil Room Temperature" in content | |
| assert "# measured at beamline 13-ID" in content | |
| def test_xdi_output_raw_comment_lines_get_hash_prefix(client): | |
| """Comment lines in metadata that don't start with '#' must have one prepended. | |
| Covers the two branches inside the comments loop in serialize_xdi: | |
| - a line without a leading '#' -> '# ' is prepended | |
| - a line already starting with '#' -> written as-is | |
| """ | |
| content = _export_to_string(client["xdi_raw_comments"], format="application/x-xdi") | |
| assert "# Cu foil at room temperature" in content # '#' was prepended | |
| assert "# measured at 13-ID" in content # existing '#' preserved | |
| def test_xdi_output_separators_always_present_without_comments(client): | |
| """Field-end and header-end separators must appear even with no user comments.""" | |
| content = _export_to_string(client["xdi_no_comments"], format="application/x-xdi") | |
| assert re.search(r"^# /{3,}", content, re.MULTILINE) | |
| assert re.search(r"^# -{3,}", content, re.MULTILINE) | |
| def test_xdi_output_column_labels_line(client): | |
| """The column label line must list all DataFrame column names, prefixed with '#'.""" | |
| content = _export_to_string(client["xdi_table"], format="application/x-xdi") | |
| assert "# energy i0 itrans mutrans" in content | |
| def test_xdi_output_column_labels_after_header_end(client): | |
| """Column labels must appear after the header-end separator.""" | |
| content = _export_to_string(client["xdi_table"], format="application/x-xdi") | |
| assert content.find("# energy i0 itrans mutrans") > content.rfind( | |
| "# ----" | |
| ), "Column labels should appear after the header-end separator" | |
| def test_xdi_output_data_rows_present(client): | |
| """Data rows must be present in the output.""" | |
| content = _export_to_string(client["xdi_table"], format="application/x-xdi") | |
| data_lines = [ln for ln in content.splitlines() if ln and not ln.startswith("#")] | |
| assert len(data_lines) == len( | |
| XDI_DF | |
| ), f"Expected {len(XDI_DF)} data rows, got {len(data_lines)}" | |
| def test_xdi_output_space_delimited_data(client): | |
| """Data rows must be whitespace-delimited (not comma-separated), per the XDI spec.""" | |
| content = _export_to_string(client["xdi_table"], format="application/x-xdi") | |
| data_lines = [ln for ln in content.splitlines() if ln and not ln.startswith("#")] | |
| for line in data_lines: | |
| assert "," not in line, f"Data line should not contain commas: {line!r}" | |
| assert len(line.split()) == 4 | |
| def test_xdi_output_numeric_data_parseable(client): | |
| """Data values must be parseable as floats.""" | |
| content = _export_to_string(client["xdi_table"], format="application/x-xdi") | |
| data_lines = [ln for ln in content.splitlines() if ln and not ln.startswith("#")] | |
| for line in data_lines: | |
| for v in line.split(): | |
| float(v) # should not raise | |
| def test_xdi_output_returns_bytes(client): | |
| """The serializer must return bytes (not str).""" | |
| buf = io.BytesIO() | |
| client["xdi_table"].export(buf, format="application/x-xdi") | |
| buf.seek(0) | |
| assert isinstance(buf.read(), bytes) | |
| # --------------------------------------------------------------------------- | |
| # Tests: roundtrip | |
| # --------------------------------------------------------------------------- | |
| def test_xdi_roundtrip(client, tmp_path): | |
| """Output must be re-parseable by the XDI header parser (roundtrip integrity).""" | |
| out = tmp_path / "roundtrip.xdi" | |
| client["xdi_table"].export(out) | |
| content = out.read_text() | |
| lines = iter(content.splitlines(keepends=True)) | |
| version_line = next(lines) | |
| m = re.match(r"#\s*XDI/(\S+)(?:\s+(\S+))?", version_line) | |
| assert m is not None, f"Version line not parseable: {version_line!r}" | |
| assert m.group(1) == "1.0" | |
| field_end_re = re.compile(r"#\s*/{3,}") | |
| header_end_re = re.compile(r"#\s*-{3,}") | |
| fields = collections.defaultdict(dict) | |
| for line in lines: | |
| if line[0] != "#": | |
| pytest.fail(f"Non-comment line in header: {line!r}") | |
| if re.match(field_end_re, line): | |
| break | |
| elif re.match(header_end_re, line): | |
| break | |
| stripped = line[1:].strip() | |
| if not stripped or ":" not in stripped: | |
| continue | |
| key, val = stripped.split(":", 1) | |
| if "." in key: | |
| namespace, tag = key.split(".", 1) | |
| fields[namespace.strip()][tag.strip()] = val.strip() | |
| assert "Element" in fields | |
| assert "symbol" in fields["Element"] | |
| assert "edge" in fields["Element"] | |
| assert "Mono" in fields | |
| assert "d_spacing" in fields["Mono"] | |
| # --------------------------------------------------------------------------- | |
| # Tests: format alias | |
| # --------------------------------------------------------------------------- | |
| def test_xdi_format_alias(client): | |
| """Requesting format='xdi' should work the same as 'application/x-xdi'.""" | |
| assert _export_to_string(client["xdi_table"], format="xdi") == _export_to_string( | |
| client["xdi_table"], format="application/x-xdi" | |
| ) | |
| # --------------------------------------------------------------------------- | |
| # Tests: plain table (no Spec("xdi")) is rejected with a 406 | |
| # --------------------------------------------------------------------------- | |
| def test_plain_table_cannot_export_as_xdi(client): | |
| """A table without Spec('xdi') must not be exportable as application/x-xdi. | |
| The server returns 406 Not Acceptable because 'application/x-xdi' is only | |
| registered for the 'xdi' spec, not for StructureFamily.table generally. | |
| """ | |
| with pytest.raises(ClientError): | |
| _export_to_string(client["plain_table"], format="application/x-xdi") | |
Xet Storage Details
- Size:
- 13.7 kB
- Xet hash:
- 1ea42c27f99788028702271b75f0cd72a4b94118b993f4c2e072bd45bde081ea
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.