Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /als-apg_osprey /tests /_graphdb_container.py
| """One recipe for a throwaway Neo4j store carrying n10s + APOC. | |
| Three lanes need the same container — the store integration lane | |
| (``tests/integration/test_graphdb_store.py``), the corpus enrichment lane | |
| (``tests/services/facility_knowledge/test_graph_enrichment_live.py``) and the | |
| runnable-examples lane (``tests/mcp_server/channel_finder_graph/test_examples_live.py``). | |
| They differ in what they do to the store, not in how it is built, so the build | |
| lives here once. | |
| **Why the plugins are mounted rather than downloaded by the server.** The | |
| shipped compose template sets ``NEO4J_PLUGINS`` and the Neo4j entrypoint honours | |
| it by fetching the n10s jar at boot, which needs egress from inside the | |
| container. :func:`resolve_plugin_dir` instead resolves both jars on the host and | |
| :func:`graphdb_store` bind-mounts them at ``/plugins``, setting the procedure | |
| allowlist/unrestricted variables directly — they are the only other thing | |
| ``NEO4J_PLUGINS`` would have done. n10s comes from the pinned release (or from a | |
| local jar named by ``OSPREY_TEST_N10S_JAR``); APOC is *bundled in the image* at | |
| ``/var/lib/neo4j/labs`` and is copied out of it. The result loads the same two | |
| plugins a deployed graphdb does. | |
| **Skip order.** :func:`resolve_plugin_dir` probes the testcontainers neo4j | |
| module first, then the daemon, then the image, and only then downloads the jar — | |
| a host without the extra installed says so instead of fetching a release it will | |
| never mount. | |
| **Scope.** The plugin directory is session-scoped (fixture | |
| ``graphdb_plugin_dir`` in ``tests/conftest.py``) because it is the expensive | |
| half and its content does not depend on the lane. The *store* is deliberately | |
| not shared: two of the three lanes wipe it between corpora, so they start their | |
| own module-scoped container from this same recipe. | |
| """ | |
| from __future__ import annotations | |
| import io | |
| import logging | |
| import os | |
| import tarfile | |
| from collections.abc import Iterator | |
| from contextlib import contextmanager | |
| from pathlib import Path | |
| import pytest | |
| import requests | |
| from tests._container_support import ( | |
| is_docker_available, | |
| is_image_present, | |
| start_or_skip, | |
| stop_quietly, | |
| ) | |
| logger = logging.getLogger(__name__) | |
| # --------------------------------------------------------------------------- | |
| # Pins | |
| # --------------------------------------------------------------------------- | |
| #: Same server pin as the shipped compose template. Load-bearing twice over: | |
| #: the 5.26 LTS line is the newest one the neosemantics plugin manifest | |
| #: covers, and the n10s release below is the build made for exactly this | |
| #: server line. | |
| NEO4J_IMAGE = "neo4j:5.26-community" | |
| #: neosemantics release matching :data:`NEO4J_IMAGE`. n10s versions track the | |
| #: server they are built against, so this moves with the image pin or not at | |
| #: all — a mismatched jar loads and then fails every procedure call. | |
| N10S_VERSION = "5.26.0" | |
| N10S_JAR_URL = ( | |
| f"https://github.com/neo4j-labs/neosemantics/releases/download/" | |
| f"{N10S_VERSION}/neosemantics-{N10S_VERSION}.jar" | |
| ) | |
| #: Escape hatch for hosts with no egress to github.com: point it at an already | |
| #: downloaded neosemantics jar and the fixture mounts that instead of fetching. | |
| N10S_JAR_ENV = "OSPREY_TEST_N10S_JAR" | |
| #: Where the image keeps the APOC jar it would otherwise copy into /plugins. | |
| IMAGE_LABS_DIR = "/var/lib/neo4j/labs" | |
| #: Password for the throwaway store. Satisfies the same rule the deploy-time | |
| #: validator enforces on ``GRAPHDB_PASSWORD`` (>= 8 chars, no ``/``), so the | |
| #: composite ``NEO4J_AUTH`` this becomes is shaped like a real one. | |
| GRAPHDB_TEST_PASSWORD = "ospreytest1234" | |
| GRAPHDB_TEST_USERNAME = "neo4j" | |
| # --------------------------------------------------------------------------- | |
| # Plugins | |
| # --------------------------------------------------------------------------- | |
| def _fetch_n10s_jar(dest_dir: Path) -> None: | |
| """Put the pinned neosemantics jar in *dest_dir*, or skip. | |
| A jar named by :data:`N10S_JAR_ENV` is used as-is; otherwise the pinned | |
| release is downloaded. A download failure skips rather than fails: it says | |
| something about the host's network, not about the graph store. | |
| """ | |
| target = dest_dir / f"neosemantics-{N10S_VERSION}.jar" | |
| override = os.environ.get(N10S_JAR_ENV) | |
| if override: | |
| source = Path(override).expanduser() | |
| if not source.is_file(): | |
| pytest.skip(f"{N10S_JAR_ENV} points at {source}, which is not a file") | |
| target.write_bytes(source.read_bytes()) | |
| logger.info(f"n10s jar taken from {N10S_JAR_ENV}={source}") | |
| return | |
| try: | |
| with requests.get(N10S_JAR_URL, timeout=120, stream=True) as response: | |
| response.raise_for_status() | |
| with target.open("wb") as handle: | |
| for chunk in response.iter_content(chunk_size=1 << 20): | |
| handle.write(chunk) | |
| except requests.exceptions.RequestException as exc: | |
| pytest.skip( | |
| f"could not download the neosemantics jar from {N10S_JAR_URL} ({exc}); " | |
| f"set {N10S_JAR_ENV} to a local copy to run this test offline" | |
| ) | |
| logger.info(f"n10s jar downloaded to {target} ({target.stat().st_size} bytes)") | |
| def _copy_bundled_apoc(dest_dir: Path) -> None: | |
| """Copy the image's own APOC jar into *dest_dir*. | |
| APOC ships inside ``neo4j:*-community`` and the entrypoint only *moves* it | |
| into ``/plugins`` when ``NEO4J_PLUGINS`` asks for it. Since this recipe | |
| does not set that variable, it does the move itself — from a container that | |
| is created and never started, so nothing but a filesystem read happens. | |
| """ | |
| import docker | |
| client = docker.from_env() | |
| container = client.containers.create(NEO4J_IMAGE) | |
| try: | |
| stream, _stat = container.get_archive(IMAGE_LABS_DIR) | |
| archive = io.BytesIO(b"".join(stream)) | |
| copied = [] | |
| with tarfile.open(fileobj=archive) as tar: | |
| for member in tar.getmembers(): | |
| name = Path(member.name).name | |
| if not (member.isfile() and name.startswith("apoc") and name.endswith(".jar")): | |
| continue | |
| extracted = tar.extractfile(member) | |
| if extracted is None: # pragma: no cover - defensive | |
| continue | |
| (dest_dir / name).write_bytes(extracted.read()) | |
| copied.append(name) | |
| finally: | |
| container.remove(force=True) | |
| if not copied: # pragma: no cover - would mean the image changed shape | |
| pytest.skip(f"no APOC jar found under {IMAGE_LABS_DIR} in {NEO4J_IMAGE}") | |
| logger.info(f"APOC taken from the image: {', '.join(copied)}") | |
| def resolve_plugin_dir(tmp_path_factory: pytest.TempPathFactory) -> Path: | |
| """Return a directory holding n10s + APOC, or skip with the reason. | |
| This is every graph lane's skip gate, because the plugins are the first | |
| thing a store depends on: a missing testcontainers extra, an unreachable | |
| daemon, a missing image or an unreachable release is reported once, with its | |
| reason, rather than as a bare skip on each test. | |
| The probes run cheapest-and-most-decisive first — the testcontainers import | |
| before the daemon, both before the jar download — so a host that could never | |
| start the container does not spend a release download finding that out. | |
| """ | |
| try: | |
| import testcontainers.community.neo4j # noqa: F401 | |
| except ImportError: # pragma: no cover - depends on the installed extras | |
| pytest.skip("testcontainers' neo4j module is not installed") | |
| if not is_docker_available(): | |
| pytest.skip( | |
| "docker daemon is not reachable — this can only be proved against a " | |
| "real Neo4j + neosemantics store" | |
| ) | |
| if not is_image_present(NEO4J_IMAGE): | |
| # Pull explicitly: the APOC copy below reads the image directly and, | |
| # unlike ``containers.run``, ``containers.create`` does not pull for us. | |
| # Testcontainers would have pulled the same image moments later anyway. | |
| import docker | |
| logger.info(f"pulling {NEO4J_IMAGE} (not present locally)") | |
| try: | |
| docker.from_env().images.pull(NEO4J_IMAGE) | |
| except Exception as exc: | |
| pytest.skip(f"could not pull {NEO4J_IMAGE}: {exc}") | |
| plugin_dir = tmp_path_factory.mktemp("graphdb-plugins") | |
| _fetch_n10s_jar(plugin_dir) | |
| _copy_bundled_apoc(plugin_dir) | |
| # The server runs as a non-root user and only ever reads these. | |
| for jar in plugin_dir.glob("*.jar"): | |
| jar.chmod(0o644) | |
| plugin_dir.chmod(0o755) | |
| return plugin_dir | |
| # --------------------------------------------------------------------------- | |
| # The store | |
| # --------------------------------------------------------------------------- | |
| def graphdb_store(plugin_dir: Path) -> Iterator[str]: | |
| """Start a throwaway graph store on *plugin_dir* and yield its bolt URI. | |
| The environment mirrors the shipped compose template minus | |
| ``NEO4J_PLUGINS`` — see the module docstring for why that one is left out | |
| and what replaces it. Testcontainers publishes bolt on an ephemeral port | |
| under a generated name, so this cannot collide with a ``graphdb`` service | |
| already deployed on the host. | |
| """ | |
| from testcontainers.community.neo4j import Neo4jContainer | |
| def _build() -> Neo4jContainer: | |
| container = Neo4jContainer(image=NEO4J_IMAGE, password=GRAPHDB_TEST_PASSWORD) | |
| container.with_volume_mapping(str(plugin_dir), "/plugins", "rw") | |
| # The allowlist is the half of NEO4J_PLUGINS that is not a download: | |
| # without it every n10s.* call fails with "not on the allowlist", and | |
| # n10s needs the unrestricted grant because it calls into APOC. | |
| container.with_env("NEO4J_dbms_security_procedures_unrestricted", "apoc.*,n10s.*") | |
| container.with_env("NEO4J_dbms_security_procedures_allowlist", "apoc.*,n10s.*") | |
| return container | |
| container = start_or_skip(_build, label="graphdb (neo4j + n10s)") | |
| try: | |
| yield container.get_connection_url() | |
| finally: | |
| stop_quietly(container) | |
Xet Storage Details
- Size:
- 10.1 kB
- Xet hash:
- ebebb38f492b1121d4556701d0b54a118013656fdd20266c931903a10bbdc6f1
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.