Download code/graph/real/expansion.py from frontal-labs/graph-reasoner-typed-compositional: direct link, hf CLI and curl.
- Browser
- Download file 15.9 kB
-
https://huggingface.co/frontal-labs/graph-reasoner-typed-compositional/resolve/main/code/graph/real/expansion.py
- Command line
-
hf download hf://frontal-labs/graph-reasoner-typed-compositional/code/graph/real/expansion.py
-
curl -L -o expansion.py https://huggingface.co/frontal-labs/graph-reasoner-typed-compositional/resolve/main/code/graph/real/expansion.py
15.9 kB
| """Loaders for the second bounded coverage-expansion tier.""" | |
| from __future__ import annotations | |
| import json | |
| import tarfile | |
| from pathlib import Path | |
| from xml.etree import ElementTree | |
| from graph.graph import EdgeRecord | |
| from graph.real.causal import _make_graph, _node_id | |
| from graph.real.corpus import RealGraph, SourceMetadata, relation_phrase | |
| def _text(element: ElementTree.Element | None) -> str: | |
| return " ".join("".join(element.itertext()).split()) if element is not None else "" | |
| def load_mesh_xml( | |
| path: str | Path, | |
| source: SourceMetadata, | |
| *, | |
| max_records: int | None = None, | |
| ) -> RealGraph: | |
| """Load MeSH descriptors, synonyms, and tree hierarchy.""" | |
| root = ElementTree.parse(path).getroot() | |
| records = list(root.findall(".//DescriptorRecord")) | |
| if max_records is not None: | |
| records = records[:max_records] | |
| nodes: dict[str, str] = {} | |
| labels: dict[str, str] = {} | |
| attributes: dict[str, dict[str, str]] = {} | |
| tree_to_node: dict[str, str] = {} | |
| edges: list[EdgeRecord] = [] | |
| for record in records: | |
| descriptor_id = _text(record.find("DescriptorUI")) | |
| name = _text(record.find("DescriptorName/String")) | |
| if not descriptor_id or not name: | |
| continue | |
| node_id = f"mesh:{descriptor_id}" | |
| nodes[node_id] = "mesh_descriptor" | |
| labels[node_id] = name | |
| synonyms = [ | |
| _text(term.find("String")) | |
| for term in record.findall("ConceptList/Concept/TermList/Term") | |
| ] | |
| attributes[node_id] = { | |
| "descriptor_id": descriptor_id, | |
| "scope_note": _text(record.find("ScopeNote"))[:4000], | |
| "synonyms": ", ".join(dict.fromkeys(item for item in synonyms if item))[ | |
| :4000 | |
| ], | |
| } | |
| for tree_number in record.findall("TreeNumberList/TreeNumber"): | |
| tree_to_node[_text(tree_number)] = node_id | |
| for record in records: | |
| descriptor_id = _text(record.find("DescriptorUI")) | |
| source_id = f"mesh:{descriptor_id}" | |
| for tree_number in record.findall("TreeNumberList/TreeNumber"): | |
| tree = _text(tree_number) | |
| parent_tree = tree.rsplit(".", 1)[0] if "." in tree else "" | |
| parent_id = tree_to_node.get(parent_tree) | |
| if parent_id and parent_id != source_id: | |
| edges.append( | |
| EdgeRecord( | |
| source_id, | |
| "is_a", | |
| parent_id, | |
| "mesh_descriptor", | |
| "mesh_descriptor", | |
| relation_phrase("is_a"), | |
| {"evidence_kind": "nlm_mesh_hierarchy"}, | |
| ) | |
| ) | |
| for related in record.findall("SeeRelatedDescriptorList/SeeRelatedDescriptor"): | |
| target_id = f"mesh:{_text(related)}" | |
| if target_id in nodes: | |
| edges.append( | |
| EdgeRecord( | |
| source_id, | |
| "associated_with", | |
| target_id, | |
| "mesh_descriptor", | |
| "mesh_descriptor", | |
| relation_phrase("associated_with"), | |
| {"evidence_kind": "nlm_mesh_related_descriptor"}, | |
| ) | |
| ) | |
| return _make_graph( | |
| source, | |
| nodes, | |
| edges, | |
| labels=labels, | |
| node_attributes=attributes, | |
| relation_ids={"is_a", "associated_with"}, | |
| ) | |
| def _dmp_fields(line: str) -> list[str]: | |
| # Every taxdump row ends in "\t|"; left on, it stops the last column from | |
| # matching "scientific name". | |
| return [field.strip() for field in line.rstrip().removesuffix("\t|").split("\t|\t")] | |
| def load_ncbi_taxdump( | |
| archive: str | Path, | |
| source: SourceMetadata, | |
| *, | |
| max_nodes: int = 150_000, | |
| ) -> RealGraph: | |
| """Load a bounded NCBI Taxonomy parent/name graph.""" | |
| node_rows: dict[str, tuple[str, str]] = {} | |
| name_rows: dict[str, str] = {} | |
| with tarfile.open(archive, "r:gz") as handle: | |
| nodes_member = next( | |
| member | |
| for member in handle.getmembers() | |
| if member.name.rsplit("/", 1)[-1] == "nodes.dmp" | |
| ) | |
| nodes_file = handle.extractfile(nodes_member) | |
| if nodes_file is None: | |
| raise ValueError("new_taxdump archive has no nodes.dmp") | |
| for raw in nodes_file: | |
| fields = _dmp_fields(raw.decode("utf-8", "replace")) | |
| if len(fields) < 3: | |
| continue | |
| tax_id, parent_id, rank = fields[:3] | |
| node_rows[tax_id] = (parent_id, rank) | |
| if len(node_rows) >= max_nodes: | |
| break | |
| selected = set(node_rows) | |
| names_member = next( | |
| member | |
| for member in handle.getmembers() | |
| if member.name.rsplit("/", 1)[-1] == "names.dmp" | |
| ) | |
| names_file = handle.extractfile(names_member) | |
| if names_file is None: | |
| raise ValueError("new_taxdump archive has no names.dmp") | |
| for raw in names_file: | |
| fields = _dmp_fields(raw.decode("utf-8", "replace")) | |
| if len(fields) < 4 or fields[0] not in selected: | |
| continue | |
| if fields[3] == "scientific name" or fields[0] not in name_rows: | |
| name_rows[fields[0]] = fields[1] | |
| nodes = {f"taxon:{tax_id}": "taxon" for tax_id in node_rows} | |
| labels = {f"taxon:{tax_id}": name_rows.get(tax_id, tax_id) for tax_id in node_rows} | |
| attributes = { | |
| f"taxon:{tax_id}": {"rank": rank, "tax_id": tax_id} | |
| for tax_id, (_, rank) in node_rows.items() | |
| } | |
| edges: list[EdgeRecord] = [] | |
| for tax_id, (parent_id, rank) in node_rows.items(): | |
| if parent_id not in node_rows or parent_id == tax_id: | |
| continue | |
| edges.append( | |
| EdgeRecord( | |
| f"taxon:{tax_id}", | |
| "is_a", | |
| f"taxon:{parent_id}", | |
| "taxon", | |
| "taxon", | |
| relation_phrase("is_a"), | |
| { | |
| "evidence_kind": "ncbi_taxonomy_hierarchy", | |
| "rank": rank, | |
| }, | |
| ) | |
| ) | |
| return _make_graph( | |
| source, | |
| nodes, | |
| edges, | |
| labels=labels, | |
| node_attributes=attributes, | |
| relation_ids={"is_a"}, | |
| ) | |
| def load_openalex_jsonl( | |
| path: str | Path, | |
| source: SourceMetadata, | |
| *, | |
| max_works: int | None = None, | |
| ) -> RealGraph: | |
| """Load bounded OpenAlex work metadata without abstracts/full text.""" | |
| nodes: dict[str, str] = {} | |
| labels: dict[str, str] = {} | |
| attributes: dict[str, dict[str, str]] = {} | |
| edges: list[EdgeRecord] = [] | |
| relations = {"written_by", "has_topic", "published_in", "affiliated_with", "cites"} | |
| with Path(path).open(encoding="utf-8") as handle: | |
| for index, line in enumerate(handle): | |
| if max_works is not None and index >= max_works: | |
| break | |
| work = json.loads(line) | |
| work_key = str(work.get("id", f"work:{index}")).rsplit("/", 1)[-1] | |
| work_id = f"openalex:work:{work_key}" | |
| title = str(work.get("title") or work_id) | |
| nodes[work_id] = "work" | |
| labels[work_id] = title[:240] | |
| attributes[work_id] = { | |
| "publication_year": str(work.get("publication_year", "")), | |
| "work_type": str(work.get("type", "")), | |
| "cited_by_count": str(work.get("cited_by_count", "")), | |
| } | |
| for authorship in work.get("authorships", ()): | |
| author = authorship.get("author") or {} | |
| author_key = str(author.get("id", "")).rsplit("/", 1)[-1] | |
| if not author_key: | |
| continue | |
| author_id = f"openalex:author:{author_key}" | |
| nodes[author_id] = "author" | |
| labels[author_id] = str(author.get("display_name", author_key))[:200] | |
| edges.append( | |
| EdgeRecord( | |
| work_id, | |
| "written_by", | |
| author_id, | |
| "work", | |
| "author", | |
| relation_phrase("written_by"), | |
| {"evidence_kind": "openalex_metadata"}, | |
| ) | |
| ) | |
| for institution in authorship.get("institutions", ()): | |
| institution_key = str(institution.get("id", "")).rsplit("/", 1)[-1] | |
| if not institution_key: | |
| continue | |
| institution_id = f"openalex:institution:{institution_key}" | |
| nodes[institution_id] = "institution" | |
| labels[institution_id] = str( | |
| institution.get("display_name", institution_key) | |
| )[:200] | |
| edges.append( | |
| EdgeRecord( | |
| author_id, | |
| "affiliated_with", | |
| institution_id, | |
| "author", | |
| "institution", | |
| relation_phrase("affiliated_with"), | |
| {"evidence_kind": "openalex_metadata"}, | |
| ) | |
| ) | |
| for topic in work.get("topics", ()): | |
| topic_key = str(topic.get("id", "")).rsplit("/", 1)[-1] | |
| if not topic_key: | |
| continue | |
| topic_id = f"openalex:topic:{topic_key}" | |
| nodes[topic_id] = "topic" | |
| labels[topic_id] = str(topic.get("display_name", topic_key))[:200] | |
| edges.append( | |
| EdgeRecord( | |
| work_id, | |
| "has_topic", | |
| topic_id, | |
| "work", | |
| "topic", | |
| relation_phrase("has_topic"), | |
| {"evidence_kind": "openalex_metadata"}, | |
| ) | |
| ) | |
| for referenced in work.get("referenced_works", ())[:64]: | |
| reference_key = str(referenced).rsplit("/", 1)[-1] | |
| target_id = f"openalex:work:{reference_key}" | |
| nodes.setdefault(target_id, "work") | |
| labels.setdefault(target_id, reference_key) | |
| edges.append( | |
| EdgeRecord( | |
| work_id, | |
| "cites", | |
| target_id, | |
| "work", | |
| "work", | |
| relation_phrase("cites"), | |
| {"evidence_kind": "openalex_metadata"}, | |
| ) | |
| ) | |
| return _make_graph( | |
| source, | |
| nodes, | |
| edges, | |
| labels=labels, | |
| node_attributes=attributes, | |
| relation_ids=relations, | |
| ) | |
| def load_clinicaltrials_jsonl( | |
| path: str | Path, | |
| source: SourceMetadata, | |
| *, | |
| max_studies: int | None = None, | |
| ) -> RealGraph: | |
| """Load normalized ClinicalTrials.gov study metadata as a graph.""" | |
| nodes: dict[str, str] = {} | |
| labels: dict[str, str] = {} | |
| attributes: dict[str, dict[str, str]] = {} | |
| edges: list[EdgeRecord] = [] | |
| relations = { | |
| "studies_condition", | |
| "tests_intervention", | |
| "sponsored_by", | |
| "has_outcome", | |
| } | |
| with Path(path).open(encoding="utf-8") as handle: | |
| for index, line in enumerate(handle): | |
| if max_studies is not None and index >= max_studies: | |
| break | |
| study = json.loads(line) | |
| protocol = study.get("protocolSection", study) | |
| identification = protocol.get("identificationModule", {}) | |
| study_id = str(identification.get("nctId", f"study-{index}")) | |
| node_id = f"trial:{study_id}" | |
| nodes[node_id] = "clinical_trial" | |
| labels[node_id] = str(identification.get("briefTitle", study_id))[:240] | |
| design = protocol.get("designModule", {}) | |
| attributes[node_id] = { | |
| "phase": ",".join(str(value) for value in design.get("phases", ())), | |
| "study_type": str(design.get("studyType", "")), | |
| "enrollment": str( | |
| (design.get("enrollmentInfo") or {}).get("count", "") | |
| ), | |
| "evidence_kind": "registered_trial_metadata", | |
| } | |
| conditions = protocol.get("conditionsModule", {}).get("conditions", ()) | |
| interventions = protocol.get("armsInterventionsModule", {}).get( | |
| "interventions", () | |
| ) | |
| sponsors = protocol.get("sponsorCollaboratorsModule", {}) | |
| for value in conditions: | |
| target_id = _node_id("trial_condition", str(value)) | |
| nodes[target_id] = "condition" | |
| labels[target_id] = str(value)[:200] | |
| edges.append( | |
| EdgeRecord( | |
| node_id, | |
| "studies_condition", | |
| target_id, | |
| "clinical_trial", | |
| "condition", | |
| relation_phrase("studies_condition"), | |
| {"evidence_kind": "registered_trial_metadata"}, | |
| ) | |
| ) | |
| for intervention in interventions: | |
| name = str(intervention.get("name", "")).strip() | |
| if not name: | |
| continue | |
| target_id = _node_id("trial_intervention", name) | |
| nodes[target_id] = "intervention" | |
| labels[target_id] = name[:200] | |
| edges.append( | |
| EdgeRecord( | |
| node_id, | |
| "tests_intervention", | |
| target_id, | |
| "clinical_trial", | |
| "intervention", | |
| relation_phrase("tests_intervention"), | |
| {"evidence_kind": "registered_trial_metadata"}, | |
| ) | |
| ) | |
| sponsor = sponsors.get("leadSponsor") or {} | |
| sponsor_name = str(sponsor.get("name", "")).strip() | |
| if sponsor_name: | |
| target_id = _node_id("trial_sponsor", sponsor_name) | |
| nodes[target_id] = "sponsor" | |
| labels[target_id] = sponsor_name[:200] | |
| edges.append( | |
| EdgeRecord( | |
| node_id, | |
| "sponsored_by", | |
| target_id, | |
| "clinical_trial", | |
| "sponsor", | |
| relation_phrase("sponsored_by"), | |
| {"evidence_kind": "registered_trial_metadata"}, | |
| ) | |
| ) | |
| outcomes = protocol.get("outcomesModule", {}).get("outcomes", ()) | |
| for outcome in outcomes[:16]: | |
| name = str(outcome.get("measure", "")).strip() | |
| if not name: | |
| continue | |
| target_id = _node_id("trial_outcome", name) | |
| nodes[target_id] = "outcome_measure" | |
| labels[target_id] = name[:240] | |
| edges.append( | |
| EdgeRecord( | |
| node_id, | |
| "has_outcome", | |
| target_id, | |
| "clinical_trial", | |
| "outcome_measure", | |
| relation_phrase("has_outcome"), | |
| {"evidence_kind": "registered_trial_metadata"}, | |
| ) | |
| ) | |
| return _make_graph( | |
| source, | |
| nodes, | |
| edges, | |
| labels=labels, | |
| node_attributes=attributes, | |
| relation_ids=relations, | |
| ) | |