chore: import scripts to process nml-docs

scripts/data/Readme.md

@@ -0,0 +1 @@
+Scripts to generate vector stores.

scripts/data/generate-single-nml-md.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Copyright 2025 Ankur Sinha
+# Author: Ankur Sinha <sanjay DOT ankur AT gmail DOT com>
+# File : data/scripts/generate-single-nml-md.sh
+#
+
+
+git clone --depth 1 https://github.com/NeuroML/Documentation nml-docs
+
+echo "Generating bibs"
+find . -name "*.bib" -type f -execdir bash -c '
+    for file; do
+        pandoc "$file" --citeproc -t markdown_strict -o "${file%.bib}.md"
+    done
+' bash {} +
+
+python3 jupyterbook2singlemd.py "nml-docs/source"
+mv -v single-page-markdown.md nml-docs-single-page.md
+mv -v single-page-markdown-schema.md nml-docs-single-page-schema.md
+mv -v url-map.json nml-docs-single-page-url-map.json
+
+echo "Please delete the nml-docs folder and copy the generated file over to the data/files folder"

scripts/data/jupyterbook2singlemd.py

@@ -0,0 +1,241 @@
+#!/usr/bin/env python3
+"""
+Generate a single page markdown from Jupyter book sources
+
+File: data/scripts/jupyterbook2singlemd.py
+
+Copyright 2026 Ankur Sinha
+Author: Ankur Sinha <sanjay DOT ankur AT gmail DOT com>
+"""
+
+import json
+import logging
+import re
+import sys
+from itertools import takewhile
+from pathlib import Path
+
+import colorlog
+
+logging.basicConfig(level=logging.NOTSET)
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+formatter = colorlog.ColoredFormatter(
+    "%(log_color)s%(name)s (%(levelname)s): %(message)s"
+)
+handler = colorlog.StreamHandler()
+handler.setLevel(logging.INFO)
+handler.setFormatter(formatter)
+
+logger.addHandler(handler)
+
+
+def runner(source: str):
+    """Main runner
+
+    :param source: TODO
+    :returns: TODO
+
+    """
+    toc = f"{source}/_toc.yml"
+    filelist = []
+    url_base = "https://docs.neuroml.org"
+    url_map = {"DEFAULT": {"url": url_base}}
+    with open(toc, "r") as toc_f:
+        for line in toc_f:
+            if "file:" in line or "root:" in line:
+                if "file:" in line:
+                    source_file = Path(line.split("file:")[1].strip())
+                else:
+                    source_file = Path(line.split("root:")[1].strip())
+
+                if source_file.suffix == "":
+                    source_file_full = f"{source_file}.md"
+                else:
+                    source_file_full = str(source_file)
+
+                # ignore notebooks
+                if source_file_full.endswith("ipynb"):
+                    continue
+                print(f"Found: {source_file_full}")
+                filelist.append(source_file_full)
+    logger.debug(f"{filelist = }")
+
+    text = ""
+    schema_text = ""
+    # ignore lines starting with these
+    start_ignores = (
+        "----",
+        "language: ",
+        "lines: ",
+        ":class: ",
+        ":align: ",
+        ":alt: ",
+        ":scale: ",
+        ":gutter: ",
+        ":columns: ",
+        ":widths: ",
+        ":width: ",
+        ":delim: ",
+        ":all:",
+        "%",
+    )
+
+    # matches replace below
+    admons = ["admonition", "note", "warning", "tip", "important"]
+
+    # note that the order in which these are listed is important, since the
+    # regular expression substitutions are done sequentially in multiple passes
+    refs = {}
+    replacements = {
+        r"{ref}`(.+?)>`": r"\1>",
+        r"{ref}`(.+?)`": r"\1",
+        r"{doc}`(.+?)>`": r"\1>",
+        r"{eq}`(.+?)>`": r" (see equation \1)",
+        r"{cite}`(.+?)`": r"[citation: \1]",
+        r"{superscript}`(.+?)`": r"^\1",
+        # table inside tabs
+        r"`{4}{tab-item} (.+)\n`{3}{csv-table}\n": r"Table of \1 (separator='$')\n```\nName $ description $ reference\n",
+        # simple tabs
+        r"`{4}{tab-item} (.+)\n": r"\1\n",
+        # figures
+        r"{(image|figure)} (.+)": r"\nFigure: \2",
+        # admons
+        r"{(admonition|tip|warning|note|important)}": r"\nNOTE: ",
+        # other bracketed bits
+        r"{(code|code-block|download|grid-item-card|grid|tab-set|csv-table)}": r"",
+        # misc
+        r"(schema:|units:|<i>|</i>|&emsp;|`{5}|`{4})": r"",
+        r"(`{4})": r"\n",
+    }
+
+    for srcfile in filelist:
+        srcfilepath = Path(f"{source}/{srcfile}")
+        print(f"Processing {srcfilepath}")
+        adding_text_to = ""
+
+        with open(srcfilepath, "r") as srcfile_f:
+            in_block: list[str] = []
+            section_ref = ""
+            for line in srcfile_f:
+                # handle code includes
+                if line.startswith(start_ignores):
+                    logger.warning(f"Ignoring line: {line = }")
+                    continue
+                # logger.debug(f"Processing line: {line = }")
+
+                # section heading
+                if line.startswith("(") and line.strip().endswith(")="):
+                    section_ref = f"<{line[1:-3]}>"
+                    continue
+
+                # header
+                if not in_block and line.startswith("#"):
+                    header = line.replace("#", "", count=-1)
+                    url_map[header.strip()] = {
+                        "url": f"{url_base}/{srcfile.replace('.md', '.html')}"
+                    }
+
+                    # section ref preceds headers, processed in the next line after section heading is processed
+                    if len(section_ref) > 0:
+                        refs[section_ref] = (
+                            "(see section: " + line.replace("#", "").strip() + ")"
+                        )
+                        section_ref = ""
+
+                    adding_text_to += "\n" + line
+                    continue
+
+                if line.startswith("`"):
+                    leading_backticks = "".join(
+                        takewhile(lambda char: char == "`", line)
+                    )
+
+                    # single backticks arent blocks
+                    if len(leading_backticks) < 2:
+                        adding_text_to += line
+                        continue
+
+                    if len(in_block) and in_block[-1] == leading_backticks:
+                        logger.debug(f"Leaving block: {leading_backticks}")
+                        adding_text_to += f"{line.rstrip()}\n"
+                        in_block.pop()
+                        continue
+
+                    in_block.append(leading_backticks)
+                    logger.debug(f"Entered block {leading_backticks}")
+
+                    if "{literalinclude}" in line:
+                        file_to_include = line.split("{literalinclude}")[1].strip()
+                        with open(
+                            f"{srcfilepath.parent}/{file_to_include}", "r"
+                        ) as incfile_f:
+                            included_cont = incfile_f.read()
+                            adding_text_to += f"\n\n```\n\n{included_cont}\n\n"
+                        continue
+
+                    if "{bibliography}" in line:
+                        file_to_include = (
+                            line.split("{bibliography}")[1]
+                            .strip()
+                            .replace(".bib", ".md")
+                        )
+                        logger.info(f"Including bibliography file {file_to_include = }")
+                        with open(
+                            f"{srcfilepath.parent}/{file_to_include}", "r"
+                        ) as incfile_f:
+                            included_cont = incfile_f.read()
+                            adding_text_to += f"\n\n{included_cont}\n\n"
+                        continue
+
+                    isadmon = False
+                    for m in admons:
+                        if f"{{{m}}}" in line:
+                            isadmon = True
+                            admon_text = line.split(f"{{{m}}}")
+                            adding_text_to += (
+                                "```\n"
+                                + m.upper()
+                                + "\n"
+                                + admon_text[1].strip()
+                                + "\n\n"
+                            )
+                            break
+
+                    if not isadmon:
+                        adding_text_to += line
+
+                else:
+                    adding_text_to += line.rstrip() + "\n"
+
+        if "Schemas/" in str(srcfilepath):
+            schema_text += adding_text_to
+        else:
+            text += adding_text_to
+
+    for pat, rep in replacements.items():
+        text = re.sub(pat, rep, text, count=0, flags=re.M)
+    for pat, rep in refs.items():
+        text = re.sub(pat, rep, text, count=0)
+
+    for pat, rep in replacements.items():
+        schema_text = re.sub(pat, rep, schema_text, count=0, flags=re.M)
+    for pat, rep in refs.items():
+        schema_text = re.sub(pat, rep, schema_text, count=0)
+
+    with open("single-page-markdown.md", "w") as out:
+        print(text, file=out)
+
+    with open("single-page-markdown-schema.md", "w") as out:
+        print(schema_text, file=out)
+
+    with open("url-map.json", "w") as f:
+        json.dump(url_map, f)
+    # print(refs)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        sys.exit("Only one argument permitted: location of source folder")
+    runner(sys.argv[1])