| |
| """ |
| Generate a single page markdown from Jupyter book sources |
| |
| File: data/scripts/jupyterbook2singlemd.py |
| |
| Copyright 2026 Ankur Sinha |
| Author: Ankur Sinha <sanjay DOT ankur AT gmail DOT com> |
| """ |
|
|
| import json |
| import logging |
| import re |
| import sys |
| from itertools import takewhile |
| from pathlib import Path |
|
|
| import colorlog |
|
|
| logging.basicConfig(level=logging.NOTSET) |
| logger = logging.getLogger(__name__) |
| logger.setLevel(logging.DEBUG) |
|
|
| formatter = colorlog.ColoredFormatter( |
| "%(log_color)s%(name)s (%(levelname)s): %(message)s" |
| ) |
| handler = colorlog.StreamHandler() |
| handler.setLevel(logging.INFO) |
| handler.setFormatter(formatter) |
|
|
| logger.addHandler(handler) |
|
|
|
|
| def runner(source: str): |
| """Main runner |
| |
| :param source: TODO |
| :returns: TODO |
| |
| """ |
| toc = f"{source}/_toc.yml" |
| filelist = [] |
| url_base = "https://docs.neuroml.org" |
| url_map = {"DEFAULT": {"url": url_base}} |
| with open(toc, "r") as toc_f: |
| for line in toc_f: |
| if "file:" in line or "root:" in line: |
| if "file:" in line: |
| source_file = Path(line.split("file:")[1].strip()) |
| else: |
| source_file = Path(line.split("root:")[1].strip()) |
|
|
| if source_file.suffix == "": |
| source_file_full = f"{source_file}.md" |
| else: |
| source_file_full = str(source_file) |
|
|
| |
| if source_file_full.endswith("ipynb"): |
| continue |
| print(f"Found: {source_file_full}") |
| filelist.append(source_file_full) |
| logger.debug(f"{filelist = }") |
|
|
| text = "" |
| schema_text = "" |
| |
| start_ignores = ( |
| "----", |
| "language: ", |
| "lines: ", |
| ":class: ", |
| ":align: ", |
| ":alt: ", |
| ":scale: ", |
| ":gutter: ", |
| ":columns: ", |
| ":widths: ", |
| ":width: ", |
| ":delim: ", |
| ":all:", |
| "%", |
| ) |
|
|
| |
| admons = ["admonition", "note", "warning", "tip", "important"] |
|
|
| |
| |
| refs = {} |
| replacements = { |
| r"{ref}`(.+?)>`": r"\1>", |
| r"{ref}`(.+?)`": r"\1", |
| r"{doc}`(.+?)>`": r"\1>", |
| r"{eq}`(.+?)>`": r" (see equation \1)", |
| r"{cite}`(.+?)`": r"[citation: \1]", |
| r"{superscript}`(.+?)`": r"^\1", |
| |
| r"`{4}{tab-item} (.+)\n`{3}{csv-table}\n": r"Table of \1 (separator='$')\n```\nName $ description $ reference\n", |
| |
| r"`{4}{tab-item} (.+)\n": r"\1\n", |
| |
| r"{(image|figure)} (.+)": r"\nFigure: \2", |
| |
| r"{(admonition|tip|warning|note|important)}": r"\nNOTE: ", |
| |
| r"{(code|code-block|download|grid-item-card|grid|tab-set|csv-table)}": r"", |
| |
| r"(schema:|units:|<i>|</i>| |`{5}|`{4})": r"", |
| r"(`{4})": r"\n", |
| } |
|
|
| for srcfile in filelist: |
| srcfilepath = Path(f"{source}/{srcfile}") |
| print(f"Processing {srcfilepath}") |
| adding_text_to = "" |
|
|
| with open(srcfilepath, "r") as srcfile_f: |
| in_block: list[str] = [] |
| section_ref = "" |
| for line in srcfile_f: |
| |
| if line.startswith(start_ignores): |
| logger.warning(f"Ignoring line: {line = }") |
| continue |
| |
|
|
| |
| if line.startswith("(") and line.strip().endswith(")="): |
| section_ref = f"<{line[1:-3]}>" |
| continue |
|
|
| |
| if not in_block and line.startswith("#"): |
| header = line.replace("#", "", count=-1) |
| url_map[header.strip()] = { |
| "url": f"{url_base}/{srcfile.replace('.md', '.html')}" |
| } |
|
|
| |
| if len(section_ref) > 0: |
| refs[section_ref] = ( |
| "(see section: " + line.replace("#", "").strip() + ")" |
| ) |
| section_ref = "" |
|
|
| adding_text_to += "\n" + line |
| continue |
|
|
| if line.startswith("`"): |
| leading_backticks = "".join( |
| takewhile(lambda char: char == "`", line) |
| ) |
|
|
| |
| if len(leading_backticks) < 2: |
| adding_text_to += line |
| continue |
|
|
| if len(in_block) and in_block[-1] == leading_backticks: |
| logger.debug(f"Leaving block: {leading_backticks}") |
| adding_text_to += f"{line.rstrip()}\n" |
| in_block.pop() |
| continue |
|
|
| in_block.append(leading_backticks) |
| logger.debug(f"Entered block {leading_backticks}") |
|
|
| if "{literalinclude}" in line: |
| file_to_include = line.split("{literalinclude}")[1].strip() |
| with open( |
| f"{srcfilepath.parent}/{file_to_include}", "r" |
| ) as incfile_f: |
| included_cont = incfile_f.read() |
| adding_text_to += f"\n\n```\n\n{included_cont}\n\n" |
| continue |
|
|
| if "{bibliography}" in line: |
| file_to_include = ( |
| line.split("{bibliography}")[1] |
| .strip() |
| .replace(".bib", ".md") |
| ) |
| logger.info(f"Including bibliography file {file_to_include = }") |
| with open( |
| f"{srcfilepath.parent}/{file_to_include}", "r" |
| ) as incfile_f: |
| included_cont = incfile_f.read() |
| adding_text_to += f"\n\n{included_cont}\n\n" |
| continue |
|
|
| isadmon = False |
| for m in admons: |
| if f"{{{m}}}" in line: |
| isadmon = True |
| admon_text = line.split(f"{{{m}}}") |
| adding_text_to += ( |
| "```\n" |
| + m.upper() |
| + "\n" |
| + admon_text[1].strip() |
| + "\n\n" |
| ) |
| break |
|
|
| if not isadmon: |
| adding_text_to += line |
|
|
| else: |
| adding_text_to += line.rstrip() + "\n" |
|
|
| if "Schemas/" in str(srcfilepath): |
| schema_text += adding_text_to |
| else: |
| text += adding_text_to |
|
|
| for pat, rep in replacements.items(): |
| text = re.sub(pat, rep, text, count=0, flags=re.M) |
| for pat, rep in refs.items(): |
| text = re.sub(pat, rep, text, count=0) |
|
|
| for pat, rep in replacements.items(): |
| schema_text = re.sub(pat, rep, schema_text, count=0, flags=re.M) |
| for pat, rep in refs.items(): |
| schema_text = re.sub(pat, rep, schema_text, count=0) |
|
|
| with open("single-page-markdown.md", "w") as out: |
| print(text, file=out) |
|
|
| with open("single-page-markdown-schema.md", "w") as out: |
| print(schema_text, file=out) |
|
|
| with open("url-map.json", "w") as f: |
| json.dump(url_map, f) |
| |
|
|
|
|
| if __name__ == "__main__": |
| if len(sys.argv) != 2: |
| sys.exit("Only one argument permitted: location of source folder") |
| runner(sys.argv[1]) |
|
|
