This view is limited to 50 files because it contains too many changes. See the raw diff here.
Files changed (50) hide show
  1. .gitmodules +0 -3
  2. Dockerfile +9 -11
  3. klea_rag_osb.json +2 -9
  4. klea_rag_osb_no_mcp.json +0 -36
  5. klea_rag_osb_test.json +2 -9
  6. klea_rag_osb_test_no_mcp.json +0 -37
  7. mcp +0 -1
  8. scripts/docker-deploy.sh +2 -2
  9. scripts/osb-repos2sources.py +67 -70
  10. store/{src/osb-data/mds/.klea-cache/xxh64_006f196d935f8dfd.pkl → osb-data/.klea-cache/doi-cache.json} +2 -2
  11. store/osb-data/{bcee67b8-5631-42ba-9b16-6aa5e38d5a49/link_lists.bin → .klea-cache/metadata-map.template.json} +2 -2
  12. store/{src/osb-data/mds/.klea-cache/xxh64_009e659760ebadeb.pkl → osb-data/.klea-cache/osb-repositories.manifest.json} +2 -2
  13. store/osb-data/{bcee67b8-5631-42ba-9b16-6aa5e38d5a49/index_metadata.pickle → .klea-cache/xxh64_9d68c40677dac9ea.pkl} +2 -2
  14. store/{src/osb-data/mds/.klea-cache/xxh64_af90366bac3efdc7.pkl → osb-data/2cb7d4ad-ed72-4279-bd78-8596b33c9d3d/data_level0.bin} +2 -2
  15. store/osb-data/{bcee67b8-5631-42ba-9b16-6aa5e38d5a49 → 2cb7d4ad-ed72-4279-bd78-8596b33c9d3d}/header.bin +1 -1
  16. store/{src/osb-data/mds/.klea-cache/xxh64_4a972373fdfd7c47.pkl → osb-data/2cb7d4ad-ed72-4279-bd78-8596b33c9d3d/index_metadata.pickle} +2 -2
  17. store/osb-data/{bcee67b8-5631-42ba-9b16-6aa5e38d5a49 → 2cb7d4ad-ed72-4279-bd78-8596b33c9d3d}/length.bin +2 -2
  18. store/{src/osb-data/mds/.klea-cache/xxh64_5c945e292716fbc5.pkl → osb-data/2cb7d4ad-ed72-4279-bd78-8596b33c9d3d/link_lists.bin} +2 -2
  19. store/osb-data/bcee67b8-5631-42ba-9b16-6aa5e38d5a49/data_level0.bin +0 -3
  20. store/osb-data/chroma.sqlite3 +2 -2
  21. store/osb-data/{osb-repositories.sqlite → osb-data-repos.sqlite} +0 -0
  22. store/osb-data/osb-repositories-bm25.pkl +2 -2
  23. store/osb-data/src/osb-data-repos.md +3 -0
  24. store/{src/osb-data → osb-data/src}/repos_v2.json +0 -0
  25. store/{src/osb-docs → osb-docs}/.klea-cache/doi-cache.json +0 -0
  26. store/{src/osb-docs → osb-docs}/.klea-cache/metadata-map.template.json +0 -0
  27. store/{src/osb-docs → osb-docs}/.klea-cache/osb-docs.manifest.json +0 -0
  28. store/{src/osb-docs → osb-docs}/.klea-cache/xxh64_a0e60c123c8f31ca.pkl +0 -0
  29. store/osb-docs/3996496d-ffbf-4825-af9a-b3a1c07f9ad6/length.bin +1 -1
  30. store/osb-docs/chroma.sqlite3 +1 -1
  31. store/{src/osb-docs → osb-docs}/osb-docs-single-page-url-map.json +0 -0
  32. store/{src/osb-docs → osb-docs}/osb-docs-single-page.md +0 -0
  33. store/src/osb-data/mds/.klea-cache/doi-cache.json +0 -3
  34. store/src/osb-data/mds/.klea-cache/metadata-map.template.json +0 -3
  35. store/src/osb-data/mds/.klea-cache/osb-repositories.manifest.json +0 -3
  36. store/src/osb-data/mds/.klea-cache/xxh64_00605ccb10f8072c.pkl +0 -3
  37. store/src/osb-data/mds/.klea-cache/xxh64_0061771528498b74.pkl +0 -3
  38. store/src/osb-data/mds/.klea-cache/xxh64_0063d5c288597021.pkl +0 -3
  39. store/src/osb-data/mds/.klea-cache/xxh64_006bb9972f79917d.pkl +0 -3
  40. store/src/osb-data/mds/.klea-cache/xxh64_006ed9a7472df23a.pkl +0 -3
  41. store/src/osb-data/mds/.klea-cache/xxh64_00732c600078d475.pkl +0 -3
  42. store/src/osb-data/mds/.klea-cache/xxh64_00a5d7fc2189b2b3.pkl +0 -3
  43. store/src/osb-data/mds/.klea-cache/xxh64_00a7803c9e534daa.pkl +0 -3
  44. store/src/osb-data/mds/.klea-cache/xxh64_00b40dd3efee9dd2.pkl +0 -3
  45. store/src/osb-data/mds/.klea-cache/xxh64_00be15f33c186916.pkl +0 -3
  46. store/src/osb-data/mds/.klea-cache/xxh64_00c990bcb55edab0.pkl +0 -3
  47. store/src/osb-data/mds/.klea-cache/xxh64_00cf0f2dac51a74e.pkl +0 -3
  48. store/src/osb-data/mds/.klea-cache/xxh64_00d77a9a57bcee32.pkl +0 -3
  49. store/src/osb-data/mds/.klea-cache/xxh64_00f6bdeff09f187b.pkl +0 -3
  50. store/src/osb-data/mds/.klea-cache/xxh64_0103811ef96a5123.pkl +0 -3
.gitmodules DELETED
@@ -1,3 +0,0 @@
1
- [submodule "mcp"]
2
- path = mcp
3
- url = https://github.com/OpenSourceBrain/osb-mcp.git
 
 
 
 
Dockerfile CHANGED
@@ -2,14 +2,14 @@ FROM fedora:44
2
 
3
  EXPOSE 7860
4
 
5
- RUN dnf install python3.13 uv git-core sudo curl procps-ng ca-certificates -y \
6
- && useradd -m -u 1000 user \
7
- && mkdir -p /src /app/scripts /app/models \
8
- && chown -R user:user /src /app \
9
- && dnf clean all
10
 
 
11
  # for the git checkout while we use git sources
 
12
  # for the app
 
 
13
 
14
  # install ollama, download model
15
  RUN curl -fsSL https://ollama.com/install.sh | sh
@@ -28,20 +28,19 @@ RUN ollama serve & \
28
  # set to invalidate cache
29
  ARG CACHE_DATE=1
30
  # currently off development
31
- RUN git clone --branch=development --depth 1 https://github.com/NeuroML/klea.git /src
 
32
 
33
  WORKDIR /src
34
 
35
  RUN uv venv --python=/usr/bin/python3.13 /app/klea-venv
36
  ENV VIRTUAL_ENV=/app/klea-venv
37
  ENV PATH="$VIRTUAL_ENV/bin:$PATH"
38
- ENV UV_LINK_MODE=hardlink
39
- ENV UV_CACHE_DIR=/tmp/uv-cache
40
 
41
  RUN pwd
42
  RUN ls -lash
43
- # needs to be updated to use the update osbv2-mcp package.
44
- RUN uv pip install './utils_pkg[chroma,huggingface,nicegui,ollama]' './mcp' './rag_pkg[chroma,huggingface,nicegui,ollama]' && rm -rf /tmp/uv-cache
45
  RUN ls -lash "$VIRTUAL_ENV/bin/"
46
 
47
  RUN uv pip freeze
@@ -60,7 +59,6 @@ RUN ls -lash /app/*
60
  RUN ls -lashR /app/store/
61
 
62
  ENV KLEA_RAG_ENV_FILE=rag-hf.env
63
- ENV OSB_DATA_DB_PATH=/app/store/osb-data/osb-repositories.sqlite
64
  ENV RUNNING_IN_DOCKER=1
65
 
66
  CMD ["./scripts/docker-deploy.sh"]
 
2
 
3
  EXPOSE 7860
4
 
5
+ RUN dnf install python3.13 uv git-core sudo curl procps-ng ca-certificates -y && dnf clean all
 
 
 
 
6
 
7
+ RUN useradd -m -u 1000 user
8
  # for the git checkout while we use git sources
9
+ RUN mkdir -p /src && chown -R user:user /src
10
  # for the app
11
+ RUN mkdir -p /app/scripts && chown -R user:user /app
12
+ RUN mkdir -p /app/models && chown -R user:user /app
13
 
14
  # install ollama, download model
15
  RUN curl -fsSL https://ollama.com/install.sh | sh
 
28
  # set to invalidate cache
29
  ARG CACHE_DATE=1
30
  # currently off development
31
+ RUN git clone --branch=development --depth 1 https://github.com/NeuroML/neuroklea.git /src
32
+ RUN chown -R user:user /src
33
 
34
  WORKDIR /src
35
 
36
  RUN uv venv --python=/usr/bin/python3.13 /app/klea-venv
37
  ENV VIRTUAL_ENV=/app/klea-venv
38
  ENV PATH="$VIRTUAL_ENV/bin:$PATH"
39
+ ENV UV_LINK_MODE=copy
 
40
 
41
  RUN pwd
42
  RUN ls -lash
43
+ RUN --mount=type=cache,target=/home/user/.cache/uv,uid=1000,gid=1000 uv pip install './utils_pkg[chroma,huggingface,nicegui,ollama]' './mcp_pkg' './rag_pkg[chroma,huggingface,nicegui,ollama]'
 
44
  RUN ls -lash "$VIRTUAL_ENV/bin/"
45
 
46
  RUN uv pip freeze
 
59
  RUN ls -lashR /app/store/
60
 
61
  ENV KLEA_RAG_ENV_FILE=rag-hf.env
 
62
  ENV RUNNING_IN_DOCKER=1
63
 
64
  CMD ["./scripts/docker-deploy.sh"]
klea_rag_osb.json CHANGED
@@ -10,7 +10,7 @@
10
  },
11
  "domains": {
12
  "Open Source Brain": {
13
- "description": "Questions about the Open Source Brain neuroscience research web platform (OSB). This includes questions on general usage and querying the data, repositories, and workspaces available on the platform. Note that there are two versions of Open Source Brain. v1, and v2. v2 should be preferred as v1 is being sunset. This domain has access to information stores.",
14
  "vector_stores": [
15
  {
16
  "name": "osb-docs",
@@ -30,14 +30,7 @@
30
  "name": "osb-repositories",
31
  "path": "/app/store/osb-data/osb-repositories-bm25.pkl"
32
  }
33
- ],
34
- "mcp_servers": {
35
- "Open Source Brain": {
36
- "url": "http://127.0.0.1:8542/mcp",
37
- "exclude_tags": ["disabled"],
38
- "include_tags": ["sqlite"]
39
- }
40
- }
41
  }
42
  }
43
  }
 
10
  },
11
  "domains": {
12
  "Open Source Brain": {
13
+ "description": "Questions about the Open Source Brain neuroscience research web platform (OSB). This includes questions on general usage and querying the data, repositories, and workspaces available on the platform. Note that there are two versions of Open Source Brain. v1, and v2. v2 should be preferred as v1 is being sunset",
14
  "vector_stores": [
15
  {
16
  "name": "osb-docs",
 
30
  "name": "osb-repositories",
31
  "path": "/app/store/osb-data/osb-repositories-bm25.pkl"
32
  }
33
+ ]
 
 
 
 
 
 
 
34
  }
35
  }
36
  }
klea_rag_osb_no_mcp.json DELETED
@@ -1,36 +0,0 @@
1
- {
2
- "general":
3
- {
4
- "default_k": 5,
5
- "k_max": 10,
6
- "non_domain_chat": false,
7
- "pre_prompt": "",
8
- "fallback_to_training_data": true,
9
- "fallback_warning": "Note: information on the query could not be found in the vector stores; this answer is generated from the LLM's training data and should be verified."
10
- },
11
- "domains": {
12
- "Open Source Brain": {
13
- "description": "Questions about the Open Source Brain neuroscience research web platform (OSB). This includes questions on general usage and querying the data, repositories, and workspaces available on the platform. Note that there are two versions of Open Source Brain. v1, and v2. v2 should be preferred as v1 is being sunset",
14
- "vector_stores": [
15
- {
16
- "name": "osb-docs",
17
- "path": "chroma:/app/store/osb-docs/"
18
- },
19
- {
20
- "name": "osb-repositories",
21
- "path": "chroma:/app/store/osb-data/"
22
- }
23
- ],
24
- "bm25_stores": [
25
- {
26
- "name": "osb-docs",
27
- "path": "/app/store/osb-docs/osb-docs-bm25.pkl"
28
- },
29
- {
30
- "name": "osb-repositories",
31
- "path": "/app/store/osb-data/osb-repositories-bm25.pkl"
32
- }
33
- ]
34
- }
35
- }
36
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
klea_rag_osb_test.json CHANGED
@@ -11,7 +11,7 @@
11
  },
12
  "domains": {
13
  "Open Source Brain": {
14
- "description": "Questions about the Open Source Brain neuroscience research web platform (OSB). This includes questions on general usage and querying the data, repositories, and workspaces available on the platform. Note that there are two versions of Open Source Brain. v1, and v2. v2 should be preferred as v1 is being sunset. This domain has access to information stores.",
15
  "vector_stores": [
16
  {
17
  "name": "osb-docs",
@@ -31,14 +31,7 @@
31
  "name": "osb-repositories",
32
  "path": "./store/osb-data/osb-repositories-bm25.pkl"
33
  }
34
- ],
35
- "mcp_servers": {
36
- "Open Source Brain": {
37
- "url": "http://127.0.0.1:8542/mcp",
38
- "exclude_tags": ["disabled"],
39
- "include_tags": ["sqlite"]
40
- }
41
- }
42
  }
43
  }
44
  }
 
11
  },
12
  "domains": {
13
  "Open Source Brain": {
14
+ "description": "Questions about the Open Source Brain neuroscience research web platform (OSB). This includes questions on general usage and querying the data, repositories, and workspaces available on the platform. Note that there are two versions of Open Source Brain. v1, and v2. v2 should be preferred as v1 is being sunset",
15
  "vector_stores": [
16
  {
17
  "name": "osb-docs",
 
31
  "name": "osb-repositories",
32
  "path": "./store/osb-data/osb-repositories-bm25.pkl"
33
  }
34
+ ]
 
 
 
 
 
 
 
35
  }
36
  }
37
  }
klea_rag_osb_test_no_mcp.json DELETED
@@ -1,37 +0,0 @@
1
-
2
- {
3
- "general":
4
- {
5
- "default_k": 5,
6
- "k_max": 10,
7
- "non_domain_chat": false,
8
- "pre_prompt": "",
9
- "fallback_to_training_data": true,
10
- "fallback_warning": "Note: information on the query could not be found in the vector stores; this answer is generated from the LLM's training data and should be verified."
11
- },
12
- "domains": {
13
- "Open Source Brain": {
14
- "description": "Questions about the Open Source Brain neuroscience research web platform (OSB). This includes questions on general usage and querying the data, repositories, and workspaces available on the platform. Note that there are two versions of Open Source Brain. v1, and v2. v2 should be preferred as v1 is being sunset",
15
- "vector_stores": [
16
- {
17
- "name": "osb-docs",
18
- "path": "chroma:./store/osb-docs/"
19
- },
20
- {
21
- "name": "osb-repositories",
22
- "path": "chroma:./store/osb-data/"
23
- }
24
- ],
25
- "bm25_stores": [
26
- {
27
- "name": "osb-docs",
28
- "path": "./store/osb-docs/osb-docs-bm25.pkl"
29
- },
30
- {
31
- "name": "osb-repositories",
32
- "path": "./store/osb-data/osb-repositories-bm25.pkl"
33
- }
34
- ]
35
- }
36
- }
37
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
mcp DELETED
@@ -1 +0,0 @@
1
- Subproject commit 11756f742271f95129a728b2f6b34785ec3ae59d
 
 
scripts/docker-deploy.sh CHANGED
@@ -16,8 +16,8 @@ then
16
  ollama ls
17
  fi
18
 
19
- echo "Starting OSB MCP server"
20
- osb-mcp &
21
 
22
  echo "Starting fastapi"
23
  klea-rag-serve --host 127.0.0.1 --port 8005 &
 
16
  ollama ls
17
  fi
18
 
19
+ #echo "Re-starting MCP server"
20
+ #nml-mcp &
21
 
22
  echo "Starting fastapi"
23
  klea-rag-serve --host 127.0.0.1 --port 8005 &
scripts/osb-repos2sources.py CHANGED
@@ -2,19 +2,13 @@
2
  """
3
  Convert the OSB repositories JSON export into two sources for the Klea RAG:
4
 
5
- * one Markdown file per repository (in a source folder) for
6
- vector-store (semantic) search, and
7
  * a read-only SQLite database for the sqlite_query RAG tool.
8
 
9
- Input: (required argument) the OSB repositories JSON export
10
- Output: <repos-dir>/<id>-<name>.md -- one file per repository
11
  store/osb-data/osb-data-repos.sqlite
12
 
13
- Each per-repository Markdown file carries a labeled ``URL:`` line near the
14
- top, so Klea's metadata-extraction cascade auto-populates that repository's
15
- ``DEFAULT`` metadata entry with its URI -- instead of every chunk inheriting
16
- a single top-level URL as happened with the old single-page file.
17
-
18
  Re-run this script any time the JSON is updated; both outputs are regenerated
19
  idempotently.
20
 
@@ -31,7 +25,6 @@ import logging
31
  import re
32
  import sqlite3
33
  import sys
34
- import unicodedata
35
  from pathlib import Path
36
 
37
  logger = logging.getLogger(__name__)
@@ -40,6 +33,9 @@ logger = logging.getLogger(__name__)
40
  #: leaking who created/maintains each repository.
41
  EXCLUDED_FIELDS = ("auto_sync", "user", "user_id", "content_types_list")
42
 
 
 
 
43
 
44
  def _clean_summary(raw: str | None) -> str:
45
  """Return a plain-text version of a repository summary.
@@ -48,7 +44,7 @@ def _clean_summary(raw: str | None) -> str:
48
  are stripped of markup and HTML entities first. The result may still
49
  contain Markdown constructs (headings, links, images, emphasis, lists),
50
  which are also stripped so the text can be embedded safely as prose in
51
- the per-repository Markdown source. Returns an empty string when there is
52
  nothing usable.
53
  """
54
  if not raw:
@@ -88,61 +84,62 @@ def _tags_of(repo: dict) -> list[str]:
88
  return [t.get("tag", "") for t in tags if isinstance(t, dict)]
89
 
90
 
91
- def _slug_name(repo: dict) -> str:
92
- """Return a filesystem-safe ``<id>-<name>.md`` filename for *repo*.
93
-
94
- The numeric id prefix guarantees a unique filename even when repository
95
- names collide (several repos share a name) or sanitise to the same slug
96
- (e.g. names with slashes such as ``Figueredo2013/1``).
97
- """
98
- rid = int(repo["id"])
99
- name = repo.get("name") or repo.get("uri") or f"Repository {rid}"
100
- name = unicodedata.normalize("NFKC", name)
101
- name = re.sub(r"[\s/]+", "-", name)
102
- name = re.sub(r'[\\:*?"<>|\x00-\x1f\x7f]', "", name)
103
- name = re.sub(r"-+", "-", name).strip("-. ")
104
- if not name:
105
- name = f"repo-{rid}"
106
- return f"{rid}-{name}.md"
107
-
108
-
109
- def _markdown_per_repo(repo: dict) -> str:
110
- """Render a single repository as its own Markdown document.
111
-
112
- The ``URL:`` line is a labeled line that Klea's metadata-extraction
113
- cascade recognises (the ``_URL_LABELED_RE`` regex), so each per-repo
114
- file's ``DEFAULT`` metadata entry is auto-populated with the repository
115
- URI rather than falling back to a single top-level URL.
116
- """
117
- rid = int(repo["id"])
118
- name = repo.get("name") or repo.get("uri") or f"Repository {rid}"
119
- lines = [f"# {name}", ""]
120
-
121
- uri = repo.get("uri") or ""
122
- if uri:
123
- lines += [f"URL: {uri}", ""]
124
-
125
- lines += [
126
- f"- **Branch/context:** {repo.get('default_context') or ''}",
127
- f"- **Content types:** {repo.get('content_types') or ''}",
128
- f"- **Repository type:** {repo.get('repository_type') or ''}",
129
- f"- **Tags:** {', '.join(_tags_of(repo)) or 'None'}",
130
- f"- **OSB ID:** {rid}",
131
  "",
132
  ]
133
 
134
- summary = _clean_summary(repo.get("summary"))
135
- lines.append(summary if summary else "No summary available.")
136
- return "\n".join(lines).rstrip() + "\n"
137
-
138
-
139
- def _markdown_repos(repos_dir: Path, repositories: dict) -> None:
140
- """Write one Markdown file per repository into *repos_dir*."""
141
- repos_dir.mkdir(parents=True, exist_ok=True)
142
  for repo in repositories.values():
143
- (repos_dir / _slug_name(repo)).write_text(
144
- _markdown_per_repo(repo), encoding="utf-8"
145
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
146
 
147
 
148
  def _sqlite(database_path: Path, repositories: dict) -> None:
@@ -226,12 +223,12 @@ def _sqlite(database_path: Path, repositories: dict) -> None:
226
  conn.close()
227
 
228
 
229
- def runner(json_path: Path, repos_dir: Path, database_path: Path) -> None:
230
  repositories = _load_repositories(json_path)
231
  logger.info("Loaded %d repositories from %s", len(repositories), json_path)
232
 
233
- _markdown_repos(repos_dir, repositories)
234
- logger.info("Wrote per-repository Markdown files to %s", repos_dir)
235
 
236
  _sqlite(database_path, repositories)
237
  logger.info("Wrote SQLite database: %s", database_path)
@@ -246,16 +243,16 @@ def main(argv: list[str] | None = None) -> int:
246
  "--json",
247
  dest="json_path",
248
  type=Path,
249
- required=True,
250
  help="Path to the OSB repositories JSON export.",
251
  )
252
  parser.add_argument(
253
  "-m",
254
  "--markdown",
255
- dest="repos_dir",
256
  type=Path,
257
- default=Path("store/src/osb-data/repos"),
258
- help="Directory for the generated per-repository Markdown files.",
259
  )
260
  parser.add_argument(
261
  "-d",
@@ -279,7 +276,7 @@ def main(argv: list[str] | None = None) -> int:
279
  logger.error("Input JSON not found: %s", args.json_path)
280
  return 1
281
 
282
- runner(args.json_path, args.repos_dir, args.database_path)
283
  return 0
284
 
285
 
 
2
  """
3
  Convert the OSB repositories JSON export into two sources for the Klea RAG:
4
 
5
+ * a single-page Markdown file for vector-store (semantic) search, and
 
6
  * a read-only SQLite database for the sqlite_query RAG tool.
7
 
8
+ Input: store/osb-data/repos_v2.json
9
+ Output: store/osb-data/osb-data-repos.md
10
  store/osb-data/osb-data-repos.sqlite
11
 
 
 
 
 
 
12
  Re-run this script any time the JSON is updated; both outputs are regenerated
13
  idempotently.
14
 
 
25
  import re
26
  import sqlite3
27
  import sys
 
28
  from pathlib import Path
29
 
30
  logger = logging.getLogger(__name__)
 
33
  #: leaking who created/maintains each repository.
34
  EXCLUDED_FIELDS = ("auto_sync", "user", "user_id", "content_types_list")
35
 
36
+ #: Ordered list of repository types for stable, grouped Markdown output.
37
+ REPOSITORY_TYPE_ORDER = ("github", "dandi", "biomodels", "figshare")
38
+
39
 
40
  def _clean_summary(raw: str | None) -> str:
41
  """Return a plain-text version of a repository summary.
 
44
  are stripped of markup and HTML entities first. The result may still
45
  contain Markdown constructs (headings, links, images, emphasis, lists),
46
  which are also stripped so the text can be embedded safely as prose in
47
+ the single-page Markdown source. Returns an empty string when there is
48
  nothing usable.
49
  """
50
  if not raw:
 
84
  return [t.get("tag", "") for t in tags if isinstance(t, dict)]
85
 
86
 
87
+ def _markdown(repositories: dict) -> str:
88
+ """Render the single-page Markdown document."""
89
+ lines = [
90
+ "# Open Source Brain - Repositories",
91
+ "",
92
+ (
93
+ "Open Source Brain (OSB) is a web platform for sharing, viewing, "
94
+ "analysing, and simulating computational models of neurons and "
95
+ "networks, and for sharing standardised neurophysiology data. "
96
+ f"This document catalogues {len(repositories)} repositories "
97
+ "available on the platform."
98
+ ),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
99
  "",
100
  ]
101
 
102
+ groups: dict[str, list[dict]] = {}
 
 
 
 
 
 
 
103
  for repo in repositories.values():
104
+ rtype = repo.get("repository_type", "other") or "other"
105
+ groups.setdefault(rtype, []).append(repo)
106
+
107
+ for rtype in list(REPOSITORY_TYPE_ORDER) + [
108
+ r for r in sorted(groups) if r not in REPOSITORY_TYPE_ORDER
109
+ ]:
110
+ repos = groups.get(rtype, [])
111
+ title = {
112
+ "github": "GitHub repositories",
113
+ "dandi": "DANDI datasets",
114
+ "biomodels": "BioModels",
115
+ "figshare": "Figshare",
116
+ }.get(rtype, rtype)
117
+ lines += [f"## {rtype} ({len(repos)})", "", f"{title}.", ""]
118
+
119
+ for repo in repos:
120
+ name = repo.get("name") or repo.get("uri") or f"Repository {repo['id']}"
121
+ lines += [f"### {name}", ""]
122
+ lines += [f"- **URI:** {repo.get('uri', '')}", ""]
123
+ lines += [f"- **Branch/context:** {repo.get('default_context') or ''}", ""]
124
+ lines += [
125
+ f"- **Content types:** {repo.get('content_types') or ''}",
126
+ "",
127
+ ]
128
+ lines += [f"- **Repository type:** {repo.get('repository_type') or ''}", ""]
129
+
130
+ tags = ", ".join(_tags_of(repo))
131
+ lines += [f"- **Tags:** {tags if tags else 'None'}", ""]
132
+ lines += [""]
133
+
134
+ summary = _clean_summary(repo.get("summary"))
135
+ if summary:
136
+ lines += [summary, ""]
137
+ else:
138
+ lines += ["No summary available.", ""]
139
+
140
+ lines += [""]
141
+
142
+ return "\n".join(lines).rstrip() + "\n"
143
 
144
 
145
  def _sqlite(database_path: Path, repositories: dict) -> None:
 
223
  conn.close()
224
 
225
 
226
+ def runner(json_path: Path, markdown_path: Path, database_path: Path) -> None:
227
  repositories = _load_repositories(json_path)
228
  logger.info("Loaded %d repositories from %s", len(repositories), json_path)
229
 
230
+ markdown_path.write_text(_markdown(repositories), encoding="utf-8")
231
+ logger.info("Wrote Markdown: %s", markdown_path)
232
 
233
  _sqlite(database_path, repositories)
234
  logger.info("Wrote SQLite database: %s", database_path)
 
243
  "--json",
244
  dest="json_path",
245
  type=Path,
246
+ default=Path("store/osb-data/repos_v2.json"),
247
  help="Path to the OSB repositories JSON export.",
248
  )
249
  parser.add_argument(
250
  "-m",
251
  "--markdown",
252
+ dest="markdown_path",
253
  type=Path,
254
+ default=Path("store/osb-data/osb-data-repos.md"),
255
+ help="Path for the generated single-page Markdown file.",
256
  )
257
  parser.add_argument(
258
  "-d",
 
276
  logger.error("Input JSON not found: %s", args.json_path)
277
  return 1
278
 
279
+ runner(args.json_path, args.markdown_path, args.database_path)
280
  return 0
281
 
282
 
store/{src/osb-data/mds/.klea-cache/xxh64_006f196d935f8dfd.pkl → osb-data/.klea-cache/doi-cache.json} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:02ba2193bfe5ee83e71c5170ff915306725804d8160a823f34026aab63b39b7f
3
- size 662
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0390b4d86e9f79803ee161eda45fd8244655959090796acdbe2f6d0f8461c2aa
3
+ size 340
store/osb-data/{bcee67b8-5631-42ba-9b16-6aa5e38d5a49/link_lists.bin → .klea-cache/metadata-map.template.json} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:7d67533450df2a4fa8a6c6d6a652831d238e859a9f9b4fac9ea059841b6a954e
3
- size 44756
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5ea62679b75100f2ae232facc8874cea0b9efb47e16747b4e677ff208e1c09c8
3
+ size 497160
store/{src/osb-data/mds/.klea-cache/xxh64_009e659760ebadeb.pkl → osb-data/.klea-cache/osb-repositories.manifest.json} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b621e0d2a1e251c73adeae02222beb81d9c7d2f4d9c549aa80a248d8cbb95148
3
- size 599
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9cda887b55cb7d1661a6c4b27d980abbb0dae0c69c60b72a5e7d5081a5b4a466
3
+ size 176
store/osb-data/{bcee67b8-5631-42ba-9b16-6aa5e38d5a49/index_metadata.pickle → .klea-cache/xxh64_9d68c40677dac9ea.pkl} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:520c483e8c911324edd40441c848f10ddb2cfb4332a0bfa76a898f698260a690
3
- size 833028
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e7d98eebbd894ccf314eec45c9cb8ab2d4f15aa6066ce974d3fbc120063005a5
3
+ size 5626138
store/{src/osb-data/mds/.klea-cache/xxh64_af90366bac3efdc7.pkl → osb-data/2cb7d4ad-ed72-4279-bd78-8596b33c9d3d/data_level0.bin} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:aed135a5a36c4ca427f1a50eed43541be8f87c6ad9682210c60fa8cdb6980f3f
3
- size 1273
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:215d9995f5febad8c5d761d344f437ee8a6bb5a5582052d3c5f483ce062d0c48
3
+ size 21688320
store/osb-data/{bcee67b8-5631-42ba-9b16-6aa5e38d5a49 → 2cb7d4ad-ed72-4279-bd78-8596b33c9d3d}/header.bin RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:c232f1dba4f4a260767ac902eefab9b622382fb4eaa096485267e91082f62eee
3
  size 100
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1c0397512c65a3e764ff96dbffa7451628ab94fef6897673081e670de8858d11
3
  size 100
store/{src/osb-data/mds/.klea-cache/xxh64_4a972373fdfd7c47.pkl → osb-data/2cb7d4ad-ed72-4279-bd78-8596b33c9d3d/index_metadata.pickle} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:7095435bd8a503eaf9d11d5051c78175c14bafa8490c786f58f2cb0f726ccf4e
3
- size 2177
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:63713b8be25ed6f1bfa2378f694c4b819743df31d4ab3b106d407d3ae93e8798
3
+ size 325608
store/osb-data/{bcee67b8-5631-42ba-9b16-6aa5e38d5a49 → 2cb7d4ad-ed72-4279-bd78-8596b33c9d3d}/length.bin RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5807024c6bc5a0e1c33537fce16ebe66aac201ef795f8acde84d2487f5ade0e0
3
- size 20004
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c70978004f57a4dda63550719b362b9ebdf4e9b115327581fccba817f877dcf
3
+ size 20480
store/{src/osb-data/mds/.klea-cache/xxh64_5c945e292716fbc5.pkl → osb-data/2cb7d4ad-ed72-4279-bd78-8596b33c9d3d/link_lists.bin} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:92e4456699a5c9b41797c908f9a351a893f8fa067f41bb5f29e76f2fe9525c32
3
- size 1727
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e8af8ddf6725c51b3b1b5d3badb9fbd2beb6ca176f86f87ee60b02581d144418
3
+ size 45980
store/osb-data/bcee67b8-5631-42ba-9b16-6aa5e38d5a49/data_level0.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:8552bcecc66bd5b4c280e4c16687ee445eaa58d951e851c9e27c0171930ac7a3
3
- size 21184236
 
 
 
 
store/osb-data/chroma.sqlite3 CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:3e414d1fddfb1ff7da34a881bd9ed88c0728d44ab49e31d8c2095cf986929e38
3
- size 50016256
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4d898b32d0bda6afd2d6f2c4ba7b381b2698ee18b9617208f2d20b8f533700c6
3
+ size 50958336
store/osb-data/{osb-repositories.sqlite → osb-data-repos.sqlite} RENAMED
File without changes
store/osb-data/osb-repositories-bm25.pkl CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:f125f0f96130d5450e79e6f4d355f9bb224952419161e063c2910f50ecb45d8b
3
- size 6722680
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8d83c2773282449c094ce8616fbd8deb9c8c34aa0d9ed7814e925ca48f327b55
3
+ size 5764802
store/osb-data/src/osb-data-repos.md ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e9d8fc248349f5e22419ed4002875bc85785a5b115d8ebbd37195ad2c64fa4fa
3
+ size 4706445
store/{src/osb-data → osb-data/src}/repos_v2.json RENAMED
File without changes
store/{src/osb-docs → osb-docs}/.klea-cache/doi-cache.json RENAMED
File without changes
store/{src/osb-docs → osb-docs}/.klea-cache/metadata-map.template.json RENAMED
File without changes
store/{src/osb-docs → osb-docs}/.klea-cache/osb-docs.manifest.json RENAMED
File without changes
store/{src/osb-docs → osb-docs}/.klea-cache/xxh64_a0e60c123c8f31ca.pkl RENAMED
File without changes
store/osb-docs/3996496d-ffbf-4825-af9a-b3a1c07f9ad6/length.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:8680581dacab8249751d2170d752dce0437067d0182acc260deb3db9d618c677
3
  size 400
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c7265dd487120006d66ce80470c53245ef0edc05595d7b1ff23b085ca9d000fe
3
  size 400
store/osb-docs/chroma.sqlite3 CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:38c46e3678d01494141978295d9ed69f723566054cbc5e8b9f31fc1c178f63f4
3
  size 3694592
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:70dcc7a7bcdef8b56ea3937b79ec869b6337a7ec67835e67354a431ba1c56b4d
3
  size 3694592
store/{src/osb-docs → osb-docs}/osb-docs-single-page-url-map.json RENAMED
File without changes
store/{src/osb-docs → osb-docs}/osb-docs-single-page.md RENAMED
File without changes
store/src/osb-data/mds/.klea-cache/doi-cache.json DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:f0c37b11e61ce5ec61d44ef7cd7a5ab0e6b8a293b712b46c27beb49e14d02c06
3
- size 161698
 
 
 
 
store/src/osb-data/mds/.klea-cache/metadata-map.template.json DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:ac3314e6c4c6d9faf959745a23e71ee975960a16dfb3828e9bd1ebec19a9fedd
3
- size 2086057
 
 
 
 
store/src/osb-data/mds/.klea-cache/osb-repositories.manifest.json DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:b6855238617a00c3a070a4b7efa1e8b55a4f641dc4da2e9e6743214504927b95
3
- size 584196
 
 
 
 
store/src/osb-data/mds/.klea-cache/xxh64_00605ccb10f8072c.pkl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:78b8d8e59a0d272cc0daa49e9e084a8090ec1700cbef31100c232c1ce333106f
3
- size 1296
 
 
 
 
store/src/osb-data/mds/.klea-cache/xxh64_0061771528498b74.pkl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:c27d2ade8bf961e7964f3c53c2cee5e43f00b1cb2ce73863275480a2670ebae5
3
- size 1469
 
 
 
 
store/src/osb-data/mds/.klea-cache/xxh64_0063d5c288597021.pkl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:5b652adf6aa43dec76cef40ec55b87d005adb5f1878843a9d1f46f3c2c635633
3
- size 3488
 
 
 
 
store/src/osb-data/mds/.klea-cache/xxh64_006bb9972f79917d.pkl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:c4e4b38714df57a4a3da1b8987752dbca93cda2f9b8ae3897707ccbcb8783234
3
- size 4579
 
 
 
 
store/src/osb-data/mds/.klea-cache/xxh64_006ed9a7472df23a.pkl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:9d171ee85dfb225539e023cd536160c7be27ac196905ceb855bad84c5bf8eb97
3
- size 1346
 
 
 
 
store/src/osb-data/mds/.klea-cache/xxh64_00732c600078d475.pkl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:3eea6f9c361945b4ba88fcdef830329645f7355cbd8bc1c8d2d36b722574ef8e
3
- size 1413
 
 
 
 
store/src/osb-data/mds/.klea-cache/xxh64_00a5d7fc2189b2b3.pkl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:7342b6b0c44e40d8439190a78bfb539163c8a426158b826362fa040a7372b4a5
3
- size 3367
 
 
 
 
store/src/osb-data/mds/.klea-cache/xxh64_00a7803c9e534daa.pkl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:7945bb7832dc96430ed4c80511ff486a8578588155bcd631bf2ab2ea79cd9b1b
3
- size 2198
 
 
 
 
store/src/osb-data/mds/.klea-cache/xxh64_00b40dd3efee9dd2.pkl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:4ca5ceba62fdbafba5ba2592a815c8a512aa51956bcbf8983ba22aec0174deb6
3
- size 1410
 
 
 
 
store/src/osb-data/mds/.klea-cache/xxh64_00be15f33c186916.pkl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:d1079b47fea76e3f1848e1400d2f5f14dc857ba0182d42858480bb156abaa44e
3
- size 1210
 
 
 
 
store/src/osb-data/mds/.klea-cache/xxh64_00c990bcb55edab0.pkl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:3010fa8f71ed2e7e2c1d6565eaa9ce08fab371b630ec0d90c3a4d997a11c1250
3
- size 1370
 
 
 
 
store/src/osb-data/mds/.klea-cache/xxh64_00cf0f2dac51a74e.pkl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:5877dcf8cab24f9636985834a1f72e315ec8f1b8e9b0bd01b2db09339bec4d88
3
- size 1360
 
 
 
 
store/src/osb-data/mds/.klea-cache/xxh64_00d77a9a57bcee32.pkl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:2266a1c1a479668537f22d87bd0f48c2b2268ef65019696315eeebbc8a33f165
3
- size 645
 
 
 
 
store/src/osb-data/mds/.klea-cache/xxh64_00f6bdeff09f187b.pkl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:b8046f4e4fb685464bb265576045cf313cbcd31513c03e76377f9b1404881c27
3
- size 994
 
 
 
 
store/src/osb-data/mds/.klea-cache/xxh64_0103811ef96a5123.pkl DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:36074326ffd98ec6936c05fcb6a4ea2d1e885e6ba5387855ab1a1ecea1a955ff
3
- size 1357