JSONL fallback for datasets without Json feature type

train_securecoder.py

@@ -399,6 +399,66 @@ def _messages_from_any(row: dict, kind: str) -> tuple[list[dict], list[dict]]:
 # --------------------------------------------------------------------------
 # Loading, rendering, dataset construction
 # --------------------------------------------------------------------------
+def _read_jsonl(path: str, limit: int) -> list[dict]:
+    """Parse a JSON-lines file, stopping at ``limit`` good rows."""
+    rows: list[dict] = []
+    with open(path, "r", encoding="utf-8") as fh:
+        for line in fh:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                parsed = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if isinstance(parsed, dict):
+                rows.append(parsed)
+            if len(rows) >= limit:
+                break
+    return rows
+
+
+def _jsonl_fallback(src: Source, token: str | None, limit: int) -> list[dict]:
+    """Read ``data/*.jsonl`` directly when ``load_dataset`` cannot.
+
+    Some repos ship only JSON-lines and no parquet. ``datasets`` then infers a
+    schema from the JSON, any nested object becomes a ``Json`` feature, and
+    ``datasets`` <4.4 - which unsloth pins via ``datasets>=3.4.1,<4.4.0`` -
+    has no such feature type, so ``load_dataset`` raises before reading a row.
+    Pulling the file and parsing it ourselves skips schema inference entirely.
+    """
+    from huggingface_hub import hf_hub_download, list_repo_files
+
+    try:
+        files = list_repo_files(src.repo, repo_type="dataset", token=token)
+    except Exception as exc:  # noqa: BLE001 - caller reports the original error
+        log.debug("  %s: list_repo_files failed (%s)", source_name(src), exc)
+        return []
+
+    names = [f for f in files if f.endswith(".jsonl")]
+    preferred = [
+        f"data/{src.split}.jsonl",
+        f"{src.split}.jsonl",
+        "data/train.jsonl",
+        "train.jsonl",
+    ]
+    ordered = [f for f in preferred if f in names]
+    ordered += [f for f in sorted(names) if f not in ordered]
+
+    for name in ordered:
+        try:
+            path = hf_hub_download(src.repo, name, repo_type="dataset", token=token)
+            rows = _read_jsonl(path, limit)
+        except Exception as exc:  # noqa: BLE001 - try the next candidate
+            log.debug("  %s: %s failed (%s)", source_name(src), name, exc)
+            continue
+        if rows:
+            log.info("  %s: jsonl fallback read %d rows from %s",
+                     source_name(src), len(rows), name)
+            return rows
+    return []
+
+
 def load_source(src: Source, token: str | None, progress: bool = False) -> list[dict]:
     """Pull up to ``limit`` rows from one Hub dataset, streaming so we never
     download more than we need.
@@ -448,6 +508,13 @@ def load_source(src: Source, token: str | None, progress: bool = False) -> list[
         except Exception as exc:  # noqa: BLE001 - try the next candidate
             last_exc = exc
 
+    # No config/split worked. If the repo is JSON-lines, `datasets` may simply
+    # be unable to express its schema (see `_jsonl_fallback`), so read the file
+    # directly instead of giving up on the source.
+    rows = _jsonl_fallback(src, token, src.limit)
+    if rows:
+        return rows
+
     raise last_exc if last_exc else RuntimeError(f"could not load {src.repo}")