Download training/kernels/preprocess.py from geraldaton20/alter_ego: direct link, hf CLI and curl.
- Browser
- Download file 24 kB
-
https://huggingface.co/geraldaton20/alter_ego/resolve/main/training/kernels/preprocess.py
- Command line
-
hf download hf://geraldaton20/alter_ego/training/kernels/preprocess.py
-
curl -L -o preprocess.py https://huggingface.co/geraldaton20/alter_ego/resolve/main/training/kernels/preprocess.py
24 kB
| """Stage 3 preprocess kernel v2: streaming mix -> dedup -> perplexity filter -> | |
| quality top-80% -> BPE-8192 tokenize -> packed numpy corpus -> Kaggle dataset | |
| ae-corpus-v1 (private). Self-contained. Resumable via HF-mirrored manifest. | |
| v2 fixes (2026-08-24, from first-run log): | |
| - ultrachat split train_sft (train doesn't exist) | |
| - synthcot source swapped to Open-Orca/SlimOrca (open-phi/synthetic-cot DNE) | |
| - stack_py: HF token passed to load_dataset (gated repo); still requires the | |
| account to have accepted bigcode terms once on the Hub - else auto-dropped | |
| - storage: zarr-v3 removed Group.create_dataset -> replaced with dependency- | |
| free append-only tokens.i16 + atomic arrays.npz | |
| Outputs: | |
| /kaggle/work/corpus/ tokens.i16, arrays.npz {lengths,scores,src}, | |
| tokenizer.json, corpus_meta.json | |
| HF <ckpt_repo> preprocess/manifest.jsonl, preprocess/tokenizer.json, | |
| telemetry/preprocess_stats.json, preprocess/DONE | |
| Secret aliases required: HF_TOKEN, KAGGLE_USERNAME, KAGGLE_API_TOKEN. | |
| """ | |
| import hashlib | |
| import json | |
| import os | |
| import subprocess | |
| import sys | |
| import time | |
| import numpy as np | |
| REPO = os.environ.get("HF_CKPT_REPO", "geraldaton20/student-ckpts-v1") | |
| OUT = os.environ.get("AE_OUT", "/kaggle/work/corpus") | |
| LOCAL = os.environ.get("AE_LOCAL", "0") == "1" | |
| PUBLISH = os.environ.get("AE_PUBLISH", "" ) != "0" | |
| HF_PREFIX = "preprocess" | |
| TARGET_TOKENS = int(os.environ.get("TARGET_TOKENS", "850000000")) | |
| CHUNK_DOCS = int(os.environ.get("CHUNK_DOCS", "20000")) | |
| TOKENIZER_SAMPLE_DOCS = int(os.environ.get("TOKENIZER_SAMPLE_DOCS", "60000")) | |
| VOCAB = 8192 | |
| MIN_LEN, MAX_LEN = 50, 256 | |
| PPL_MIN, PPL_MAX = 5.0, 50.0 | |
| PROBE_MODEL = "gpt2" | |
| QUALITY_KEEP_FRAC = 0.80 | |
| MAX_MINUTES = int(os.environ.get("MAX_MINUTES", "360")) | |
| SEED = 42 | |
| SOURCES = [ | |
| {"name": "cosmopedia", "repo": "HuggingFaceTB/cosmopedia", | |
| "config": "stanford", "ratio": 0.40, "split": "train"}, | |
| {"name": "stack_py", "repo": "bigcode/the-stack-v2", | |
| "config": None, "ratio": 0.30, "field": "content", "split": "train", | |
| "gated": True, | |
| # non-gated fallback so the pipeline never blocks on token scopes: | |
| "alt": {"repo": "code_search_net", "config": "python", | |
| "ratio": 0.30, "field": "func_code_string", "split": "train"}}, | |
| {"name": "ultrachat", "repo": "openbmb/UltraChat", | |
| "config": None, "ratio": 0.20, "field": "data", "split": "train"}, | |
| {"name": "synthcot", "repo": "casperhansen/gsm8k_synthetic_cot", | |
| "config": None, "ratio": 0.10, "split": "train", | |
| "join_fields": ["question", "cot", "answer"]}, | |
| ] | |
| # SPEC CORRECTIONS 2026-08-24: open-phi/synthetic-cot does not exist -> | |
| # casperhansen/gsm8k_synthetic_cot (user-selected); ultrachat moved to the | |
| # original openbmb/UltraChat (single 'train' split, 'data' turn list). | |
| def install_deps(): | |
| missing = [] | |
| for m in ("datasets", "transformers", "tokenizers", "huggingface_hub"): | |
| try: | |
| __import__(m) | |
| except Exception: | |
| missing.append(m) | |
| if missing: | |
| subprocess.run([sys.executable, "-m", "pip", "install", "-q", *missing], check=True) | |
| print("[bootstrap] deps ok", flush=True) | |
| def hf_token(): | |
| tok = os.environ.get("HF_TOKEN") | |
| if tok: | |
| return tok | |
| try: | |
| from kaggle_secrets import UserSecretsClient | |
| return UserSecretsClient().get_secret("HF_TOKEN") | |
| except Exception: | |
| return None | |
| class HF: | |
| def __init__(self): | |
| self.token = hf_token() | |
| from huggingface_hub import HfApi | |
| self.api = HfApi(token=self.token) if self.token else None | |
| def push_file(self, local, path_in_repo): | |
| if not self.api: | |
| return | |
| last = None | |
| for i in range(4): | |
| try: | |
| self.api.upload_file(path_or_fileobj=local, path_in_repo=path_in_repo, | |
| repo_id=REPO, repo_type="model") | |
| return | |
| except Exception as e: | |
| last = e | |
| time.sleep(2 * (i + 1)) | |
| print(f"[hf] push {path_in_repo} failed: {last}", flush=True) | |
| def download(self, path_in_repo, local_dir): | |
| if not self.api: | |
| return None | |
| try: | |
| return self.api.hf_hub_download(repo_id=REPO, filename=path_in_repo, | |
| repo_type="model", local_dir=local_dir) | |
| except Exception: | |
| return None | |
| def sha_bytes(b): | |
| return hashlib.sha256(b).hexdigest()[:16] | |
| def extract_text(ex): | |
| src = ex.get("__src", "") | |
| spec = next((s for s in SOURCES if s["name"] == src), {}) | |
| if spec.get("join_fields"): | |
| parts = [] | |
| for f in spec["join_fields"]: | |
| v = ex.get(f) | |
| if isinstance(v, list): | |
| parts.extend(x for x in v if isinstance(x, str)) | |
| elif isinstance(v, str): | |
| parts.append(v) | |
| if parts: | |
| return src, "\n".join(parts) | |
| f = spec.get("field") | |
| if f and f in ex: | |
| v = ex[f] | |
| if isinstance(v, str): | |
| return src, v | |
| if isinstance(v, list): | |
| out = [] | |
| for m in v: | |
| if isinstance(m, str): | |
| out.append(m) | |
| elif isinstance(m, dict): | |
| for k in ("content", "text", "value"): | |
| if isinstance(m.get(k), str): | |
| out.append(m[k]) | |
| break | |
| return src, "\n".join(out) | |
| for k in ("text", "content", "question", "instruction", "prompt"): | |
| v = ex.get(k) | |
| if isinstance(v, str) and v.strip(): | |
| return src, v | |
| return src, "" | |
| def build_stream(): | |
| """Load every source (with optional non-gated fallbacks) and interleave.""" | |
| from datasets import interleave_datasets, load_dataset | |
| def try_one(spec): | |
| kw = dict(path=spec["repo"], split=spec["split"], streaming=True, | |
| token=hf_token()) | |
| if spec.get("config"): | |
| kw["name"] = spec["config"] | |
| return load_dataset(**kw) | |
| sets, probs, dropped = [], [], [] | |
| for spec in SOURCES: | |
| candidates = [spec] + ([spec["alt"]] if spec.get("alt") else []) | |
| loaded = None | |
| for cand in candidates: | |
| try: | |
| ds = try_one(cand) | |
| loaded = (ds, cand) | |
| tag = "" if cand is spec else " [fallback]" | |
| print(f"[stream] {spec['name']}{tag} OK ({cand['ratio']:.0%})", | |
| flush=True) | |
| break | |
| except Exception as e: | |
| hint = "" | |
| if cand.get("gated") and "gated" in str(e).lower(): | |
| hint = (" (gated: token must cover this repo - a CLASSIC " | |
| "token carries account grants; fine-grained needs " | |
| f"the repo explicitly selected)") | |
| print(f"[stream] try {cand['repo']} FAILED ({e}){hint}", flush=True) | |
| if loaded is None: | |
| dropped.append(spec["name"]) | |
| continue | |
| ds, cand = loaded | |
| # merge the winning candidate's extraction hints onto the source spec | |
| for k in ("field", "join_fields", "config"): | |
| if k in cand: | |
| spec[k] = cand[k] | |
| name = spec["name"] | |
| ds = ds.map(lambda ex, _n=name: {"__src": _n}) | |
| sets.append(ds) | |
| probs.append(spec["ratio"]) | |
| if not sets: | |
| raise SystemExit("no sources loaded") | |
| tot = sum(probs) | |
| probs = [p / tot for p in probs] | |
| mixed = interleave_datasets(sets, probabilities=probs, seed=SEED) | |
| return mixed, dropped | |
| class ExactDedup: | |
| def __init__(self, cap=20_000_000): | |
| self.seen = set() | |
| self.cap = cap | |
| def check(self, text): | |
| h = int.from_bytes(hashlib.blake2b( | |
| " ".join(text.lower().split())[:4000].encode(), | |
| digest_size=8).digest(), "little") | |
| if h in self.seen: | |
| return False | |
| if len(self.seen) < self.cap: | |
| self.seen.add(h) | |
| return True | |
| def near_dup_filter(items): | |
| """Per-chunk MinHash LSH near-dup removal (chunk-scoped, idempotent).""" | |
| try: | |
| from datasketch import MinHash, MinHashLSH | |
| except Exception: | |
| return items | |
| lsh = MinHashLSH(threshold=0.85, num_perm=128) | |
| out = [] | |
| for j, it in enumerate(items): | |
| words = it["text"].split() | |
| m = MinHash(num_perm=128) | |
| n = 0 | |
| for s in range(0, max(1, len(words) - 4), 3): | |
| m.update(" ".join(words[s:s + 5]).encode("utf8")) | |
| n += 1 | |
| if n >= 200: | |
| break | |
| if n == 0: | |
| out.append(it) | |
| continue | |
| dup = False | |
| try: | |
| for _key in lsh.query(m): | |
| dup = True | |
| break | |
| except Exception: | |
| pass | |
| if not dup: | |
| try: | |
| lsh.insert(str(j), m) | |
| except Exception: | |
| pass | |
| out.append(it) | |
| return out | |
| def quality_score(text): | |
| score = 0.0 | |
| if any(kw in text for kw in ("def ", "class ", "import ", "function")): | |
| score += 0.3 | |
| if any(marker in text for marker in ("Therefore,", "Step 1:", "First,", "Because")): | |
| score += 0.2 | |
| toks = len(text.split()) | |
| if toks < 30: | |
| score -= 0.3 | |
| elif toks > 200: | |
| score += 0.1 | |
| uniq = len(set(text.split())) / max(1, len(text.split())) | |
| return max(0.0, min(1.0, score + uniq * 0.2)) | |
| class PplProbe: | |
| def __init__(self): | |
| import torch | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| self.torch = torch | |
| self.dev = "cuda" if torch.cuda.is_available() else "cpu" | |
| self.tk = AutoTokenizer.from_pretrained(PROBE_MODEL) | |
| if self.tk.pad_token_id is None: | |
| self.tk.pad_token = self.tk.eos_token | |
| dt = torch.float16 if self.dev == "cuda" else torch.float32 | |
| self.model = AutoModelForCausalLM.from_pretrained( | |
| PROBE_MODEL, torch_dtype=dt).to(self.dev).eval() | |
| def ppl_batch(self, texts, batch=32): | |
| torch = self.torch | |
| out = [] | |
| with torch.no_grad(): | |
| for s in range(0, len(texts), batch): | |
| part = [t[:1500] for t in texts[s:s + batch]] | |
| enc = self.tk(part, return_tensors="pt", padding=True, | |
| truncation=True, max_length=256).to(self.dev) | |
| logits = self.model(**enc).logits.float() | |
| labels = enc["input_ids"][:, 1:] | |
| mask = enc["attention_mask"][:, 1:].float() | |
| nll = torch.nn.functional.cross_entropy( | |
| logits[:, :-1].reshape(-1, logits.size(-1)), | |
| labels.reshape(-1), reduction="none").view(labels.shape) | |
| nll = (nll * mask).sum(1) / mask.sum(1).clamp(min=1) | |
| out.extend(torch.exp(nll).tolist()) | |
| return out | |
| class Corpus: | |
| """Dependency-free corpus store: tokens.i16 (append-only, int16 LE) + | |
| arrays.npz {lengths i4, scores f2, src u1} rewritten atomically per flush.""" | |
| def __init__(self, directory): | |
| self.dir = directory | |
| os.makedirs(directory, exist_ok=True) | |
| self.tok_path = os.path.join(directory, "tokens.i16") | |
| self.npz_path = os.path.join(directory, "arrays.npz") | |
| if os.path.exists(self.npz_path): | |
| z = np.load(self.npz_path) | |
| self.lengths = z["lengths"].astype("i4") | |
| self.scores = z["scores"].astype("f2") | |
| self.src = z["src"].astype("u1") | |
| else: | |
| self.lengths = np.zeros(0, dtype="i4") | |
| self.scores = np.zeros(0, dtype="f2") | |
| self.src = np.zeros(0, dtype="u1") | |
| def n_docs(self): | |
| return int(self.lengths.shape[0]) | |
| def total_tokens(self): | |
| size = os.path.getsize(self.tok_path) if os.path.exists(self.tok_path) else 0 | |
| return size // 2 | |
| def append(self, flat, lens, scores, src_ids): | |
| if not lens: | |
| return | |
| with open(self.tok_path, "ab") as f: | |
| f.write(np.asarray(flat, dtype="<i2").tobytes()) | |
| self.lengths = np.concatenate([self.lengths, np.asarray(lens, "i4")]) | |
| self.scores = np.concatenate([self.scores, np.asarray(scores, "f2")]) | |
| self.src = np.concatenate([self.src, np.asarray(src_ids, "u1")]) | |
| tmp = self.npz_path + ".tmp.npz" | |
| np.savez(tmp, lengths=self.lengths, scores=self.scores, src=self.src) | |
| os.replace(tmp, self.npz_path) | |
| SRC_IDS = {s["name"]: i for i, s in enumerate(SOURCES)} | |
| def main(): | |
| install_deps() | |
| import torch | |
| nm = "cpu" | |
| if torch.cuda.is_available(): | |
| nm = torch.cuda.get_device_name(0) | |
| if not LOCAL and ("Tesla T4" not in nm): | |
| raise SystemExit(f"HARDWARE GUARD FAIL: {nm}") | |
| if LOCAL: | |
| print(f"[bootstrap] LOCAL mode: device={nm} (guard bypassed)", flush=True) | |
| else: | |
| print(f"[bootstrap] hardware OK: {nm}", flush=True) | |
| hf = HF() | |
| os.makedirs(OUT, exist_ok=True) | |
| man_local = os.path.join(OUT, "manifest.jsonl") | |
| seen_docs, done_chunks, rows, cum_tokens = 0, set(), [], 0 | |
| dl = hf.download(f"{HF_PREFIX}/manifest.jsonl", "/kaggle/tmp/man") | |
| if dl and os.path.exists(dl): | |
| txt = open(dl).read() | |
| for line in txt.splitlines(): | |
| if line.strip(): | |
| r = json.loads(line) | |
| rows.append(r) | |
| done_chunks.add(r["chunk"]) | |
| cum_tokens += int(r["tokens_out"]) | |
| seen_docs = max(seen_docs, int(r["seen_end"])) | |
| open(man_local, "w").write(txt) | |
| print(f"[resume] chunks_done={len(done_chunks)} seen={seen_docs} " | |
| f"cum={cum_tokens:,}", flush=True) | |
| from tokenizers import Tokenizer, decoders, models, pre_tokenizers, trainers | |
| tok_path = os.path.join(OUT, "tokenizer.json") | |
| have_tok = os.path.exists(tok_path) | |
| if not have_tok: | |
| got = hf.download(f"{HF_PREFIX}/tokenizer.json", "/kaggle/tmp/tk") | |
| if got and os.path.exists(got): | |
| import shutil | |
| shutil.copyfile(got, tok_path) | |
| have_tok = True | |
| stream, dropped = build_stream() | |
| # breadcrumb IMMEDIATELY after streams resolve — if this lands on HF and | |
| # no heartbeat follows within ~6 min, the stall is between here and loop. | |
| json.dump({"stage": "preprocess", "phase": "streams-loaded", "ts": time.time()}, | |
| open(os.path.join(OUT, "hb.json"), "w")) | |
| hf.push_file(os.path.join(OUT, "hb.json"), "heartbeat_preprocess.json") | |
| t0 = time.time() | |
| last_hb = t0 | |
| if not have_tok: | |
| print(f"[tok] sampling {TOKENIZER_SAMPLE_DOCS} docs ...", flush=True) | |
| sample, taken = [], 0 | |
| for ex in stream: | |
| if taken >= TOKENIZER_SAMPLE_DOCS: | |
| break | |
| taken += 1 | |
| _, t = extract_text(ex) | |
| if len(t) >= 200: | |
| sample.append(t[:8000]) | |
| tk = Tokenizer(models.BPE(unk_token="<unk>")) | |
| tk.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) | |
| tk.decoder = decoders.ByteLevel() | |
| tk.train_from_iterator(sample, trainers.BpeTrainer( | |
| vocab_size=VOCAB, special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"], | |
| min_frequency=2)) | |
| tk.save(tok_path) | |
| hf.push_file(tok_path, f"{HF_PREFIX}/tokenizer.json") | |
| seen_docs += taken | |
| print(f"[tok] vocab={tk.get_vocab_size()} pushed; sampled={taken}", flush=True) | |
| else: | |
| tk = Tokenizer.from_file(tok_path) | |
| print(f"[tok] loaded existing ({tk.get_vocab_size()})", flush=True) | |
| corpus = Corpus(OUT) | |
| probe = PplProbe() | |
| exact = ExactDedup() | |
| def encode_batch(texts): | |
| enc = tk.encode_batch(texts, add_special_tokens=False) | |
| keep_idx, lens, flat = [], [], [] | |
| for j, e in enumerate(enc): | |
| ids = e.ids | |
| if MIN_LEN <= len(ids) <= MAX_LEN: | |
| keep_idx.append(j) | |
| lens.append(len(ids)) | |
| flat.extend(ids) | |
| return keep_idx, lens, flat | |
| cum_tokens = max(cum_tokens, corpus.total_tokens) | |
| chunk_idx = (max(done_chunks) + 1) if done_chunks else 0 | |
| if seen_docs: | |
| print(f"[resume] fast-forwarding past {seen_docs} docs ...", flush=True) | |
| skipped = 0 | |
| for _ex in stream: | |
| skipped += 1 | |
| if skipped >= seen_docs: | |
| break | |
| buf = [] | |
| def flush(): | |
| nonlocal buf, chunk_idx, cum_tokens | |
| if not buf: | |
| return | |
| items = near_dup_filter(buf) | |
| texts = [it["text"] for it in items] | |
| try: | |
| ppls = probe.ppl_batch(texts) | |
| except Exception as e: | |
| print(f"[ppl] batch failed ({e}); keeping chunk w/o ppl filter", flush=True) | |
| ppls = [(PPL_MIN + PPL_MAX) / 2] * len(texts) | |
| passed = [(it, float(p)) for it, p in zip(items, ppls) | |
| if PPL_MIN <= float(p) <= PPL_MAX] | |
| scored = sorted(((quality_score(it["text"]), p, it) for it, p in passed), | |
| key=lambda x: -x[0]) | |
| scored = scored[:max(1, int(len(scored) * QUALITY_KEEP_FRAC))] | |
| final_texts = ["<bos> " + it["text"] + " <eos>" for _, _, it in scored] | |
| # PACK (not discard): docs flow through one id stream separated by | |
| # <eos>; cut fixed 256-token windows so long docs contribute fully. | |
| enc = tk.encode_batch(final_texts, add_special_tokens=False) | |
| eos = tk.token_to_id("<eos>") or 0 | |
| stream: list[int] = [] | |
| bounds: list[tuple[int, float, int]] = [] # (end_excl, score, src) | |
| pos = 0 | |
| for e, (sc, _p, it) in zip(enc, scored): | |
| ids = e.ids | |
| n = len(ids) + 1 | |
| stream.extend(ids) | |
| stream.append(eos) | |
| pos += n | |
| bounds.append((pos, sc, SRC_IDS.get(it["src"], 255))) | |
| windows = [] | |
| cur = 0 | |
| while cur < len(stream): | |
| w = stream[cur:cur + MAX_LEN] | |
| cur += MAX_LEN | |
| if len(w) >= MIN_LEN: | |
| windows.append(w) | |
| def meta_at(start_idx): | |
| acc_end = 0 | |
| for end_excl, sc, src in bounds: | |
| acc_end = end_excl | |
| if start_idx < end_excl: | |
| return sc, src | |
| return (bounds[-1][1], bounds[-1][2]) if bounds else (0.0, 255) | |
| lens, flat, sc_out, src_out = [], [], [], [] | |
| for wi, w in enumerate(windows): | |
| start = wi * MAX_LEN | |
| sc, src = meta_at(start) | |
| lens.append(len(w)) | |
| flat.extend(w) | |
| sc_out.append(sc) | |
| src_out.append(src) | |
| # text sidecar: raw kept docs so ANY tokenizer/model track can consume later | |
| for _j, (_s, _p, it) in enumerate(scored): | |
| pass | |
| with open(os.path.join(OUT, "texts.jsonl"), "a", encoding="utf-8") as tf: | |
| for t in final_texts: | |
| tf.write(json.dumps({"text": t}, ensure_ascii=False) + "\n") | |
| row = {"chunk": chunk_idx, "docs_in": len(buf), "after_dedup": len(items), | |
| "after_ppl": len(passed), "docs_kept": len(enc), | |
| "windows": len(lens), "tokens_out": sum(lens), | |
| "seen_end": seen_docs, "ts": time.time()} | |
| if lens: | |
| corpus.append(flat, lens, sc_out, src_out) | |
| cum_tokens += sum(lens) | |
| row["cum_tokens"] = cum_tokens | |
| rows.append(row) | |
| done_chunks.add(chunk_idx) | |
| with open(man_local, "a") as f: | |
| f.write(json.dumps(row) + "\n") | |
| hf.push_file(man_local, f"{HF_PREFIX}/manifest.jsonl") | |
| print(f"[chunk {chunk_idx}] in={row['docs_in']} dedup={row['after_dedup']} " | |
| f"ppl={row['after_ppl']} wins={row['windows']} tok={row['tokens_out']} " | |
| f"cum={cum_tokens:,}", flush=True) | |
| buf = [] | |
| chunk_idx += 1 | |
| for ex in stream: | |
| seen_docs += 1 | |
| src, t = extract_text(ex) | |
| if t and len(t) >= 200 and exact.check(t): | |
| buf.append({"text": t, "src": src}) | |
| if len(buf) >= CHUNK_DOCS: | |
| flush() | |
| if corpus.total_tokens >= TARGET_TOKENS: | |
| print(f"[target] hit {TARGET_TOKENS:,}", flush=True) | |
| break | |
| now = time.time() | |
| if now - last_hb >= 300: | |
| hb = {"stage": "preprocess", "seen": seen_docs, | |
| "corpus_tokens": corpus.total_tokens, "corpus_docs": corpus.n_docs, | |
| "ts": now} | |
| p = os.path.join(OUT, "hb.json") | |
| json.dump(hb, open(p, "w")) | |
| hf.push_file(p, "heartbeat_preprocess.json") | |
| last_hb = now | |
| if MAX_MINUTES and (now - t0) >= MAX_MINUTES * 60: | |
| flush() | |
| print("[exit] MAX_MINUTES reached; graceful stop WITHOUT DONE", flush=True) | |
| return | |
| flush() | |
| exhausted = corpus.total_tokens < TARGET_TOKENS | |
| meta = {"vocab": VOCAB, "min_len": MIN_LEN, "max_len": MAX_LEN, | |
| "total_tokens": corpus.total_tokens, "docs": corpus.n_docs, | |
| "target_tokens": TARGET_TOKENS, "exhausted_stream": exhausted, | |
| "tokenizer_sha": sha_bytes(open(tok_path, "rb").read()), "seed": SEED} | |
| json.dump(meta, open(os.path.join(OUT, "corpus_meta.json"), "w"), indent=2) | |
| hf.push_file(os.path.join(OUT, "corpus_meta.json"), f"{HF_PREFIX}/corpus_meta.json") | |
| per_src = {} | |
| for sid in np.unique(corpus.src): | |
| nm_ = next((s["name"] for s, i in SRC_IDS.items() if i == sid), f"unk{sid}") | |
| mask = corpus.src == sid | |
| per_src[str(nm_)] = {"docs": int(mask.sum()), | |
| "tokens": int(corpus.lengths[mask].sum(dtype="i8"))} | |
| stats = {"final": meta, "per_source": per_src, "dropped_sources": dropped, | |
| "chunks": len(rows), "wall_hours": round((time.time() - t0) / 3600, 2)} | |
| sp = "/kaggle/work/preprocess_stats.json" | |
| json.dump(stats, open(sp, "w")) | |
| hf.push_file(sp, "telemetry/preprocess_stats.json") | |
| do_publish = PUBLISH and not LOCAL | |
| published = publish_kaggle_dataset(OUT, user_creds()) if do_publish else False | |
| if not published: | |
| print("[publish] Kaggle dataset NOT created (LOCAL mode or creds missing); corpus stays in kernel output.", flush=True) | |
| with open(os.path.join(OUT, "DONE"), "w") as f: | |
| f.write(json.dumps(meta)) | |
| hf.push_file(os.path.join(OUT, "DONE"), f"{HF_PREFIX}/DONE") | |
| print(f"[done] tokens={corpus.total_tokens:,} docs={corpus.n_docs} " | |
| f"published={published} exhausted={exhausted}", flush=True) | |
| def user_creds(): | |
| try: | |
| from kaggle_secrets import UserSecretsClient | |
| c = UserSecretsClient() | |
| return c.get_secret("KAGGLE_USERNAME"), c.get_secret("KAGGLE_API_TOKEN") | |
| except Exception: | |
| return None, None | |
| def publish_kaggle_dataset(folder, creds): | |
| user, key = creds | |
| if not (user and key): | |
| print("[publish] no Kaggle creds in secrets; skipping", flush=True) | |
| return False | |
| subprocess.run([sys.executable, "-m", "pip", "install", "-q", "kaggle"], check=True) | |
| env = dict(os.environ, KAGGLE_USERNAME=user, KAGGLE_KEY=key) | |
| json.dump({"title": "ae-corpus-v1", "id": f"{user}/ae-corpus-v1", | |
| "licenses": [{"name": "CC0-1.0"}]}, | |
| open(os.path.join(folder, "dataset-metadata.json"), "w")) | |
| for cmd, label in ( | |
| ([sys.executable, "-m", "kaggle", "datasets", "create", "-p", folder, | |
| "--dir-mode", "zip"], "create"), | |
| ([sys.executable, "-m", "kaggle", "datasets", "version", "-p", folder, | |
| "--dir-mode", "zip", "-m", "update"], "version"), | |
| ): | |
| r = subprocess.run(cmd, env=env, capture_output=True, text=True) | |
| tail = ((r.stdout or "") + (r.stderr or ""))[-300:] | |
| print(f"[publish] {label} rc={r.returncode} {tail}", flush=True) | |
| if r.returncode == 0: | |
| return True | |
| return False | |
| if __name__ == "__main__": | |
| main() | |