p0d v4: worker receives its chunk filename; note rayon non-scaling

probes/p0d_ingest_throughput.py

@@ -92,10 +92,14 @@ if len(sys.argv) > 1 and sys.argv[1] == "tokworker":
 
     tk = Tokenizer.from_file(os.environ["TOK_JSON"])
     tk.no_truncation()
-    with open("/dev/shm/" + sys.argv[2]) as f:
+    chunk_file = sys.argv[2] if len(sys.argv) > 2 else "p0d_chunks.json"
+    with open("/dev/shm/" + chunk_file) as f:
         chunks = json.load(f)
     t = time.monotonic()
-    enc = tk.encode_batch(chunks)          # rayon-parallel across cores, released GIL
+    enc = tk.encode_batch(chunks)
+    # Locally (same tokenizers 0.22.2) RAYON_NUM_THREADS did not move the needle at all: 198k tok/s
+    # at 1 thread vs 199k at 4. So `encode_batch` may not be core-parallel after all, which is why
+    # this probe re-execs a fresh process per setting instead of trusting an env var mid-process.
     el = time.monotonic() - t
     n = sum(len(e.ids) for e in enc)
     print("TOKRESULT " + json.dumps({"tokens": n, "seconds": round(el, 2),