build_mix: no stray print inside BUILD_JSON block (E-019); manifest gains n_shards
Browse files- build/build_mix.py +5 -3
build/build_mix.py
CHANGED
|
@@ -598,6 +598,7 @@ def main():
|
|
| 598 |
for k, v in st.d["staged"].items()},
|
| 599 |
"staged_tokens": st.staged_tokens_total(),
|
| 600 |
"shards": st.d["merged_shards"],
|
|
|
|
| 601 |
"val_shards": st.d.get("val_shards", []),
|
| 602 |
"total_tokens": sum(s["tokens"] for s in st.d["merged_shards"]),
|
| 603 |
"val_tokens": sum(s["tokens"] for s in st.d.get("val_shards", [])),
|
|
@@ -609,11 +610,12 @@ def main():
|
|
| 609 |
}
|
| 610 |
with open(os.path.join(args.root, "manifest.json"), "w") as f:
|
| 611 |
json.dump(manifest, f, indent=1)
|
|
|
|
|
|
|
| 612 |
print("BUILD_JSON_BEGIN")
|
| 613 |
print(json.dumps({k: manifest[k] for k in
|
| 614 |
-
("staged_tokens", "total_tokens", "
|
| 615 |
-
"seconds")}, indent=1))
|
| 616 |
-
print("shards=", len(manifest["shards"]), flush=True)
|
| 617 |
print("BUILD_JSON_END")
|
| 618 |
|
| 619 |
|
|
|
|
| 598 |
for k, v in st.d["staged"].items()},
|
| 599 |
"staged_tokens": st.staged_tokens_total(),
|
| 600 |
"shards": st.d["merged_shards"],
|
| 601 |
+
"n_shards": len(st.d["merged_shards"]),
|
| 602 |
"val_shards": st.d.get("val_shards", []),
|
| 603 |
"total_tokens": sum(s["tokens"] for s in st.d["merged_shards"]),
|
| 604 |
"val_tokens": sum(s["tokens"] for s in st.d.get("val_shards", [])),
|
|
|
|
| 610 |
}
|
| 611 |
with open(os.path.join(args.root, "manifest.json"), "w") as f:
|
| 612 |
json.dump(manifest, f, indent=1)
|
| 613 |
+
# Nothing may be printed between the JSON and its END marker: the caller parses this span with
|
| 614 |
+
# json.loads, and a stray status line turned a passing rehearsal red (E-019).
|
| 615 |
print("BUILD_JSON_BEGIN")
|
| 616 |
print(json.dumps({k: manifest[k] for k in
|
| 617 |
+
("staged_tokens", "total_tokens", "val_tokens", "n_shards",
|
| 618 |
+
"distinct_sources_per_shard_min", "seconds")}, indent=1))
|
|
|
|
| 619 |
print("BUILD_JSON_END")
|
| 620 |
|
| 621 |
|