File size: 3,016 Bytes
17a55cc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | """Second pass: domain- and region-based sections for the weakest classes (health, tech, cities, science, education)."""
import json, sys, collections, urllib.parse
from datasets import load_dataset
BLOCK = ("folha", "uol.com.br")
DOMAINS = {
"saude": ("minhavida.com.br", "tuasaude.com", "saude.abril.com.br", "saude.ig.com.br", "drauziovarella", "vidaesaude",
"boasaude.com.br", "saudebemestar", "medicina", "msdmanuals.com/pt", "bemestar"),
"tecnologia": ("tecmundo.com.br", "olhardigital.com.br", "canaltech.com.br", "tecnoblog.net", "showmetech.com.br",
"tudocelular.com", "techtudo.com.br", "meiobit.com", "b9.com.br"),
"ciencia": ("revistagalileu.globo.com", "superinteressante", "super.abril.com.br", "hypescience.com", "revistapesquisa.fapesp.br",
"jornal.usp.br", "agencia.fapesp.br"),
"educacao": ("guiadoestudante.abril.com.br", "educacao.globo.com", "novaescola.org.br", "brasilescola.uol", "porvir.org"),
}
UFS = {"ac","al","ap","am","ba","ce","df","es","go","ma","mt","ms","mg","pa","pb","pr","pe","pi","rj","rn","rs","ro","rr","sc","sp","se","to",
"sao-paulo","rio-de-janeiro","minas-gerais","bahia","pernambuco","parana","rio-grande-do-sul","distrito-federal","ceara","goias"}
def section(url):
u = urllib.parse.urlparse(url); host = u.netloc.lower(); full = host + u.path.lower()
if any(b in host for b in BLOCK):
return None
for s, ds in DOMAINS.items():
if any(d in full for d in ds):
return s
segs = [x for x in u.path.lower().split("/") if x]
if host.endswith("g1.globo.com") and segs and segs[0] in UFS and "politica" not in segs and "economia" not in segs:
return "cidades"
if host in ("oglobo.globo.com", "extra.globo.com") and segs and segs[0] in ("rio", "brasil", "sao-paulo"):
return "cidades" if segs[0] != "brasil" else None
return None
target, out = int(sys.argv[1]), sys.argv[2]
skip = set()
for l in open(sys.argv[3], encoding="utf-8"):
skip.add(json.loads(l)["url"])
ds = load_dataset("HuggingFaceFW/fineweb-2", "por_Latn", split="train", streaming=True)
c = collections.Counter(); seen = kept = 0; cap = target // 5
with open(out, "w", encoding="utf-8") as f:
for r in ds:
seen += 1
s = section(r["url"])
if s and r["url"] not in skip and len(r["text"]) > 500 and c[s] < cap:
lines = [l for l in r["text"].split("\n") if l.strip()]
title = lines[0][:200] if lines and len(lines[0]) < 200 else ""
body = "\n".join(lines[1:] if title else lines)[:4000]
f.write(json.dumps({"url": r["url"], "section": s, "title": title, "text": body}, ensure_ascii=False) + "\n")
c[s] += 1; kept += 1
if seen % 500000 == 0:
print(json.dumps({"seen": seen, "kept": kept, "by": dict(c)}), flush=True)
if kept >= target or seen >= 15_000_000:
break
print(json.dumps({"seen": seen, "kept": kept, "by": dict(c)}), flush=True)
|