File size: 3,988 Bytes
17a55cc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | """Portuguese news pages from FineWeb-2 with the newsroom section read from the URL (no Folha/UOL)."""
import json, re, sys, collections, urllib.parse
from datasets import load_dataset
OUTLETS = ("g1.globo.com", "oglobo.globo.com", "globoesporte.globo.com", "ge.globo.com", "estadao.com.br", "r7.com",
"terra.com.br", "correiobraziliense.com.br", "em.com.br", "gazetadopovo.com.br", "veja.abril.com.br",
"exame.abril.com.br", "exame.com", "istoe.com.br", "cartacapital.com.br", "metropoles.com", "poder360.com.br",
"cnnbrasil.com.br", "agenciabrasil.ebc.com.br", "zh.clicrbs.com.br", "gauchazh.clicrbs.com.br",
"jornaldocommercio", "diariodepernambuco.com.br", "opovo.com.br", "otempo.com.br", "correio24horas.com.br",
"acritica.com", "valor.globo.com", "epoca.globo.com", "extra.globo.com", "odia.ig.com.br", "ig.com.br",
"noticias.r7.com", "bbc.com", "dw.com", "publico.pt", "dn.pt", "jn.pt", "expresso.pt", "observador.pt", "sapo.pt",
"rtp.pt", "cmjornal.pt")
BLOCK = ("folha", "uol.com.br")
SECTIONS = {
"politica": ("politica", "poder", "eleicoes", "eleicoes-2014", "eleicao", "politics"),
"economia": ("economia", "negocios", "mercado", "mercados", "financas", "seu-dinheiro", "dinheiro", "agronegocio", "economy", "business"),
"mundo": ("mundo", "internacional", "world", "america-latina"),
"esporte": ("esporte", "esportes", "futebol", "desporto", "globoesporte", "ge", "sport", "sports"),
"cidades": ("cidades", "cotidiano", "gerais", "metropole", "sociedade", "pais", "local"),
"policia": ("policia", "seguranca", "crime", "justica"),
"cultura": ("cultura", "pop-arte", "diversao", "diversao-e-arte", "entretenimento", "cinema", "musica", "televisao",
"famosos", "celebridades", "artes", "pop", "gente", "tv"),
"tecnologia": ("tecnologia", "tec", "tech", "link", "games", "informatica"),
"saude": ("saude", "bem-estar", "bem-viver", "vida-e-estilo-saude", "medicina"),
"ciencia": ("ciencia", "science", "espaco"),
"educacao": ("educacao", "vestibular", "enem", "ensino"),
"meio_ambiente": ("natureza", "meio-ambiente", "sustentabilidade", "ambiente", "clima"),
"turismo": ("turismo", "viagem", "viagens"),
"carros": ("carros", "autos", "automoveis", "autoesporte", "jornal-do-carro"),
"comida": ("gastronomia", "receitas", "comida", "paladar", "comer-e-beber"),
}
KEY2SEC = {k: s for s, ks in SECTIONS.items() for k in ks}
def section(url):
u = urllib.parse.urlparse(url)
host = u.netloc.lower()
if any(b in host for b in BLOCK) or not any(o in host for o in OUTLETS):
return None
sub = host.split(".")[0]
if host.startswith(("globoesporte.", "ge.")):
return "esporte"
if sub in KEY2SEC and sub not in ("www",):
return KEY2SEC[sub]
segs = [s for s in u.path.lower().split("/") if s][:3]
for s in segs:
if s in KEY2SEC:
return KEY2SEC[s]
return None
target, out = int(sys.argv[1]), sys.argv[2]
ds = load_dataset("HuggingFaceFW/fineweb-2", "por_Latn", split="train", streaming=True)
c = collections.Counter(); seen = 0; kept = 0
per_cap = target // 8
with open(out, "w", encoding="utf-8") as f:
for r in ds:
seen += 1
s = section(r["url"])
if s and len(r["text"]) > 500 and c[s] < per_cap:
lines = [l for l in r["text"].split("\n") if l.strip()]
title = lines[0][:200] if lines and len(lines[0]) < 200 else ""
body = "\n".join(lines[1:] if title else lines)[:4000]
f.write(json.dumps({"url": r["url"], "section": s, "title": title, "text": body}, ensure_ascii=False) + "\n")
c[s] += 1; kept += 1
if seen % 200000 == 0:
print(json.dumps({"seen": seen, "kept": kept, "by": dict(c)}), flush=True)
if kept >= target or seen >= 12_000_000:
break
print(json.dumps({"seen": seen, "kept": kept, "by": dict(c)}), flush=True)
|