Download entrenamiento/fuentes.py from EagleDev-Innovation/Modelos-EagleDev-Innovation: direct link, hf CLI and curl.
- Browser
- Download file 11.3 kB
-
https://huggingface.co/EagleDev-Innovation/Modelos-EagleDev-Innovation/resolve/main/entrenamiento/fuentes.py
- Command line
-
hf download hf://EagleDev-Innovation/Modelos-EagleDev-Innovation/entrenamiento/fuentes.py
-
curl -L -o fuentes.py https://huggingface.co/EagleDev-Innovation/Modelos-EagleDev-Innovation/resolve/main/entrenamiento/fuentes.py
11.3 kB
| """Catálogo de datasets públicos usados para entrenar y evaluar. | |
| Solo se incluyen datasets cuya licencia permite uso comercial (MIT, CC0, CC BY 4.0...). | |
| La licencia se vuelve a comprobar al descargar (Roboflow, Kaggle) y la fuente se descarta | |
| si no es compatible. Para buscar más: `python -m entrenamiento.buscar_roboflow --todo`. | |
| Tres formas de aprovechar un dataset: | |
| - Material por carpetas (`clases`): carpeta/clase → material. Foto entera. | |
| - Material por recortes (`recortes`): datasets de detección donde cada recuadro es un | |
| objeto (botella, caja, lata...). Cada objeto se recorta y se usa como una foto de ese | |
| material; así un dataset de escenas con varios objetos da muchas fotos limpias. | |
| - Defectos (`material` fijo): cada imagen trae las clases de defecto que contiene. Las | |
| clases se traducen con mapeo.defectos_de_clase. Para los defectos que el dataset sí | |
| etiqueta, una imagen sin esa etiqueta cuenta como "no lo tiene"; los demás defectos | |
| quedan como "desconocidos" y no se entrenan con ella. | |
| Además, `piezas` marca los datasets cuyos recuadros son piezas contables: sirven para medir | |
| el conteo de SAM 3 (evaluar_conteo.py). | |
| """ | |
| from dataclasses import dataclass, field | |
| from typing import Literal | |
| Origen = Literal["hf", "url", "kaggle", "roboflow"] | |
| # Licencias aceptadas (en minúsculas, por subcadena). Se excluye todo lo "NC" (no | |
| # comercial), "other", "unknown" y las que reservan los derechos de las fotos. | |
| LICENCIAS_COMERCIALES = ( | |
| "mit", | |
| "cc0", | |
| "public domain", | |
| "cc by 4.0", | |
| "cc-by-4.0", | |
| "apache", | |
| "odbl", | |
| ) | |
| class Fuente: | |
| nombre: str | |
| origen: Origen | |
| ref: str # repo de HF, URL, slug de Kaggle o "workspace/proyecto" de Roboflow | |
| licencia: str | |
| web: str | |
| # Material por carpetas: carpeta/clase → código de material (None = descartar la clase). | |
| clases: dict[str, str | None] = field(default_factory=dict) | |
| # Material por recortes: expresión regular sobre el nombre de la clase → material. | |
| recortes: dict[str, str] = field(default_factory=dict) | |
| # Defectos: material de todas sus imágenes (también se usa para evaluar el estado). | |
| material: str | None = None | |
| usar_para_material: bool = True | |
| # Defectos que se pueden dar por ausentes en todo el dataset (fotos de objetos limpios). | |
| defectos_ausentes: tuple[str, ...] = () | |
| max_por_clase: int = 1500 | |
| archivo: str | None = None # archivo concreto dentro del repo de HF | |
| # Expresión regular de las clases que son piezas contables (para evaluar el conteo). | |
| piezas: str | None = None | |
| # False si el dataset solo sirve para evaluar el conteo (no para entrenar). | |
| entrenar: bool = True | |
| def es_de_defectos(self) -> bool: | |
| return not self.clases and not self.recortes | |
| # ------------------------------------------------------------------- material por carpetas | |
| TRASHNET = Fuente( | |
| nombre="trashnet", | |
| origen="hf", | |
| ref="garythung/trashnet", | |
| archivo="dataset-resized.zip", | |
| licencia="MIT", | |
| web="https://huggingface.co/datasets/garythung/trashnet", | |
| clases={ | |
| "cardboard": "carton", | |
| "glass": "vidrio", | |
| "metal": "metal", | |
| "paper": "papel", | |
| "plastic": "plastico", | |
| "trash": "otro", | |
| }, | |
| # Objetos limpios fotografiados sobre fondo blanco. | |
| defectos_ausentes=("oxido", "podredumbre"), | |
| ) | |
| REALWASTE = Fuente( | |
| nombre="realwaste", | |
| origen="url", | |
| ref="https://archive.ics.uci.edu/static/public/908/realwaste.zip", | |
| licencia="CC BY 4.0", | |
| web="https://archive.ics.uci.edu/dataset/908/realwaste", | |
| clases={ | |
| "cardboard": "carton", | |
| "food organics": None, | |
| "glass": "vidrio", | |
| "metal": "metal", | |
| "miscellaneous trash": "otro", | |
| "paper": "papel", | |
| "plastic": "plastico", | |
| "textile trash": "tela", | |
| "vegetation": None, | |
| }, | |
| ) | |
| # Nota: "mostafaabla/garbage-classification" (Kaggle) se descartó: su licencia ODbL cubre la | |
| # base de datos, pero las fotos son "© de sus autores" (recogidas de internet, sin fines de lucro). | |
| GARBAGE_V2 = Fuente( | |
| nombre="garbage-v2", | |
| origen="kaggle", | |
| ref="sumn2u/garbage-classification-v2", | |
| licencia="(se comprueba al descargar)", | |
| web="https://www.kaggle.com/datasets/sumn2u/garbage-classification-v2", | |
| clases={ | |
| "battery": "otro", | |
| "biological": None, | |
| "cardboard": "carton", | |
| "clothes": "tela", | |
| "glass": "vidrio", | |
| "metal": "metal", | |
| "paper": "papel", | |
| "plastic": "plastico", | |
| "shoes": "otro", | |
| "trash": "otro", | |
| }, | |
| max_por_clase=1200, | |
| ) | |
| def _rf(nombre: str, ref: str, material: str | None = None, **extra) -> Fuente: | |
| return Fuente( | |
| nombre=nombre, | |
| origen="roboflow", | |
| ref=ref, | |
| licencia="(se comprueba al descargar)", | |
| web=f"https://universe.roboflow.com/{ref}", | |
| material=material, | |
| **extra, | |
| ) | |
| # ------------------------------------------------------------------- material por recortes | |
| RECORTES = ( | |
| _rf( | |
| "rf-cajas-carton", | |
| "ece4191-xcxot/cardboard-box-detection-mxqjh", | |
| recortes={r"box": "carton"}, | |
| piezas=r"box", | |
| ), | |
| _rf( | |
| "rf-plastico-residuos", | |
| "edwin-daza-saavedra-s-workspace/plastic-waste-ag4eg", | |
| recortes={r"plastic": "plastico"}, | |
| piezas=r"plastic bottle|plastic cup", | |
| ), | |
| _rf( | |
| "rf-botellas-plastico", | |
| "fyp-li8zz/plastic-bottle-2.0", | |
| recortes={r"bottle": "plastico"}, | |
| piezas=r"bottle", | |
| ), | |
| _rf( | |
| "rf-frascos-vidrio", | |
| "gdp27/glass-jars", | |
| recortes={r"jar|glass": "vidrio"}, | |
| piezas=r"jar", | |
| ), | |
| _rf( | |
| "rf-vidrio-residuos", | |
| "waste-seregation/glass-detection-1jobo", | |
| recortes={r"glass": "vidrio"}, | |
| piezas=r"glass", | |
| ), | |
| _rf("rf-latas", "albaath2/can-o9wiu", recortes={r"metal": "metal"}, piezas=r"metal"), | |
| _rf("rf-tubos", "john-tai-fa0rl/pipe-detector3", recortes={r"pipe": "metal"}, piezas=r"pipe"), | |
| _rf( | |
| "rf-chatarra", | |
| "moses-orjyo/steel-scrap-sowop", | |
| recortes={r"scrap|steel|rebar": "metal"}, | |
| ), | |
| _rf("rf-papel", "trashdetection-bhjmn/paper-2jpbv", recortes={r"paper": "papel"}), | |
| _rf( | |
| "rf-papel-periodico", | |
| "mmt-kcsv2/paper-nitaa", | |
| recortes={r"^newspaper$|^paper$": "papel", r"cardboard": "carton"}, | |
| ), | |
| _rf( | |
| "rf-materiales-mixtos", | |
| "material-identification/cardboard-1hdgz", | |
| recortes={ | |
| r"^cardboard$": "carton", | |
| r"^metal$": "metal", | |
| r"^paper$": "papel", | |
| r"^plastic$": "plastico", | |
| r"^glass$": "vidrio", | |
| }, | |
| ), | |
| _rf( | |
| "rf-ropa", | |
| "grad-proj-nakmt/clothes-liagy", | |
| recortes={r"dress|outwear|shirt|shorts|trousers|skirt|vest|sling": "tela"}, | |
| piezas=r"dress|outwear|shirt|shorts|trousers|skirt|vest|sling", | |
| ), | |
| _rf( | |
| "rf-tablas-madera", | |
| "davids-workspace-rc8oh/lumber-board-ident", | |
| recortes={r"wood|lumber": "madera"}, | |
| piezas=r"wood|lumber", | |
| ), | |
| _rf( | |
| "rf-residuos-electronicos", | |
| "mehmet-wswds/e-waste-qmxtt", | |
| recortes={r"battery|keyboard|laptop|phone|monitor|mouse|printer|electronic": "otro"}, | |
| ), | |
| ) | |
| # ---------------------------------------------------------------------------- defectos | |
| # Madera: material (tablas, estibas, contrachapado) y defectos (grietas, nudos, manchas). | |
| MADERA = ( | |
| _rf( | |
| "rf-madera-tablas", | |
| "masood/wood-w4h97", | |
| "madera", | |
| piezas=r"plank|timber|plywood|pallet|wood", | |
| ), | |
| _rf("rf-madera-planchas", "rolos/wood_plank_final-69kb7", "madera"), | |
| _rf("rf-madera-estibas", "pallets-dzqdc/only-wood", "madera"), | |
| _rf("rf-madera-defectos-kodytek", "tim-4ijf0/wood-surface-defect-detection", "madera"), | |
| _rf("rf-madera-defectos-11", "alima-aife2/wood-defect-detection-dxzor", "madera"), | |
| _rf("rf-madera-defectos-5", "detect-detect/new-defects-in-wood", "madera"), | |
| _rf("rf-madera-defectos-10k", "ye-csex5/wood-defect-o06zb", "madera"), | |
| _rf( | |
| "rf-madera-defectos-tarumt", | |
| "tunku-abdul-rahman-university-of-management-and-technology-ttt1n/wood-defects-8lcdv", | |
| "madera", | |
| ), | |
| _rf("rf-madera-sana", "school-mys25/wood-7ao2p", "madera"), | |
| _rf("rf-madera-estibas-agujeros", "deep-learning-exxwi/wood-pallet-deep-learning", "madera"), | |
| ) | |
| # Metal: óxido; abolladuras, rayones y agujeros (carrocerías: solo para defectos). | |
| METAL = ( | |
| _rf("rf-oxido-1", "qp-55vgp/rust-2-qtreu", "metal"), | |
| _rf("rf-oxido-2", "personal-iisqb/rust-detection-xj8bu", "metal"), | |
| _rf("rf-oxido-3", "workspacesteijn/rust-hbbwg", "metal"), | |
| _rf("rf-oxido-niveles", "zenil/rust-detection-4cr1b", "metal"), | |
| _rf("rf-metal-superficie", "university-of-bristol-kxv0a/metal-surface-7ng08", "metal"), | |
| _rf( | |
| "rf-metal-superficie-2", | |
| "vision-oi1kw/metal-surface-defect-detection-n7x0l", | |
| "metal", | |
| ), | |
| _rf("rf-abolladuras-1", "mesors-workspace/dent-ggvqt", "metal", usar_para_material=False), | |
| _rf( | |
| "rf-abolladuras-2", | |
| "roboflow-demo/car-dents-and-scratches", | |
| "metal", | |
| usar_para_material=False, | |
| ), | |
| ) | |
| # Tela: agujeros, manchas, arrugas. | |
| TELA = ( | |
| _rf("rf-tela-defectos-1", "fabric-defect-detection-jq7gf/fabric-defect-detection-v2", "tela"), | |
| _rf("rf-tela-defectos-3", "datasetannot/fabric-defect-detection-lfyix", "tela"), | |
| _rf( | |
| "rf-tela-agujeros", "textile-defect-detection-plje9/textile-defect-detection-yvgmo", "tela" | |
| ), | |
| _rf("rf-tela-agujeros-manchas", "iicl-31hjn/textile-ofqd1", "tela"), | |
| _rf("rf-tela-manchas-arrugas", "zco/textile-defect-detection-ixt2r", "tela"), | |
| ) | |
| # Cartón: cajas dañadas (aplastadas, rotas, mojadas) y en buen estado. | |
| CARTON = ( | |
| _rf("rf-cajas-1", "bekarys0504-gmail-com/damaged_box", "carton"), | |
| _rf("rf-cajas-3", "project-33xgh/damaged-box-auh4b", "carton"), | |
| _rf("rf-cajas-4", "robot-arm-xlohb/boxcondition", "carton"), | |
| _rf("rf-cajas-5", "project-33xgh/damaged-box-detection", "carton"), | |
| ) | |
| # Vidrio: roto o agrietado y en buen estado. | |
| VIDRIO = ( | |
| _rf("rf-vidrio-1", "focus/detect_broken_glass", "vidrio"), | |
| _rf("rf-vidrio-2", "nhabduls-workspace/cracked-glass-detection", "vidrio"), | |
| _rf("rf-vidrio-3", "glassdefectsdetection/glassdefectscheckingmodel", "vidrio"), | |
| ) | |
| FUENTES: tuple[Fuente, ...] = ( | |
| TRASHNET, | |
| REALWASTE, | |
| GARBAGE_V2, | |
| *RECORTES, | |
| *MADERA, | |
| *METAL, | |
| *TELA, | |
| *CARTON, | |
| *VIDRIO, | |
| ) | |
| def licencia_comercial(licencia: str) -> bool: | |
| texto = licencia.lower() | |
| no_comercial = "-nc" in texto or " nc" in texto or "noncommercial" in texto | |
| derechos_reservados = "©" in texto or "original authors" in texto | |
| return ( | |
| any(lic in texto for lic in LICENCIAS_COMERCIALES) | |
| and not no_comercial | |
| and not derechos_reservados | |
| ) | |