diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c9f57c0a8cb3b0c5979ccce5e8bf4aaaa5e829d4 --- /dev/null +++ b/README.md @@ -0,0 +1,29 @@ +--- +library_name: tensorflow-lite +license: other +tags: +- android +- on-device-training +- fire-detection +--- + +# Modèles FireViewer pour Android + +Modèles avec dimensions d’image variables et apprentissage local des têtes d’adaptation. Les réseaux visuels restent figés. Les checkpoints contiennent les paramètres, les moments SGD et le compteur d’étapes pour reprendre et continuer l’apprentissage. + +| Modèle | Livraison | Apprentissage et reprise sur hôte 2.16.1 | Exécution Android | +|---|---|---|---| +| FireViewer D-FINE-M strict v1 | [modèle et contrat](models/fireviewer_dfine_m_strict_v1_learning) | validés | pas de test de ce modèle sur appareil | +| FireViewer RT-DETRv2-R50 | [modèle et contrat](models/fireviewer_rtdetr_v2_r50_learning) | validés | pas de test de ce modèle sur appareil | +| FireViewer YOLO11-M strict v1 | [modèle et contrat](models/fireviewer_yolo11m_strict_v1_learning) | validés | validée sur émulateur x86_64 | +| FireViewer RF-DETR Medium 1.10 strict v1 | [modèle et contrat](models/fireviewer_rfdetr_medium_v110_learning) | validés | pas de test de ce modèle sur appareil | +| FireViewer DINOv3 pointing pilot v1 | [modèle et contrat](models/fireviewer_dinov3_pointing_pilot_v1_learning) | validés | pas de test de ce modèle sur appareil | +| FireViewer SegFormer-B2 baseline v1 | source optionnelle introuvable | — | — | + +Les fichiers de configuration précisent la normalisation, les labels et les sorties. YOLO accepte H/W variables par multiples de 32. Les autres modèles redimensionnent dans le graphe vers leur grille interne ; le dataset peut contenir des images de dimensions différentes. + +- [SDK Kotlin et guide d’apprentissage continu](android/README.md) +- [Rapport de livraison et empreintes vérifiées](reports/fireviewer-completion.json) +- [Sources de conversion](conversion) + +Les essais utilisent des annotations synthétiques pour vérifier les mises à jour et leur persistance. Ils ne mesurent pas la précision de détection ni l’oubli catastrophique. Aucun téléphone ARM physique n’a été testé. Les notices et conditions des modèles sources sont conservées dans chaque dossier. diff --git a/android/DetectorTargets.kt b/android/DetectorTargets.kt new file mode 100644 index 0000000000000000000000000000000000000000..fcde7196ab4028ce58b40f9283e3978520b6c34d --- /dev/null +++ b/android/DetectorTargets.kt @@ -0,0 +1,18 @@ +package org.fireviewer.litert.training + +/** Coordinates refer to the normalized image supplied to the model. */ +data class ReviewedBox(val x1: Float, val y1: Float, val x2: Float, val y2: Float, val label: Int) + +fun detectorTargets(boxes: List, classCount: Int): FloatArray { + require(classCount > 0 && boxes.size <= 100) + val values = FloatArray(600) + boxes.forEachIndexed { index, box -> + require(listOf(box.x1, box.y1, box.x2, box.y2).all { it.isFinite() && it in 0f..1f }) + require(box.x1 < box.x2 && box.y1 < box.y2 && box.label in 0 until classCount) + val row = index * 6 + values[row] = box.x1; values[row + 1] = box.y1 + values[row + 2] = box.x2; values[row + 3] = box.y2 + values[row + 4] = box.label.toFloat(); values[row + 5] = 1f + } + return values +} diff --git a/android/FireDetectorRuntimeCheck.java b/android/FireDetectorRuntimeCheck.java new file mode 100644 index 0000000000000000000000000000000000000000..8b2c929304ccc1861ab8061e9fd68738654b712d --- /dev/null +++ b/android/FireDetectorRuntimeCheck.java @@ -0,0 +1,44 @@ +import java.io.File; +import java.util.Arrays; +import java.util.Map; +import org.fireviewer.litert.training.OnDeviceLearning; + +/** Real Android CPU exercise: two image sizes, train, durable save and optimizer resume. */ +public final class FireDetectorRuntimeCheck { + public static void main(String[] args) { + File model=new File(args[0]), root=new File(args[1]); + int threads=args.length>2?Integer.parseInt(args[2]):2; + String schema="0000000000000000000000000000000000000000000000000000000000000000"; + Map square=TrainingRuntimeCheck.inputs("x",TrainingRuntimeCheck.image(128,128)); + Map rectangle=TrainingRuntimeCheck.inputs("x",TrainingRuntimeCheck.image(128,192)); + float[] target=new float[600]; + target[0]=.3f;target[1]=.3f;target[2]=.7f;target[3]=.7f;target[4]=0;target[5]=1; + File checkpoint;float[] trained,continued;float firstLoss,secondLoss; + System.out.println("FIRE_CHECK opening "+model.getName()); + try(OnDeviceLearning session=new OnDeviceLearning(model,schema,threads)) { + System.out.println("FIRE_CHECK opened"); + float[] original=session.infer(square).get("detections").getValues(); + System.out.println("FIRE_CHECK infer "+original.length); + firstLoss=session.train(square,target,.001f); + trained=session.infer(square).get("detections").getValues(); + TrainingRuntimeCheck.check(!Arrays.equals(original,trained),"Training did not change outputs"); + checkpoint=session.save(root,"synthetic-android-runtime-fixture",1L); + System.out.println("FIRE_CHECK saved loss="+firstLoss); + secondLoss=session.train(rectangle,target,.001f); + continued=session.infer(rectangle).get("detections").getValues(); + System.out.println("FIRE_CHECK continued loss="+secondLoss); + } + try(OnDeviceLearning resumed=new OnDeviceLearning(model,schema,threads)) { + resumed.restore(checkpoint); + TrainingRuntimeCheck.check(Arrays.equals(trained,resumed.infer(square).get("detections").getValues()),"Restored predictions differ"); + resumed.train(rectangle,target,.001f); + TrainingRuntimeCheck.check(Arrays.equals(continued,resumed.infer(rectangle).get("detections").getValues()),"Resumed optimizer differs"); + } + try(OnDeviceLearning invalid=new OnDeviceLearning(model,"1000000000000000000000000000000000000000000000000000000000000000",threads)) { + boolean rejected=false; + try { invalid.restore(checkpoint); } catch(IllegalArgumentException expected) { rejected=true; } + TrainingRuntimeCheck.check(rejected,"Incompatible class schema accepted"); + } + System.out.println("ANDROID_FIRE_TRAINING_PASS {\"runtime\":\""+org.tensorflow.lite.TensorFlowLite.runtimeVersion()+"\",\"mixed_shapes\":[[128,128],[128,192]],\"train\":true,\"restore_exact\":true,\"optimizer_resume_exact\":true,\"incompatible_schema_rejected\":true,\"first_loss\":"+firstLoss+",\"second_loss\":"+secondLoss+"}"); + } +} diff --git a/android/OnDeviceLearning.kt b/android/OnDeviceLearning.kt new file mode 100644 index 0000000000000000000000000000000000000000..942b698086a7594aa22a167692eaedc4c5b6dac6 --- /dev/null +++ b/android/OnDeviceLearning.kt @@ -0,0 +1,121 @@ +package org.fireviewer.litert.training + +import android.util.AtomicFile +import org.json.JSONObject +import org.tensorflow.lite.Interpreter +import java.io.File +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.security.MessageDigest +import java.util.UUID + +/** CPU training and durable optimizer resume; call from a background dispatcher. + * One session is serialized. Only reviewed annotations belong in targets. + * The model's contract states which parameters are trainable. + */ +class OnDeviceLearning( + private val model: File, + private val labelSchemaSha256: String, + threads: Int = 2 +) : AutoCloseable { + private val modelSha256 = sha256(model) + private val engine = Interpreter(model, Interpreter.Options().setNumThreads(threads).setUseXNNPACK(false)) + init { + require(threads in 1..8) + require(labelSchemaSha256.matches(Regex("[0-9a-f]{64}"))) + require(engine.signatureKeys.toSet().containsAll(setOf("train", "infer", "save", "restore"))) + } + + data class Output(val shape: List, val values: FloatArray) + + /** Nested primitive arrays carry their current shape to runSignature. + * Image models accept x as Array>> in NCHW order. + * Pixels must already use the normalization from android_model_config.json. + */ + @Synchronized fun infer(inputs: Map): Map { + engine.runSignature(inputs, mutableMapOf(), "infer") + return engine.getSignatureOutputs("infer").associateWith { name -> + val tensor = engine.getOutputTensorFromSignature(name, "infer") + require(tensor.dataType().name == "FLOAT32") + require(tensor.numBytes() <= 128 * 1024 * 1024) + val values = FloatArray(tensor.numElements()) + tensor.asReadOnlyBuffer().order(ByteOrder.nativeOrder()).asFloatBuffer().get(values) + require(values.all { it.isFinite() }) + Output(tensor.shape().toList(), values) + } + } + + @Synchronized fun train(inputs: Map, target: FloatArray, learningRate: Float): Float { + require(learningRate.isFinite() && learningRate in .000001f..1f) + val tensor = engine.getInputTensorFromSignature("y", "train") + require(tensor.dataType().name == "FLOAT32" && target.size == tensor.numElements()) + require(target.all { it.isFinite() }) + val call = inputs.toMutableMap() + call["y"] = floats(target) + call["learning_rate"] = floats(floatArrayOf(learningRate)) + engine.runSignature(call, mutableMapOf(), "train") + val loss = engine.getOutputTensorFromSignature("loss", "train") + .asReadOnlyBuffer().order(ByteOrder.nativeOrder()).getFloat(0) + require(loss.isFinite()) { "Non-finite loss; discard this candidate and restore its last checkpoint" } + return loss + } + + /** The generation becomes current only after all checkpoint files are hashed. + * Previous generations stay available for rollback; this method never deletes them. + */ + @Synchronized fun save(root: File, datasetRevision: String, reviewedSamplesSeen: Long): File { + require(reviewedSamplesSeen >= 0) + root.mkdirs() + val generation = File(root, UUID.randomUUID().toString()).apply { check(mkdir()) } + val prefix = File(generation, "weights") + engine.runSignature(mapOf("checkpoint_path" to prefix.absolutePath), mutableMapOf(), "save") + val files = generation.walkTopDown().filter { it.isFile }.toList() + require(files.isNotEmpty() && files.all { it.length() > 0 }) + val hashes = JSONObject() + files.forEach { file -> hashes.put(file.relativeTo(generation).invariantSeparatorsPath, sha256(file)) } + val receipt = JSONObject().put("schema", 1).put("modelSha256", modelSha256) + .put("labelSchemaSha256", labelSchemaSha256).put("datasetRevision", datasetRevision) + .put("reviewedSamplesSeen", reviewedSamplesSeen).put("checkpointPrefix", "weights") + .put("files", hashes) + writeAtomic(File(generation, "receipt.json"), receipt.toString(2)) + writeAtomic(File(root, "CURRENT"), generation.name) + return generation + } + + @Synchronized fun restore(generation: File) { + val receipt = JSONObject(AtomicFile(File(generation, "receipt.json")).openRead().bufferedReader().use { it.readText() }) + require(receipt.getInt("schema") == 1) + require(receipt.getString("modelSha256") == modelSha256) { "Checkpoint belongs to another model revision" } + require(receipt.getString("labelSchemaSha256") == labelSchemaSha256) { "Checkpoint label schema differs" } + val hashes = receipt.getJSONObject("files") + require(hashes.length() > 0) + hashes.keys().forEach { name -> + val file = File(generation, name) + require(file.canonicalPath.startsWith(generation.canonicalPath + File.separator)) + require(file.isFile && sha256(file) == hashes.getString(name)) { "Checkpoint missing or changed" } + } + val prefix = File(generation, receipt.getString("checkpointPrefix")) + require(prefix.canonicalPath.startsWith(generation.canonicalPath + File.separator)) + engine.runSignature(mapOf("checkpoint_path" to prefix.absolutePath), mutableMapOf(), "restore") + } + + @Synchronized override fun close() = engine.close() + + companion object { + private fun floats(values: FloatArray) = ByteBuffer.allocateDirect(values.size * 4) + .order(ByteOrder.nativeOrder()).apply { asFloatBuffer().put(values) } + private fun sha256(file: File): String { + val digest = MessageDigest.getInstance("SHA-256") + file.inputStream().use { stream -> + val buffer = ByteArray(1024 * 1024) + while (true) { val count = stream.read(buffer); if (count < 0) break; digest.update(buffer, 0, count) } + } + return digest.digest().joinToString("") { "%02x".format(it) } + } + private fun writeAtomic(file: File, text: String) { + val atomic = AtomicFile(file); val stream = atomic.startWrite() + try { stream.write(text.toByteArray(Charsets.UTF_8)); atomic.finishWrite(stream) } + catch (error: Exception) { atomic.failWrite(stream); throw error } + } + } +} diff --git a/android/README.md b/android/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d81d868ed2cc45fea8d5c52c56ba3ddecb7d93e9 --- /dev/null +++ b/android/README.md @@ -0,0 +1,60 @@ +# Apprentissage FireViewer sur Android + +Les fichiers `models/*_learning/model.tflite` contiennent les signatures exécutables `infer`, `train`, `save` et `restore`. Les têtes d'adaptation sont entraînables localement ; les réseaux visuels restent figés. Le modèle livré démarre avec ses poids source et une adaptation identité, sans les mises à jour synthétiques utilisées pour les tests. + +Utiliser les deux dépendances de même version : + +```kotlin +implementation("org.tensorflow:tensorflow-lite:2.16.1") +implementation("org.tensorflow:tensorflow-lite-select-tf-ops:2.16.1") +``` + +`OnDeviceLearning.kt` fournit les appels, la sauvegarde atomique, les empreintes SHA-256 et la vérification des checkpoints. Exécuter ces opérations dans un travail de fond avec CPU, XNNPACK désactivé. Le mélange LiteRT 2.2 / Select TF Ops 2.16 n'est pas la configuration qualifiée. Les résultats Android exacts figurent dans `reports/fireviewer-completion.json` et les journaux joints ; les essais hôte sont séparés des essais Android. Un émulateur ne mesure pas la vitesse sur téléphone ARM. + +## Images et cibles de détection + +Chaque image est un tableau float `[1][3][H][W]` RGB. Appliquer la normalisation de `android_model_config.json`. Les images d'un même dataset peuvent avoir des tailles différentes et sont traitées individuellement. YOLO accepte des côtés multiples de 32. Les autres architectures redimensionnent dans le graphe vers leur grille interne ; les coordonnées de sortie sont rapportées à la taille de l'entrée `x`. + +Pour les détecteurs, `y` contient 100 lignes `[x1,y1,x2,y2,classe,présente]`, aplaties en 600 floats. Les coordonnées sont normalisées dans l'image présentée au modèle, après prétraitement. `classe` indexe les labels du contrat. `présente=1` indique une annotation validée ; les lignes inutilisées sont nulles. Une image explicitement négative utilise toutes les lignes nulles. Une image sans annotation connue doit être exclue de l'apprentissage. `DetectorTargets.kt` construit et vérifie ce tableau. + +```kotlin +val imageInputs: Map = mapOf("x" to nchwImage) +OnDeviceLearning(modelFile, labelSchemaSha256, threads = 1).use { session -> + if (previousGeneration != null) session.restore(previousGeneration) + val loss = session.train(imageInputs, reviewedTargets, learningRate = 0.001f) + val predictions = session.infer(imageInputs) + val generation = session.save(checkpointRoot, datasetRevision, reviewedSamplesSeen) +} +``` + +Les adaptations modifient les scores des classes et les boîtes des propositions disponibles. Elles ne créent pas de propositions absentes du détecteur figé. Changer le nombre de classes nécessite de reconstruire la tête avec les scripts fournis. + +## DINOv3 : quatre tâches + +`infer` renvoie `segmentation_logits`, `point_logits`, `presence_logits` et `abstention_logits`. Appliquer une sigmoïde pour obtenir les probabilités. Les cartes restent sur la grille 448 × 448 ; remettre les cartes et points à l'échelle de l'image d'origine. + +Pour `train`, `y` est `[1,3,448,448]` : masque de segmentation, carte de pointage, puis masque des pixels valides. Redimensionner les masques binaires au plus proche voisin et préparer le pointage sur cette même grille. Fournir aussi dans `inputs` : + +- `presence` : float `[1,2]`, ordre `flame_visible`, `smoke_visible`. +- `abstention` : float `[1]`. +- `supervision` : float `[4]`, ordre segmentation, pointage, abstention, présence ; 1 si la tâche est annotée, 0 sinon. + +Passer ces entrées auxiliaires à `train` uniquement. Une absence d'annotation n'est pas un label négatif. Les adaptateurs de sortie des quatre tâches sont entraînables ; DINOv3 et le décodeur DPT restent figés. + +## Reprendre et continuer + +Chaque génération sauvegarde les paramètres entraînables, les moments SGD et le compteur d'étapes. Son `receipt.json` contient l'empreinte du modèle, celle du schéma de labels, la révision du dataset, le nombre d'exemples vus et les empreintes des fichiers. `CURRENT` désigne la dernière génération complète. `restore` refuse un autre modèle, un autre schéma de labels ou un fichier modifié. Les anciennes générations permettent un retour arrière. + +Conserver les annotations humaines, l'ordre ou la graine d'échantillonnage et un ensemble de rejeu mêlant anciens et nouveaux exemples. Reprendre cet ordre à partir du compteur sauvegardé. Les checkpoints ne remplacent pas ces données. Évaluer un candidat sur des images réservées avant de l'activer ; les tests d'exécution ne mesurent ni la précision ni l'oubli catastrophique. En cas de perte non finie, restaurer la dernière génération valide. + +Les scripts et ce SDK sont livrés avec les modèles. Cette livraison ne modifie pas le code de l'application Android en cours de travail dans un autre dossier. + +## Reproduire le contrôle Android + +Avec JDK 21, Kotlin 2.2.10 et Android SDK (plateforme 36, build-tools 36.0.0), définir `JAVA_HOME`, `KOTLIN_HOME` et `ANDROID_SDK_ROOT`, puis lancer `python3 prepare_runtime_check.py` dans ce dossier. Le script télécharge les AAR 2.16.1, compile le SDK et produit `training-check.jar` et `lib/` pour l'émulateur x86_64. Copier ces fichiers et le modèle dans `/data/local/tmp/vds-fire-training` avec `adb push`. + +```sh +adb shell 'CLASSPATH=/data/local/tmp/vds-fire-training/training-check.jar LD_LIBRARY_PATH=/data/local/tmp/vds-fire-training/lib:/system/lib64:/vendor/lib64 app_process /data/local/tmp/vds-fire-training FireDetectorRuntimeCheck /data/local/tmp/vds-fire-training/model.tflite /data/local/tmp/vds-fire-training/checkpoints 1 128' +``` + +La dernière ligne `ANDROID_FIRE_TRAINING_PASS` signifie que toutes les assertions ont réussi. Ce programme utilise des cibles synthétiques et des checkpoints distincts ; il ne réécrit pas le fichier modèle. Le paramètre final fixe la hauteur de test, et la seconde image est rectangulaire. Le test final YOLO a été exécuté avec 64 et 96 pixels, en complément de la première version testée avec 128 et 192 pixels. diff --git a/android/TrainingRuntimeCheck.java b/android/TrainingRuntimeCheck.java new file mode 100644 index 0000000000000000000000000000000000000000..149debde8dc37c03f5ce05c4a36e03272c714e06 --- /dev/null +++ b/android/TrainingRuntimeCheck.java @@ -0,0 +1,51 @@ +import java.io.File; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import org.tensorflow.lite.Interpreter; + +/** Runs under Android app_process with the matched TensorFlow Lite 2.16.1 AARs. */ +public final class TrainingRuntimeCheck { + static float[][][][] image(int h,int w) { + float[][][][] x=new float[1][3][h][w]; + for(int c=0;c<3;c++)for(int y=0;y inputs(Object... values) { + Map out=new HashMap<>();for(int n=0;n(),"save"); + check(new File(checkpoint).length()>0,"empty checkpoint"); + train(i,x,target);continued=infer(i,x,classes); + } + try(Interpreter restored=new Interpreter(model,options)) { + restored.runSignature(inputs("checkpoint_path",checkpoint),new HashMap<>(),"restore"); + check(Arrays.equals(trained,infer(restored,x,classes)),"restore differs"); + train(restored,x,target);check(Arrays.equals(continued,infer(restored,x,classes)),"optimizer resume differs"); + } + System.out.println("ANDROID_TRAINING_PASS {\"model\":\""+model.getName()+"\",\"mixed_shapes\":[[224,224],[192,320]],\"first_loss\":"+first+",\"last_loss\":"+last+",\"restart_exact\":true,\"optimizer_resume_exact\":true,\"runtime\":\""+org.tensorflow.lite.TensorFlowLite.runtimeVersion()+"\",\"device\":\"Android emulator x86_64\"}"); + } +} diff --git a/android/prepare_runtime_check.py b/android/prepare_runtime_check.py new file mode 100644 index 0000000000000000000000000000000000000000..4bc0625ad3cd1923f8e7221cb9ee0405b757064a --- /dev/null +++ b/android/prepare_runtime_check.py @@ -0,0 +1,26 @@ +import pathlib,zipfile,subprocess,os,urllib.request +root=pathlib.Path(__file__).resolve().parent +cache=pathlib.Path(os.environ.get('GRADLE_USER_HOME','/workspace/cache/gradle'))/'caches/modules-2/files-2.1' +java=pathlib.Path(os.environ.get('JAVA_HOME','/workspace/toolchains/jdk-21')) +android=pathlib.Path(os.environ.get('ANDROID_SDK_ROOT','/workspace/toolchains/android-sdk')) +kotlin=pathlib.Path(os.environ.get('KOTLIN_HOME','/workspace/toolchains/kotlin-2.2.10')) +abi=os.environ.get('VDS_ANDROID_ABI','x86_64') +env={**os.environ,'JAVA_HOME':str(java),'PATH':str(java/'bin')+':'+os.environ['PATH']} +jars=[] +dependencies=[('org.tensorflow','tensorflow-lite','2.16.1'),('org.tensorflow','tensorflow-lite-api','2.16.1'),('org.tensorflow','tensorflow-lite-select-tf-ops','2.16.1')] +for group,name,version in dependencies: + matches=list((cache/group/name/version).glob('*/*.aar')) + aar=matches[0] if matches else root/(name+'-'+version+'.aar') + if not aar.exists():urllib.request.urlretrieve(f'https://repo.maven.apache.org/maven2/{group.replace(".","/")}/{name}/{version}/{name}-{version}.aar',aar) + with zipfile.ZipFile(aar) as z: + jar=root/(name+'.jar');jar.write_bytes(z.read('classes.jar'));jars.append(str(jar)) + for item in z.namelist(): + if item.startswith('jni/'+abi+'/') and item.endswith('.so'): + lib=root/'lib'/pathlib.Path(item).name;lib.parent.mkdir(exist_ok=True);lib.write_bytes(z.read(item)) +classes=root/'classes';classes.mkdir(exist_ok=True) +sdk=root/'on-device-learning.jar' +subprocess.run([str(kotlin/'bin/kotlinc'),*[str(p) for p in root.glob('*.kt')],'-cp',':'.join([str(android/'platforms/android-36/android.jar'),*jars]),'-d',str(sdk)],check=True,env=env) +jars.extend([str(sdk),str(kotlin/'lib/kotlin-stdlib.jar')]) +subprocess.run([str(java/'bin/javac'),'-source','11','-target','11','-cp',':'.join(jars),'-d',str(classes),*[str(p) for p in root.glob('*.java')]],check=True,env=env) +subprocess.run([str(android/'build-tools/36.0.0/d8'),'--min-api','28','--output',str(root/'training-check.jar'),*[str(p) for p in classes.glob('*.class')],*jars],check=True,env=env) +print(root/'training-check.jar') diff --git a/conversion/README.md b/conversion/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ffbe9de5082175f1bf71d79e7d5f54710dde05a --- /dev/null +++ b/conversion/README.md @@ -0,0 +1,33 @@ +# Reconstruction des modèles FireViewer + +Les artefacts publiés sont directement exécutables sur Android ; ces outils servent à reconstruire les conversions sur Linux. L'environnement de production des fichiers est enregistré dans `reports/environments/`. Les révisions HF immuables, empreintes, labels, grilles et normalisations sont dans chaque `runtime_contract.json` et `source.json`. + +Le dossier de travail des scripts est `/workspace/vds-litert-batch2`. Copier le contenu de `conversion/` à cet emplacement et recréer les environnements Python avec les versions enregistrées. Les poids sources se placent sous `models/upstream//` à partir des dépôts HF et révisions des contrats. Utiliser l'authentification HF habituelle ; aucun secret n'est livré. + +Routes utilisées : + +| Modèle | Réseau figé | Apprentissage | +|---|---|---| +| YOLO11M | ONNX source, conversion H/W dynamiques dans `dynamic_hf_worker.py` | `build_trainable_detector.py` | +| RF-DETR Medium | adaptateur officiel dans `package/scripts/convert_batch2.py`, via `prepare_learning_backbone.py` | `build_trainable_detector.py` | +| RT-DETRv2 R50 | `finish_fire_backbone.py`, grille source 768 × 768, TopK natif et positions pré-calculées | `build_trainable_detector.py` | +| DINOv3 | adaptateur source dans `package/scripts/convert_batch2.py`, contrôle `check_dinov3_source_parity.py` | `build_trainable_fire_multitask.py` | +| D-FINE-M | `convert_dfine_final.py`, détails de conservation mémoire dans le rapport de parité | `build_trainable_detector.py` | + +`merge_training_graph.py` partage les poids figés entre les signatures et ajoute le redimensionnement interne pour les grilles fixes. Les scripts de construction écrivent un modèle initialisé avec la tête identité puis exécutent séparément les mises à jour de test. Les poids entraînés sur ces exemples synthétiques ne sont pas enregistrés dans le fichier modèle de livraison. + +Pour les détecteurs, copier le `source.json` de `reports//` dans `learning//source.json`, puis exécuter avec TensorFlow 2.19 : + +```sh +/opt/vds-tf/bin/python build_trainable_detector.py +``` + +Pour DINOv3 : + +```sh +/opt/vds-tf/bin/python build_trainable_fire_multitask.py +``` + +`check_fire_runtime216.py ...` exécute ensuite les mises à jour, checkpoints et reprises avec TensorFlow Lite 2.16.1 sur Linux, la version des dépendances Android. Les tests Android utilisent le SDK et le programme de `android/` ; ils sont identifiés séparément dans les rapports. Les tests de parité comparent les conversions aux modèles source avant toute adaptation. Ne pas publier une conversion qui échoue à ces contrôles. + +Les scripts de publication de la session utilisent le fichier d'authentification local `/workspace/.secrets/hf_token`. Ce chemin est une configuration de l'exécution, pas un secret inclus dans les artefacts. Le dépôt cible conserve sa visibilité existante. diff --git a/conversion/build_trainable_detector.py b/conversion/build_trainable_detector.py new file mode 100644 index 0000000000000000000000000000000000000000..f5d8e52b985b65d1eade4134de5d42a0b8f92e7c --- /dev/null +++ b/conversion/build_trainable_detector.py @@ -0,0 +1,142 @@ +"""Frozen detector plus trainable candidate classification/box adaptation on Android. + +The adaptation cannot generate proposals absent from the frozen detector. It is +not full detector fine-tuning. Targets are reviewed xyxy/class/mask annotations. +""" +import os +os.environ.update(TF_CPP_MIN_LOG_LEVEL='2',TF_NUM_INTRAOP_THREADS='6',TF_NUM_INTEROP_THREADS='2') +import sys,json,hashlib +from pathlib import Path +import tensorflow as tf,numpy as np +from merge_training_graph import merge,dynamic_resize_backbone,freeze_auxiliary_inputs +root=Path('/workspace/vds-litert-batch2');key=sys.argv[1];out=root/'learning'/key;out.mkdir(parents=True,exist_ok=True) +source=json.loads((out/'source.json').read_text());C=source['class_count'];kind=source['kind'] +class_indices=source.get('class_indices',list(range(C))) +specs=([tf.TensorSpec([1,C+4,None],tf.float32,name='features')] if kind=='yolo' else [tf.TensorSpec([1,None,4],tf.float32,name='features_boxes'),tf.TensorSpec([1,None,C],tf.float32,name='features_logits')]) +if kind=='rtmdet':specs=[tf.TensorSpec([1,None,None,C+4],tf.float32,name=f'features_{stride}') for stride in (8,16,32)] +specs+=[tf.TensorSpec([1,3,None,None],tf.float32,name='image')] + +class Adaptation(tf.Module): + def __init__(self): + super().__init__() + self.cw=tf.Variable(tf.eye(C),name='class_weight');self.cb=tf.Variable(tf.zeros([C]),name='class_bias') + self.bw=tf.Variable(tf.eye(4),name='box_weight');self.bb=tf.Variable(tf.zeros([4]),name='box_bias') + self.parameters=[self.cw,self.cb,self.bw,self.bb] + self.momentum=[tf.Variable(tf.zeros_like(v),name='momentum_'+v.name.split(':')[0]) for v in self.parameters] + self.step=tf.Variable(0,dtype=tf.int64,name='step');self.state=self.parameters+self.momentum+[self.step] + self.names=['class_weight','class_bias','box_weight','box_bias','momentum_class_weight','momentum_class_bias','momentum_box_weight','momentum_box_bias','step'] + self.infer=tf.function(self._infer,input_signature=specs) + self.train=tf.function(self._train,input_signature=specs+[tf.TensorSpec([1,100,6],tf.float32,name='y'),tf.TensorSpec([],tf.float32,name='learning_rate')]) + def predict(self,args): + image=args[-1];shape=tf.cast(tf.shape(image),tf.float32);scale=tf.stack([shape[3],shape[2],shape[3],shape[2]]) + if kind=='yolo': + values=tf.transpose(args[0],[0,2,1]);boxes=values[:,:,:4]/scale + p=tf.clip_by_value(values[:,:,4:],1e-6,1-1e-6);logits=tf.math.log(p)-tf.math.log1p(-p) + elif kind=='rtmdet': + all_boxes=[];all_logits=[] + for feature,stride in zip(args[:3],(8,16,32)): + fs=tf.shape(feature);grid_x,grid_y=tf.meshgrid(tf.range(fs[2]),tf.range(fs[1]));grid=tf.cast(tf.stack([grid_x,grid_y],-1),tf.float32)*stride + d=feature[:,:,:,C:]*stride;lo=grid[None]-d[:,:,:,:2];hi=grid[None]+d[:,:,:,2:] + boxes=tf.concat([(lo+hi)/2,hi-lo],-1)/scale + all_boxes.append(tf.reshape(boxes,[1,-1,4]));all_logits.append(tf.reshape(feature[:,:,:,:C],[1,-1,C])) + boxes=tf.concat(all_boxes,1);logits=tf.concat(all_logits,1) + else:boxes,logits=args[:2] + return boxes@self.bw+self.bb,logits@self.cw+self.cb,scale + def _infer(self,*args): + boxes,logits,scale=self.predict(args) + if kind=='yolo':return {'detections':tf.transpose(tf.concat([boxes*scale,tf.sigmoid(logits)],-1),[0,2,1])} + probabilities=tf.nn.softmax(logits,axis=-1) if source.get('class_activation')=='softmax' else tf.sigmoid(logits) + probabilities=tf.gather(probabilities,class_indices,axis=-1) + return {'boxes':boxes,'logits':logits,'detections':tf.transpose(tf.concat([boxes*scale,probabilities],-1),[0,2,1])} + def _train(self,*args): + y,lr=args[-2:];targets=y[0];valid=targets[:,5];gt=targets[:,:4];labels=tf.gather(tf.constant(class_indices,tf.int32),tf.cast(targets[:,4],tf.int32)) + with tf.GradientTape() as tape: + boxes,logits,_=self.predict(args[:-2]);b=boxes[0];scores=logits[0] + xyxy=tf.concat([b[:,:2]-b[:,2:]/2,b[:,:2]+b[:,2:]/2],-1) + lo=tf.maximum(xyxy[:,None,:2],gt[None,:,:2]);hi=tf.minimum(xyxy[:,None,2:],gt[None,:,2:]) + inter=tf.reduce_prod(tf.maximum(hi-lo,0),-1) + area=tf.reduce_prod(tf.maximum(xyxy[:,2:]-xyxy[:,:2],0),-1) + ga=tf.reduce_prod(tf.maximum(gt[:,2:]-gt[:,:2],0),-1) + iou=inter/tf.maximum(area[:,None]+ga[None,:]-inter,1e-7)*valid[None,:] + # Assignment is discrete; gradients update the prediction, never the matching. + best_candidate=tf.argmax(iou,axis=0,output_type=tf.int32) + forced_pairs=tf.transpose(tf.one_hot(best_candidate,tf.shape(b)[0])*valid[:,None]) + forced=tf.reduce_max(forced_pairs,axis=1) + # Assign a forced candidate to the annotation that selected it, + # including zero-IoU cases. Collisions keep the larger-IoU target. + best_target=tf.argmax(iou+forced_pairs*(1+iou),axis=1,output_type=tf.int32) + positive=tf.stop_gradient(tf.maximum(tf.cast(tf.reduce_max(iou,axis=1)>.5,tf.float32),forced)) + assigned=tf.gather(gt,best_target);assigned_class=tf.gather(labels,best_target) + onehot=tf.one_hot(assigned_class,C)*positive[:,None] + if source.get('class_activation')=='softmax': + onehot+=tf.one_hot(source['background_index'],C)[None,:]*(1-positive[:,None]) + ce=tf.nn.softmax_cross_entropy_with_logits(labels=onehot,logits=scores) + else:ce=tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=onehot,logits=scores),axis=-1) + cls=tf.reduce_sum(ce*positive)/tf.maximum(tf.reduce_sum(positive),1)+.05*tf.reduce_sum(ce*(1-positive))/tf.maximum(tf.reduce_sum(1-positive),1) + target_box=tf.concat([(assigned[:,:2]+assigned[:,2:])/2,assigned[:,2:]-assigned[:,:2]],-1) + geometry=tf.reduce_sum(tf.reduce_mean(tf.square(b-target_box),axis=-1)*positive)/tf.maximum(tf.reduce_sum(positive),1) + loss=cls+5*geometry + gradients=tape.gradient(loss,self.parameters) + gradients,_=tf.clip_by_global_norm(gradients,10.) + for v,m,g in zip(self.parameters,self.momentum,gradients):m.assign(.9*m+g);v.assign_sub(lr*m) + self.step.assign_add(1);return {'loss':loss,'step':self.step.read_value()} + @tf.function(input_signature=[tf.TensorSpec([],tf.string,name='checkpoint_path')]) + def save(self,checkpoint_path): + tf.raw_ops.Save(filename=checkpoint_path,tensor_names=self.names,data=[v.read_value() for v in self.state]);return {'step':self.step.read_value()} + @tf.function(input_signature=[tf.TensorSpec([],tf.string,name='checkpoint_path')]) + def restore(self,checkpoint_path): + for n,v in zip(self.names,self.state):v.assign(tf.raw_ops.Restore(file_pattern=checkpoint_path,tensor_name=n,dt=v.dtype)) + return {'step':self.step.read_value()} + @tf.function(input_signature=[]) + def training_state(self):return {'head_kernel':self.cw.read_value(),'step':self.step.read_value()} + +module=Adaptation();names=['infer','train','save','restore','training_state'];saved=out/'head_saved_model' +tf.saved_model.save(module,str(saved),signatures={n:getattr(module,n).get_concrete_function() for n in names}) +converter=tf.lite.TFLiteConverter.from_saved_model(str(saved));converter.target_spec.supported_ops=[tf.lite.OpsSet.TFLITE_BUILTINS,tf.lite.OpsSet.SELECT_TF_OPS];converter.experimental_enable_resource_variables=True +head=out/'head.tflite';head.write_bytes(converter.convert());artifact=out/'model.tflite';backbone=Path(source['backbone']) +if source.get('freeze_auxiliary_inputs'): + frozen=out/'frozen_inputs.tflite';freeze_auxiliary_inputs(backbone,frozen,source['freeze_auxiliary_inputs']);backbone=frozen +preprocessing={'mode':'native_dynamic','stride':32} +if not source.get('native_dynamic',True): + wrapped=out/'dynamic_backbone.tflite';preprocessing=dynamic_resize_backbone(backbone,wrapped);backbone=wrapped +merge(backbone,head,artifact,source['bindings'],image_size=source.get('app_size',320)) +def load(): + i=tf.lite.Interpreter(model_path=str(artifact),num_threads=6,experimental_op_resolver_type=tf.lite.experimental.OpResolverType.BUILTIN_WITHOUT_DEFAULT_DELEGATES) + return i,{n:i.get_signature_runner(n) for n in names} +i,s=load();h,w=source.get('test_size',[320,320]);x=np.random.default_rng(34).uniform(0,1,(1,3,h,w)).astype(np.float32) +before=s['infer'](x=x) +# Initial adaptation must preserve detector predictions. +base=tf.lite.Interpreter(model_path=str(backbone),num_threads=6,experimental_op_resolver_type=tf.lite.experimental.OpResolverType.BUILTIN_WITHOUT_DEFAULT_DELEGATES) +d=base.get_input_details()[0];base.resize_tensor_input(d['index'],x.shape,strict=True);base.allocate_tensors();base.set_tensor(d['index'],x);base.invoke() +raw=[base.get_tensor(d['index']) for d in base.get_output_details()] +if kind=='yolo': + np.testing.assert_allclose(before['detections'],raw[source['bindings']['features']],atol=1e-3,rtol=1e-3) + pred=before['detections'][0].T;index=np.argmax(pred[:,4:].max(-1));box=pred[index,:4]/np.array([w,h,w,h]);label=int(np.argmax(pred[index,4:])) +elif kind=='rtmdet': + bb=[];ss=[] + for stride in (8,16,32): + feature=raw[source['bindings'][f'features_{stride}']];yy,xx=np.mgrid[:feature.shape[1],:feature.shape[2]];grid=np.stack([xx,yy],-1)*stride + d=feature[:,:,:,C:]*stride;lo=grid[None]-d[:,:,:,:2];hi=grid[None]+d[:,:,:,2:] + bb.append((np.concatenate([(lo+hi)/2,hi-lo],-1)/np.array([w,h,w,h])).reshape(1,-1,4));ss.append(feature[:,:,:,:C].reshape(1,-1,C)) + np.testing.assert_allclose(before['boxes'],np.concatenate(bb,1),atol=1e-4,rtol=1e-4);np.testing.assert_allclose(before['logits'],np.concatenate(ss,1),atol=1e-5,rtol=1e-5) + index=np.argmax(before['logits'][0].max(-1));box=before['boxes'][0,index];label=int(np.argmax(before['logits'][0,index])) +else: + np.testing.assert_allclose(before['boxes'],raw[source['bindings']['features_boxes']],atol=1e-5,rtol=1e-5) + np.testing.assert_allclose(before['logits'],raw[source['bindings']['features_logits']],atol=1e-5,rtol=1e-5) + candidates=before['logits'][0][:,class_indices];index=np.argmax(candidates.max(-1));box=before['boxes'][0,index];label=int(np.argmax(candidates[index])) +y=np.zeros((1,100,6),np.float32);y[0,0,:4]=np.clip(np.r_[box[:2]-box[2:]/2,box[:2]+box[2:]/2],0,1);y[0,0,4:]=[label,1] +losses=[] +for _ in range(8):losses.append(float(s['train'](x=x,y=y,learning_rate=np.float32(.005))['loss'])) +assert np.isfinite(losses).all() and losses[-1]=0);ops.append(op) +g.operators=list(reversed(ops));g.outputs=np.array(indices,np.int32) +for s in m.signatureDefs or []: + s.outputs=[] + for name,index in zip(['logits','boxes'],indices): + t=schema.TensorMapT();t.name=name.encode();t.tensorIndex=index;s.outputs.append(t) +m.metadata=None;m.metadataBuffer=None +b=flatbuffers.Builder(1024);b.Finish(m.Pack(b),file_identifier=b'TFL3');p=r/'simplified-converted/model-cropped.tflite';p.write_bytes(b.Output()) +print(json.dumps({'output':str(p),'tensors':indices,'ops':len(ops)})) diff --git a/conversion/dfine-arena-protection.json b/conversion/dfine-arena-protection.json new file mode 100644 index 0000000000000000000000000000000000000000..ed9ceac2ae0fb8a246072de1a9b154ddae14439b --- /dev/null +++ b/conversion/dfine-arena-protection.json @@ -0,0 +1,36 @@ +{ + "tensors": [ + { + "index": 1354, + "name": "wa/model/decoder/decoder/integral/Softmax_output_0", + "shape": [ + 1200, + 33 + ] + }, + { + "index": 1857, + "name": "wa/model/decoder/decoder/integral_1/Softmax_output_0", + "shape": [ + 1200, + 33 + ] + }, + { + "index": 2338, + "name": "wa/model/decoder/decoder/integral_2/Softmax_output_0", + "shape": [ + 1200, + 33 + ] + }, + { + "index": 2816, + "name": "wa/model/decoder/decoder/integral_3/Softmax_output_0", + "shape": [ + 1200, + 33 + ] + } + ] +} diff --git a/conversion/dynamic_hf_worker.py b/conversion/dynamic_hf_worker.py new file mode 100644 index 0000000000000000000000000000000000000000..14f117537ea92f7b09f0eaff10bdf460a8c3eaf8 --- /dev/null +++ b/conversion/dynamic_hf_worker.py @@ -0,0 +1,101 @@ +import os,sys,re,json,shutil +from pathlib import Path +os.environ.update(VDS_BATCH2_ROOT='/workspace/vds-litert-batch2',HF_TOKEN_FILE='/workspace/.secrets/hf_token',HF_FIREVIEWER_DEST_REPO='fireviewer/litert-models',TORCH_NUM_THREADS='8',HF_HOME='/workspace/cache/huggingface') +import torch,numpy as np,litert_torch +from ai_edge_litert.interpreter import Interpreter,OpResolverType +from huggingface_hub import HfApi +import dynamic_lowerings +root=Path('/workspace/vds-litert-batch2');sys.path.insert(0,str(root/'package/scripts')) +import convert_batch2 as worker +worker.WORK=root/'dynamic-work' +key=sys.argv[1] +spec=next((s for s in worker.all_specs() if s['key']==key),None) +if spec is None: + import previous_native + old=next(s for s in json.loads((root/'previous-models.json').read_text()) if s['key']==key) + spec={**old,'group':'general_models','route':key,'title':key,'task':'vision embeddings' if key=='dinov2' else 'pose estimation'} + worker.CONV[key]=lambda s,e,o:getattr(previous_native,key)(s,e,o,worker) + original_prefetched=worker.prefetched + worker.prefetched=lambda k:json.loads((root/'previous-prefetch.json').read_text())[k] if k==key else original_prefetched(k) +validation={} +adapter_sha256=worker.sha256(root/'dynamic_lowerings.py') + +def dynamic_export(module,args,out): + global validation + module=module.cpu().eval();args=tuple(a.cpu() for a in args) + for sub in module.modules(): + if hasattr(sub,'set_attn_implementation'): + try:sub.set_attn_implementation('eager') + except (ValueError,AttributeError):pass + h,w=args[0].shape[-2:] + stride=14 if spec['route'] in {'depth','dinov2'} else 16 if spec['route']=='dinov3_fireviewer' else 32 + if h%stride or w%stride:stride=1 + height=stride*torch.export.Dim('height_blocks',min=4,max=128) if stride>1 else torch.export.Dim('height',min=128,max=2048) + width=stride*torch.export.Dim('width_blocks',min=4,max=128) if stride>1 else torch.export.Dim('width',min=128,max=2048) + dims=[{} for a in args];dims[0]={2:height,3:width} + masks=[] + for i,a in enumerate(args[1:],1): + if a.ndim==3 and tuple(a.shape[-2:])==(h,w):dims[i]={1:height,2:width};masks.append(i) + shapes=[(h,w),(h-2*stride,w+stride),(h+stride,w-2*stride)] + def inputs(hh,ww): + aa=list(args);aa[0]=torch.from_numpy(np.random.default_rng(42).normal(size=(1,3,hh,ww)).astype(np.float32)) + for i in masks:aa[i]=torch.ones((1,hh,ww),dtype=args[i].dtype) + return tuple(aa) + with torch.inference_mode(): + for hh,ww in shapes:module(*inputs(hh,ww)) + if os.environ.get('REUSE_ARTIFACT'): + shutil.copy2(os.environ['REUSE_ARTIFACT'],out) + else: + converted=litert_torch.convert(module,args,dynamic_shapes=tuple(dims)) + out.parent.mkdir(parents=True,exist_ok=True);converted.export(str(out));del converted + interp=Interpreter(model_path=str(out),num_threads=8,experimental_op_resolver_type=OpResolverType.BUILTIN_WITHOUT_DEFAULT_DELEGATES) + details=interp.get_input_details();mapping={} + for j,d in enumerate(details): + match=re.search(r'args_(\d+)',d['name']);index=int(match.group(1)) if match else j + mapping[index]=d + if mapping[0]['shape_signature'].tolist()[2:]!=[-1,-1]:raise RuntimeError('Spatial signature was fixed by converter') + checks=[] + for hh,ww in shapes: + aa=inputs(hh,ww) + for i,a in enumerate(aa):interp.resize_tensor_input(mapping[i]['index'],tuple(a.shape),strict=True) + interp.allocate_tensors() + for i,a in enumerate(aa):interp.set_tensor(mapping[i]['index'],a.numpy()) + interp.invoke() + with torch.inference_mode():ref=module(*aa) + if isinstance(ref,torch.Tensor):ref=(ref,) + refs=[v.detach().numpy() for v in ref] + errors=[] + for j,d in enumerate(interp.get_output_details()): + a=interp.get_tensor(d['index']);candidates=[r for r in refs if r.shape==a.shape] + if len(candidates)==1:r=candidates[0] + else: + match=re.search(r'output_(\d+)',d['name']) + if not match:raise RuntimeError(f'Ambiguous output mapping for {d["name"]} {a.shape}') + r=refs[int(match.group(1))] + if r.shape!=a.shape:raise RuntimeError('Output signature mismatch');np.testing.assert_allclose(a,r,rtol=1e-3,atol=1e-3) + if not np.isfinite(a).all():raise RuntimeError('Nonfinite output') + errors.append(float(np.max(np.abs(a-r)))) + checks.append({'image_shape':[1,3,hh,ww],'max_abs_errors':errors}) + print('DYNAMIC_PARITY_OK',key,hh,ww,errors,flush=True) + validation={'dynamic_validated':True,'checks':checks,'atol':1e-3,'rtol':1e-3,'alignment':stride,'source_resolution_resampled':False,'runtime_requirement':'LiteRT builtin CPU; disable default XNNPACK delegates for these dynamic graphs','android_tested':False,'accuracy_evaluated':False,'conversion_adapter_sha256':adapter_sha256} + return {**worker.inspect_tflite(out),**validation} + +base_publish=worker.publish +def publish(api,s,e,out,arts,report): + if not validation.get('dynamic_validated'):raise RuntimeError('Dynamic validation gate did not pass') + report.update(validation) + return base_publish(api,s,e,out,arts,report) + +worker.direct_export=dynamic_export;worker.publish=publish +if spec['route']=='rfdetr_medium': + from rfdetr.export._litert.exporter import LiteRTExporter + def patched_convert(self,litert,wrapped_model,input_tensors,output_file): + dynamic_export(wrapped_model,(input_tensors,),Path(output_file)) + LiteRTExporter._convert_and_save=patched_convert +if key=='fireviewer_rfdetr_medium_v110' and os.environ.get('RF_NATIVE')=='1': + import rfdetr_native + worker.CONV['rfdetr_medium']=lambda s,e,o:rfdetr_native.convert(s,e,o,worker) +if key=='fireviewer_dfine_m_strict_v1': + import dfine_native + worker.CONV['onnx_existing']=lambda s,e,o:dfine_native.convert(s,e,o,worker) +worker.process(HfApi(token=worker.token()),spec) diff --git a/conversion/dynamic_lowerings.py b/conversion/dynamic_lowerings.py new file mode 100644 index 0000000000000000000000000000000000000000..4f16bae7f2456997ab89dbc21fe763635bbec9bd --- /dev/null +++ b/conversion/dynamic_lowerings.py @@ -0,0 +1,161 @@ +"""Native LiteRT lowerings for shape-dependent reductions and shape arithmetic.""" +import operator +import torch +from litert_torch import fx_infra +from litert_torch.backend.experimental import torch_tfl +from litert_torch.backend.experimental.torch_tfl._lowerings import _ir_operation +from litert_torch.backend.lowerings import registry,utils +from litert_converter.mlir import ir + +fx_infra.decomp.update_pre_lower_decomp(torch_tfl.decomps) + +def dynamic_layer_norm(x,normalized_shape,weight,bias,eps): + axes=list(range(x.ndim-len(normalized_shape),x.ndim)) + mean=x.mean(axes,keepdim=True) + variance=((x-mean)*(x-mean)).mean(axes,keepdim=True) + invstd=torch.rsqrt(variance+eps) + output=(x-mean)*invstd + if weight is not None:output=output*weight + if bias is not None:output=output+bias + return output,mean,invstd + +fx_infra.decomp.update_pre_lower_decomp({torch.ops.aten.native_layer_norm.default:dynamic_layer_norm}) + +@registry.lower(operator.floordiv) +def lower_floor_div(lctx,lhs,rhs): + if isinstance(lhs,int) and isinstance(rhs,int):return lhs//rhs + if isinstance(lhs,int):lhs=utils.splat(lhs,rhs.type.element_type,rhs.type.shape) + if isinstance(rhs,int):rhs=utils.splat(rhs,lhs.type.element_type,lhs.type.shape) + if lhs.type!=rhs.type:raise TypeError('Shape floor division requires matching scalar integer types') + return _ir_operation('tfl.floor_div',results=[lhs.type],operands=[lhs,rhs]) + +@registry.lower(torch.ops.aten.relu.default) +def lower_relu(lctx,x): + return _ir_operation('tfl.relu',results=utils.node_meta_to_ir_types(lctx.node),operands=[x]) + +@registry.lower(torch.ops.aten.cumsum.default) +def lower_cumsum(lctx,x,dim,dtype=None): + result=utils.node_meta_to_ir_types(lctx.node) + if x.type.element_type!=result[0].element_type: + typ=ir.RankedTensorType.get(x.type.shape,result[0].element_type) + x=_ir_operation('tfl.cast',results=[typ],operands=[x]) + axis=utils.splat(dim,ir.IntegerType.get_signless(32),[]) + return _ir_operation('tfl.cumsum',results=result,operands=[x,axis],attributes={'exclusive':ir.BoolAttr.get(False),'reverse':ir.BoolAttr.get(False)}) + +@registry.lower(torch.ops.aten._is_all_true.default) +def lower_all_true(lctx,x): + if len(x.type.shape)==0:return x + import numpy as np + axes=utils.numpy_array_constant(np.arange(len(x.type.shape),dtype=np.int32)) + return _ir_operation('tfl.reduce_all',results=utils.node_meta_to_ir_types(lctx.node),operands=[x,axes],attributes={'keep_dims':ir.BoolAttr.get(False)}) + +# Preserve symbolic dimensions in the operator schema. The upstream int[] +# schema can specialize reshape dimensions during decomposition. +@torch.library.custom_op('vds::reshape',mutates_args=(),schema='(Tensor x, SymInt[] shape) -> Tensor') +def dynamic_reshape(x,shape): + return x.reshape(shape).clone() +@dynamic_reshape.register_fake +def dynamic_reshape_fake(x,shape): + return x.reshape(shape).clone() +@registry.lower(torch.ops.vds.reshape.default) +def lower_reshape(lctx,x,shape): + return _ir_operation('tfl.reshape',results=utils.node_meta_to_ir_types(lctx.node),operands=[x,utils.convert_shape_to_ir_value(shape)]) + +def dynamic_view(x,shape):return torch.ops.vds.reshape(x,shape) +fx_infra.decomp.update_pre_lower_decomp({torch.ops.aten.view.default:dynamic_view,torch.ops.aten._unsafe_view.default:dynamic_view}) + +def dynamic_arange(start,end,step=1,*,dtype=None,layout=None,device=None,pin_memory=None): + result=torch.ops.tfl.range(start,end,step) + return result.to(dtype=dtype) if dtype is not None else result +fx_infra.decomp.update_pre_lower_decomp({torch.ops.aten.arange.start_step:dynamic_arange}) + +@torch.library.custom_op('vds::broadcast_to',mutates_args=(),schema='(Tensor x, SymInt[] shape) -> Tensor') +def dynamic_broadcast(x,shape):return x.expand(shape).clone() +@dynamic_broadcast.register_fake +def dynamic_broadcast_fake(x,shape):return x.expand(shape).clone() +@registry.lower(torch.ops.vds.broadcast_to.default) +def lower_broadcast(lctx,x,shape): + return _ir_operation('tfl.broadcast_to',results=utils.node_meta_to_ir_types(lctx.node),operands=[x,utils.convert_shape_to_ir_value(shape)]) +fx_infra.decomp.update_pre_lower_decomp({torch.ops.aten.expand.default:lambda x,shape,implicit=False:torch.ops.vds.broadcast_to(x,shape)}) + +@torch.library.custom_op('vds::linear',mutates_args=()) +def dynamic_linear(x:torch.Tensor,weight:torch.Tensor,bias:torch.Tensor|None)->torch.Tensor: + return torch.nn.functional.linear(x,weight,bias) +@dynamic_linear.register_fake +def dynamic_linear_fake(x,weight,bias): + return x.new_empty(x.shape[:-1] if weight.ndim==1 else (*x.shape[:-1],weight.shape[0])) +@registry.lower(torch.ops.vds.linear.default) +def lower_linear(lctx,x,weight,bias): + if len(weight.type.shape)==1: + from litert_converter.mlir.dialects import stablehlo + from litert_torch.backend.experimental.torch_tfl._lowerings import _tfl_squeeze_lowering + weight=stablehlo.reshape(ir.RankedTensorType.get([1,weight.type.shape[0]],weight.type.element_type),weight) + if bias is None:bias=utils.splat(0.0,weight.type.element_type,[1]) + typ=utils.node_meta_to_ir_types(lctx.node)[0] + expanded=ir.RankedTensorType.get([*typ.shape,1],typ.element_type) + y=_ir_operation('tfl.fully_connected',results=[expanded],operands=[x,weight,bias],attributes={'fused_activation_function':ir.StringAttr.get('NONE'),'weights_format':ir.StringAttr.get('DEFAULT'),'keep_num_dims':ir.BoolAttr.get(True),'asymmetric_quantize_inputs':ir.BoolAttr.get(False)}) + return _tfl_squeeze_lowering(lctx,y,[-1]) + if bias is None:bias=utils.splat(0.0,weight.type.element_type,[weight.type.shape[0]]) + return _ir_operation('tfl.fully_connected',results=utils.node_meta_to_ir_types(lctx.node),operands=[x,weight,bias],attributes={'fused_activation_function':ir.StringAttr.get('NONE'),'weights_format':ir.StringAttr.get('DEFAULT'),'keep_num_dims':ir.BoolAttr.get(True),'asymmetric_quantize_inputs':ir.BoolAttr.get(False)}) +fx_infra.decomp.update_pre_lower_decomp({torch.ops.aten.linear.default:lambda x,weight,bias=None:torch.ops.vds.linear(x,weight,bias)}) + +fx_infra.decomp.update_pre_convert_decomp({ + torch.ops.aten.linear.default:lambda x,weight,bias=None:torch.ops.vds.linear(x,weight,bias), + torch.ops.aten.view.default:dynamic_view, + torch.ops.aten._unsafe_view.default:dynamic_view, + torch.ops.aten.native_layer_norm.default:dynamic_layer_norm, + torch.ops.aten.arange.start_step:dynamic_arange, + torch.ops.aten.expand.default:lambda x,shape,implicit=False:torch.ops.vds.broadcast_to(x,shape), +}) + +@torch.library.custom_op('vds::resize2d',mutates_args=(),schema='(Tensor x, SymInt[] size, bool align_corners, bool nearest) -> Tensor') +def dynamic_resize2d(x,size,align_corners,nearest): + if nearest:return torch.nn.functional.interpolate(x,size=size,mode='nearest') + return torch.nn.functional.interpolate(x,size=size,mode='bilinear',align_corners=align_corners) +@dynamic_resize2d.register_fake +def dynamic_resize2d_fake(x,size,align_corners,nearest):return x.new_empty((x.shape[0],x.shape[1],*size)) +@registry.lower(torch.ops.vds.resize2d.default) +def lower_resize2d(lctx,x,size,align_corners,nearest): + from litert_converter.mlir.dialects import stablehlo + nhwc=stablehlo.transpose(x,[0,2,3,1]) + result=utils.node_meta_to_ir_types(lctx.node)[0] + typ=ir.RankedTensorType.get([result.shape[i] for i in [0,2,3,1]],result.element_type) + resized=_ir_operation('tfl.resize_nearest_neighbor' if nearest else 'tfl.resize_bilinear',results=[typ],operands=[nhwc,utils.convert_shape_to_ir_value(size)],attributes={'align_corners':ir.BoolAttr.get(align_corners),'half_pixel_centers':ir.BoolAttr.get(not align_corners and not nearest)}) + return stablehlo.transpose(resized,[0,3,1,2]) +def bilinear_vec(x,output_size,align_corners,scale_factors=None): + size=output_size if output_size is not None else [int(x.shape[-2+i]*scale_factors[i]) for i in range(2)] + return torch.ops.vds.resize2d(x,size,align_corners,False) +def bilinear_default(x,output_size,align_corners,scales_h=None,scales_w=None): + return torch.ops.vds.resize2d(x,output_size,align_corners,False) +fx_infra.decomp.update_pre_convert_decomp({torch.ops.aten.upsample_bilinear2d.vec:bilinear_vec,torch.ops.aten.upsample_bilinear2d.default:bilinear_default}) + +from litert_torch.backend.experimental.torch_tfl._lowerings import _tfl_mul_lowering +@registry.lower(torch.ops.tfl.mul.default) +def lower_bool_mul(lctx,lhs,rhs,fused_activation_function='NONE'): + if isinstance(lhs,ir.Value) and isinstance(rhs,ir.Value) and str(lhs.type.element_type)=='i1' and str(rhs.type.element_type)=='i1': + return _ir_operation('tfl.logical_and',results=utils.node_meta_to_ir_types(lctx.node),operands=[lhs,rhs]) + return _tfl_mul_lowering(lctx,lhs,rhs,fused_activation_function) + +@torch.library.custom_op('vds::strided_slice',mutates_args=(),schema='(Tensor x, SymInt[] begin, SymInt[] end, int[] strides) -> Tensor') +def dynamic_strided_slice(x,begin,end,strides): + return x[tuple(slice(b,e,s) for b,e,s in zip(begin,end,strides))].clone() +@dynamic_strided_slice.register_fake +def dynamic_strided_slice_fake(x,begin,end,strides): + return x[tuple(slice(b,e,s) for b,e,s in zip(begin,end,strides))].clone() +@registry.lower(torch.ops.vds.strided_slice.default) +def lower_strided_slice(lctx,x,begin,end,strides): + from litert_torch.backend.experimental.torch_tfl._lowerings import _tfl_strided_slice_lowering + return _tfl_strided_slice_lowering(lctx,x,begin,end,strides) +def dynamic_slice(x,dim=0,start=None,end=None,step=1): + n=x.shape[dim];b=0 if start is None else start;e=n if end is None else end + if b<0:b+=n + if e<0:e+=n + b=max(0,min(b,n));e=max(b,min(e,n)) + begin=[0]*x.ndim;ends=list(x.shape);strides=[1]*x.ndim + begin[dim]=b;ends[dim]=e;strides[dim]=step + return torch.ops.vds.strided_slice(x,begin,ends,strides) +def dynamic_embedding(weight,indices,padding_idx=-1,scale_grad_by_freq=False,sparse=False): + flat=torch.ops.vds.reshape(indices,[-1]).to(torch.int32) + result=torch.ops.tfl.embedding_lookup(flat,weight) + return torch.ops.vds.reshape(result,[*indices.shape,weight.shape[-1]]) +fx_infra.decomp.update_pre_convert_decomp({torch.ops.aten.slice.Tensor:dynamic_slice,torch.ops.aten.embedding.default:dynamic_embedding}) diff --git a/conversion/finish_fire_backbone.py b/conversion/finish_fire_backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..7ba423b73f05b736caa6a444eec806f11c4e1432 --- /dev/null +++ b/conversion/finish_fire_backbone.py @@ -0,0 +1,61 @@ +"""Fixed internal image grid, checked against upstream, for Android adaptation.""" +import os,sys,json,traceback +os.environ.update(TORCH_NUM_THREADS='6') +from pathlib import Path +import torch,numpy as np,litert_torch +import static_topk_lowering +from ai_edge_litert.interpreter import Interpreter,OpResolverType +root=Path('/workspace/vds-litert-batch2');key=sys.argv[1];out=root/'learning-backbones'/key/'final';out.mkdir(parents=True,exist_ok=True) +torch.set_num_threads(6);source=root/'models/upstream'/key +try: + if 'rtdetr_v2' in key: + from transformers import AutoModelForObjectDetection + import transformers.models.rt_detr_v2.modeling_rt_detr_v2 as implementation + net=AutoModelForObjectDetection.from_pretrained(source,local_files_only=True,attn_implementation='eager').eval();size=768 + class Wrapped(torch.nn.Module): + def __init__(self):super().__init__();self.m=net + def forward(self,x): + o=self.m(pixel_values=x);return o.logits,o.pred_boxes + else: + sys.path.insert(0,str(root/'vendor/D-FINE'));from src.core import YAMLConfig + cfg=YAMLConfig(str(root/'vendor/D-FINE/configs/dfine/dfine_hgnetv2_m_coco.yml')) + cfg.yaml_cfg.update(num_classes=2,remap_mscoco_category=False,eval_spatial_size=[704,704]);cfg.yaml_cfg['HGNetv2']['pretrained']=False + net=cfg.model;checkpoint=torch.load(source/'model.pth',map_location='cpu',weights_only=False) + state=checkpoint['ema']['module'] if 'ema' in checkpoint else checkpoint.get('model',checkpoint);net.load_state_dict(state,strict=True);net=net.eval().deploy();size=704 + class Wrapped(torch.nn.Module): + def __init__(self):super().__init__();self.m=net + def forward(self,x): + o=self.m(x);return o['pred_logits'],o['pred_boxes'] + wrapped=Wrapped().eval();x=np.random.default_rng(842).uniform(0,1,(1,3,size,size)).astype(np.float32) + with torch.no_grad():reference=[t.numpy() for t in wrapped(torch.from_numpy(x))] + if 'rtdetr_v2' in key: + # The upstream runtime assertion checks only the spatial grid versus sequence + # length. It was executed above; the serialized grid here is immutable. + implementation.torch_compilable_check=lambda condition,message:None + # Position embeddings depend only on the fixed feature grid. Materialize + # the original float64 calculation before export, preserving its values. + class FixedPosition(torch.nn.Module): + def __init__(self, original): + super().__init__(); self.original=original; self.cache={} + def forward(self,width,height,device,dtype): + shape=(int(width),int(height)) + return getattr(self, 'grid_'+str(shape[0])+'_'+str(shape[1])).to(device=device,dtype=dtype) + for parent in list(net.modules()): + for name,child in list(parent.named_children()): + if isinstance(child,implementation.RTDetrV2SinePositionEmbedding): + fixed=FixedPosition(child) + for stride in (8,16,32): + grid=size//stride + fixed.register_buffer('grid_'+str(grid)+'_'+str(grid),child(grid,grid,device=torch.device('cpu'),dtype=torch.float32)) + setattr(parent,name,fixed) + artifact=out/'model.tflite' + with torch.no_grad():edge=litert_torch.convert(wrapped,(torch.zeros(1,3,size,size),)) + edge.export(str(artifact));del edge + i=Interpreter(model_path=str(artifact),num_threads=6,experimental_op_resolver_type=OpResolverType.BUILTIN_WITHOUT_DEFAULT_DELEGATES);i.allocate_tensors();i.set_tensor(i.get_input_details()[0]['index'],x);i.invoke();errors=[] + for d in i.get_output_details(): + a=i.get_tensor(d['index']);r=next(r for r in reference if r.shape==a.shape);np.testing.assert_allclose(a,r,atol=2e-3,rtol=2e-3);errors.append(float(np.max(np.abs(a-r)))) + report={'key':key,'phase':'validated_frozen_component','backbone':str(artifact),'source_pytorch_parity_passed':True,'input_shape':list(x.shape),'outputs':[{'shape':list(map(int,d['shape'])),'index':n} for n,d in enumerate(i.get_output_details())],'max_abs_errors':errors,'atol':2e-3,'rtol':2e-3,'fixed_shape_assertion_checked_before_export':'rtdetr_v2' in key} + (out/'parity.json').write_text(json.dumps(report,indent=2)+'\n') +except Exception as e: + report={'key':key,'phase':'failed','error':str(e),'traceback':traceback.format_exc()};print(report['traceback'],flush=True) +(out/'status.json').write_text(json.dumps(report,indent=2)+'\n');print(json.dumps(report),flush=True) diff --git a/conversion/merge_training_graph.py b/conversion/merge_training_graph.py new file mode 100644 index 0000000000000000000000000000000000000000..60de31a518ef8dee2dfb57f3e4d7417b1edb4a5f --- /dev/null +++ b/conversion/merge_training_graph.py @@ -0,0 +1,118 @@ +"""Join a frozen, single-subgraph visual encoder to mutable training signatures. + +Weights remain shared buffers; inference operators are copied into train/infer. +No Python callback, custom runtime, or remote computation is used on Android. +""" +import copy +import flatbuffers +import numpy as np +from tensorflow.lite.python import schema_py_generated as schema + + +def freeze_auxiliary_inputs(backbone_path,destination,values): + model=schema.ModelT.InitFromObj(schema.Model.GetRootAsModel(backbone_path.read_bytes(),0)) + graph=model.subgraphs[0];removed=[] + for position,fill in values.items(): + index=int(graph.inputs[int(position)]);tensor=graph.tensors[index] + dtype={schema.TensorType.INT32:np.int32,schema.TensorType.INT64:np.int64,schema.TensorType.FLOAT32:np.float32}[tensor.type] + data=np.full(tensor.shape,fill,dtype=dtype) + buffer=schema.BufferT();buffer.data=np.frombuffer(data.tobytes(),np.uint8) + tensor.buffer=len(model.buffers);model.buffers.append(buffer);removed.append(index) + graph.inputs=np.array([i for i in graph.inputs if i not in removed],np.int32) + for signature in model.signatureDefs or []:signature.inputs=[t for t in signature.inputs if t.tensorIndex not in removed] + model.metadata=None;model.metadataBuffer=None + builder=flatbuffers.Builder(1024);builder.Finish(model.Pack(builder),file_identifier=b'TFL3');destination.write_bytes(builder.Output()) + + +def dynamic_resize_backbone(backbone_path,destination): + """Explicit stretch preprocessing for architectures whose token grid is fixed.""" + import tensorflow as tf + backbone=schema.ModelT.InitFromObj(schema.Model.GetRootAsModel(backbone_path.read_bytes(),0)) + assert len(backbone.subgraphs)==1 and len(backbone.subgraphs[0].inputs)==1 + graph=backbone.subgraphs[0];old_input=int(graph.inputs[0]);shape=list(map(int,graph.tensors[old_input].shape)) + assert shape[0:2]==[1,3] + h,w=map(int,shape[2:]) + class Resize(tf.Module): + @tf.function(input_signature=[tf.TensorSpec([1,3,None,None],tf.float32,name='x')]) + def run(self,x):return tf.transpose(tf.image.resize(tf.transpose(x,[0,2,3,1]),[h,w]),[0,3,1,2]) + resize=Resize();converter=tf.lite.TFLiteConverter.from_concrete_functions([resize.run.get_concrete_function()],resize) + prefix=schema.ModelT.InitFromObj(schema.Model.GetRootAsModel(converter.convert(),0));p=prefix.subgraphs[0] + offset=len(graph.tensors);bo=len(backbone.buffers);oo=len(backbone.operatorCodes) + for t in p.tensors:t.buffer+=bo + graph.tensors.extend(p.tensors);backbone.buffers.extend(prefix.buffers);backbone.operatorCodes.extend(prefix.operatorCodes) + resized=int(p.outputs[0])+offset;new_input=int(p.inputs[0])+offset + for op in graph.operators: + if op.inputs is not None:op.inputs=np.array([resized if i==old_input else i for i in op.inputs],np.int32) + for op in p.operators: + op.opcodeIndex+=oo + for field in ('inputs','outputs','intermediates'): + values=getattr(op,field) + if values is not None:setattr(op,field,np.array([int(i)+offset if i>=0 else -1 for i in values],np.int32)) + graph.operators=p.operators+graph.operators;graph.inputs=np.array([new_input],np.int32) + graph.tensors[new_input].shape=np.array(shape,np.int32) + for s in backbone.signatureDefs or []: + for t in s.inputs: + if t.tensorIndex==old_input:t.tensorIndex=new_input + backbone.metadata=None;backbone.metadataBuffer=None + builder=flatbuffers.Builder(1024);builder.Finish(backbone.Pack(builder),file_identifier=b'TFL3');destination.write_bytes(builder.Output()) + return {'mode':'bilinear_stretch_inside_graph','native_backbone_shape':shape} + + +def merge(backbone_path, head_path, destination, output_index, image_size=224): + backbone = schema.ModelT.InitFromObj(schema.Model.GetRootAsModel(backbone_path.read_bytes(), 0)) + head = schema.ModelT.InitFromObj(schema.Model.GetRootAsModel(head_path.read_bytes(), 0)) + assert len(backbone.subgraphs) == 1, 'Nested backbone control flow needs explicit remapping' + source = backbone.subgraphs[0] + assert len(source.inputs) == 1 + assert not any(t.isVariable for t in source.tensors), 'Backbone must be frozen' + for op in source.operators: + assert op.mutatingVariableInputs is None or not any(op.mutatingVariableInputs) + assert backbone.operatorCodes[op.opcodeIndex].builtinCode not in [schema.BuiltinOperator.IF, schema.BuiltinOperator.WHILE, schema.BuiltinOperator.CALL_ONCE] + buffer_offset = len(head.buffers) + head.buffers.extend(copy.deepcopy(backbone.buffers)) + opcode_offset = len(head.operatorCodes) + head.operatorCodes.extend(copy.deepcopy(backbone.operatorCodes)) + for signature in head.signatureDefs: + name = signature.signatureKey.decode() + if name not in ('infer', 'train'): + continue + graph = head.subgraphs[signature.subgraphIndex] + bindings = output_index if isinstance(output_index,dict) else {'features':output_index} + feature_maps=[t for t in signature.inputs if t.name.decode() in bindings] + assert len(feature_maps)==len(bindings) + offset = len(graph.tensors) + tensors = copy.deepcopy(source.tensors) + for tensor in tensors: + tensor.buffer += buffer_offset + graph.tensors.extend(tensors) + image_input = int(source.inputs[0]) + offset + sig=list(graph.tensors[image_input].shapeSignature) + if len(sig)==4 and sig[1:]==[3,-1,-1]: + hw=image_size if isinstance(image_size,(list,tuple)) else [image_size,image_size] + graph.tensors[image_input].shape=np.array([1,3,*hw],dtype=np.int32) + replacements={t.tensorIndex:int(source.outputs[bindings[t.name.decode()]])+offset for t in feature_maps} + image_maps=[t for t in signature.inputs if t.name==b'image'] + replacements.update({t.tensorIndex:image_input for t in image_maps}) + for op in graph.operators: + if op.inputs is not None: + op.inputs = np.array([replacements.get(int(i),int(i)) for i in op.inputs], dtype=np.int32) + ops = copy.deepcopy(source.operators) + for op in ops: + op.opcodeIndex += opcode_offset + for field in ('inputs', 'outputs', 'intermediates'): + values = getattr(op, field) + if values is not None: + setattr(op, field, np.array([int(i)+offset if i >= 0 else -1 for i in values], dtype=np.int32)) + graph.operators = ops + graph.operators + removed={t.tensorIndex for t in feature_maps+image_maps} + graph.inputs = np.array([image_input]+[int(i) for i in graph.inputs if i not in removed], dtype=np.int32) + feature_map=feature_maps[0] + signature.inputs=[t for t in signature.inputs if t.tensorIndex not in removed]+[feature_map] + feature_map.tensorIndex = image_input + feature_map.name = b'x' + # Converter metadata from either graph does not describe the combined model. + head.metadata = None + head.metadataBuffer = None + builder = flatbuffers.Builder(1024) + builder.Finish(head.Pack(builder), file_identifier=b'TFL3') + destination.write_bytes(builder.Output()) diff --git a/conversion/package/model_manifest.json b/conversion/package/model_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..a90f72f5abe180391c6d62a73218054ff41a2a6f --- /dev/null +++ b/conversion/package/model_manifest.json @@ -0,0 +1,29 @@ +{ + "destination_repo": "Charlbi/Lite_rt_prepared_for_android_dataset_builder", + "general_models": [ + {"key":"grounding_dino_base","title":"Grounding DINO Base","repo":"IDEA-Research/grounding-dino-base","license":"apache-2.0","route":"grounding_dino","task":"open-vocabulary object detection"}, + {"key":"owlv2_base_patch16","title":"OWLv2 Base Patch16 Ensemble","repo":"google/owlv2-base-patch16-ensemble","license":"apache-2.0","route":"owlv2","task":"zero-shot object detection"}, + {"key":"depth_anything_v2_small","title":"Depth Anything V2 Small","repo":"depth-anything/Depth-Anything-V2-Small-hf","license":"apache-2.0","route":"depth","task":"monocular depth estimation"}, + {"key":"rtmdet_tiny","title":"RTMDet Tiny","repo":null,"license":"apache-2.0","route":"rtmdet","task":"object detection","checkpoint_url":"https://download.openmmlab.com/mmyolo/v0/rtmdet/rtmdet_tiny_syncbn_fast_8xb32-300e_coco/rtmdet_tiny_syncbn_fast_8xb32-300e_coco_20230102_140117-dbb1dc83.pth","config":"configs/rtmdet/rtmdet_tiny_syncbn_fast_8xb32-300e_coco.py"}, + {"key":"efficientformer_l1","title":"EfficientFormer L1","repo":"timm/efficientformer_l1.snap_dist_in1k","license":"apache-2.0","route":"timm","task":"classification + visual embeddings"}, + {"key":"repvit_m1","title":"RepViT M1","repo":"timm/repvit_m1.dist_in1k","license":"apache-2.0","route":"timm","task":"classification + visual embeddings"}, + {"key":"edgenext_xx_small","title":"EdgeNeXt XX-Small","repo":"timm/edgenext_xx_small.in1k","license":"mit","route":"timm","task":"classification + visual embeddings"}, + {"key":"edgenext_x_small","title":"EdgeNeXt X-Small","repo":"timm/edgenext_x_small.in1k","license":"mit","route":"timm","task":"classification + visual embeddings"}, + {"key":"edgenext_small_usi","title":"EdgeNeXt Small USI","repo":"timm/edgenext_small.usi_in1k","license":"mit","route":"timm","task":"classification + visual embeddings"}, + {"key":"hgnetv2_b0","title":"HGNetV2 B0","repo":"timm/hgnetv2_b0.ssld_stage2_ft_in1k","license":"apache-2.0","route":"timm","task":"classification + visual embeddings"}, + {"key":"caformer_s18","title":"CAFormer S18","repo":"timm/caformer_s18.sail_in1k","license":"apache-2.0","route":"timm","task":"classification + visual embeddings"}, + {"key":"convformer_s18","title":"ConvFormer S18","repo":"timm/convformer_s18.sail_in1k","license":"apache-2.0","route":"timm","task":"classification + visual embeddings"}, + {"key":"picodet_layout_1x","title":"PicoDet Layout 1x","repo":"PaddlePaddle/PicoDet_layout_1x","license":"apache-2.0","route":"picodet","task":"document layout detection"}, + {"key":"picodet_l_layout_3cls","title":"PicoDet-L Layout 3cls","repo":"PaddlePaddle/PicoDet-L_layout_3cls","license":"apache-2.0","route":"picodet","task":"document layout detection"}, + {"key":"table_transformer_detection","title":"Table Transformer Detection","repo":"microsoft/table-transformer-detection","license":"mit","route":"object_detection","task":"table detection"}, + {"key":"table_transformer_structure","title":"Table Transformer Structure Recognition","repo":"microsoft/table-transformer-structure-recognition","license":"mit","route":"object_detection","task":"table structure recognition"} + ], + "fireviewer_models": [ + {"key":"fireviewer_dfine_m_strict_v1","title":"FireViewer D-FINE-M strict v1","repo":"fireviewer/fire-smoke-dfine-m-strict-v1","license":"source-specific","route":"object_detection","task":"fire/smoke detection","publish_default":true}, + {"key":"fireviewer_rtdetr_v2_r50","title":"FireViewer RT-DETRv2-R50","repo":"fireviewer/rtdetr-v2-r50-fire-smoke","license":"other/source-specific","route":"object_detection","task":"fire/smoke detection","publish_default":true}, + {"key":"fireviewer_yolo11m_strict_v1","title":"FireViewer YOLO11-M strict v1","repo":"fireviewer/fire-smoke-yolo11m-strict-v1","license":"agpl-3.0","route":"onnx_existing","task":"fire/smoke detection","publish_default":true}, + {"key":"fireviewer_rfdetr_medium_v110","title":"FireViewer RF-DETR Medium 1.10 strict v1","repo":"fireviewer/fire-smoke-rfdetr-medium-v110-strict-v1","license":"composite-research-release","route":"rfdetr_medium","task":"fire/smoke detection","publish_default":true}, + {"key":"fireviewer_dinov3_pointing_pilot_v1","title":"FireViewer DINOv3 pointing pilot v1","repo":"fireviewer/dinov3-pointing-pilot-v1","license":"dinov3-license-and-source-specific-terms","route":"dinov3_fireviewer","task":"segmentation + pointing + abstention + presence","publish_default":false,"base_repo":"facebook/dinov3-vitb16-pretrain-lvd1689m","base_revision":"5931719e67bbdb9737e363e781fb0c67687896bc"}, + {"key":"fireviewer_segformer_b2_baseline_v1","title":"FireViewer SegFormer-B2 baseline v1","repo":"fireviewer/segformer-b2-fire-smoke-baseline-v1","license":"source-specific","route":"segmentation","task":"fire/smoke semantic segmentation","publish_default":true,"legacy_optional":true} + ] +} diff --git a/conversion/package/scripts/convert_batch2.py b/conversion/package/scripts/convert_batch2.py new file mode 100644 index 0000000000000000000000000000000000000000..6bf64864982cd8e36e56779c6b09a9bc101fb87b --- /dev/null +++ b/conversion/package/scripts/convert_batch2.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +from __future__ import annotations +import argparse, fcntl, gc, hashlib, importlib.util, json, os, shutil, subprocess, sys, tempfile, time, traceback +from pathlib import Path +from typing import Any + +import numpy as np +import torch +import torch.nn as nn +from huggingface_hub import HfApi, snapshot_download + +ROOT = Path(os.environ.get('VDS_BATCH2_ROOT','/workspace/vds-litert-batch2')) +PKG = Path(__file__).resolve().parents[1] +MANIFEST_PATH = PKG / 'model_manifest.json' +TOKEN_FILE = Path(os.environ.get('HF_TOKEN_FILE','/workspace/HF.txt')) +DEST_REPO = os.environ.get('HF_DEST_REPO','Charlbi/Lite_rt_prepared_for_android_dataset_builder') +FIREVIEWER_DEST_REPO = os.environ.get('HF_FIREVIEWER_DEST_REPO','fireviewer/litert-models') +PREFETCH = ROOT/'models'/'upstream' +WORK = ROOT/'work' +ALLOWED = {'apache-2.0','mit','bsd-2-clause','bsd-3-clause'} +THREADS = max(1,int(os.environ.get('TORCH_NUM_THREADS','8'))) +torch.set_num_threads(THREADS) + + +def read_json(p: Path): return json.loads(p.read_text(encoding='utf-8')) +def write_json(p: Path, x: Any): p.parent.mkdir(parents=True,exist_ok=True); p.write_text(json.dumps(x,indent=2,sort_keys=True,default=str)+'\n',encoding='utf-8') +def sha256(p: Path): + h=hashlib.sha256() + with p.open('rb') as f: + for c in iter(lambda:f.read(8*1024*1024),b''): h.update(c) + return h.hexdigest() +def token(): + if not TOKEN_FILE.is_file(): raise RuntimeError(f'missing {TOKEN_FILE}') + v=TOKEN_FILE.read_text(encoding='utf-8-sig').strip().splitlines()[0].strip() + if len(v)<10: raise RuntimeError('HF token file is empty') + return v + +def run(cmd, cwd=None, env=None): + print('+',' '.join(map(str,cmd)),flush=True) + subprocess.run(list(map(str,cmd)),cwd=cwd,env=env,check=True) + +def inspect_tflite(p: Path): + from ai_edge_litert.interpreter import Interpreter + with p.open('rb') as f: + if f.read(8)[4:8] != b'TFL3': raise RuntimeError(f'bad flatbuffer: {p}') + i=Interpreter(model_path=str(p),num_threads=THREADS) + if not any(-1 in d.get('shape_signature',[]) for d in i.get_input_details()): i.allocate_tensors() + def d(t): return {'name':str(t.get('name','')),'shape':[int(x) for x in t['shape']],'shape_signature':[int(x) for x in t.get('shape_signature',t['shape'])],'dtype':str(np.dtype(t['dtype']))} + return {'inputs':[d(x) for x in i.get_input_details()],'outputs':[d(x) for x in i.get_output_details()]} + +def direct_export(module: nn.Module, args: tuple[torch.Tensor,...], out: Path): + import litert_torch + out.parent.mkdir(parents=True,exist_ok=True) + module=module.cpu().eval(); args=tuple(x.detach().cpu() for x in args) + with torch.inference_mode(): edge=litert_torch.convert(module,args) + tmp=out.with_suffix('.partial.tflite'); edge.export(str(tmp)); del edge + check=inspect_tflite(tmp); tmp.replace(out); return check + +def onnx_to_tflite(onnx: Path, out: Path, spatial_size: int=640): + temp=out.parent/(out.stem+'_onnx2tf') + shutil.rmtree(temp,ignore_errors=True); temp.mkdir(parents=True,exist_ok=True) + run([ROOT/'.venv-onnx/bin/python',PKG/'scripts/convert_onnx.py',str(onnx),str(temp),str(spatial_size)]) + cands=sorted(temp.rglob('*float32.tflite')) or sorted(temp.rglob('*.tflite'), key=lambda p:p.stat().st_size, reverse=True) + if not cands: raise RuntimeError('onnx2tf produced no tflite') + shutil.copy2(cands[0],out); return inspect_tflite(out) + +def all_specs(): + m=read_json(MANIFEST_PATH) + out=[] + for group in ('general_models','fireviewer_models'): + for s in m[group]: + if s['key']=='fireviewer_dfine_m_strict_v1': s={**s,'route':'onnx_existing'} + out.append({**s,'group':group}) + return out + +def resolve_hf(api,s): + info=api.model_info(s['repo']) + lic=None + card=getattr(info,'card_data',None) + if card is not None: + try: lic=card.get('license') + except Exception: lic=getattr(card,'license',None) + tags=getattr(info,'tags',[]) or [] + if not lic: + for x in tags: + if str(x).startswith('license:'): lic=str(x).split(':',1)[1] + if s['group']=='general_models' and str(lic or s['license']).lower() not in ALLOWED: + raise RuntimeError(f"non permissive license for public catalog: {lic}") + return {'repo_id':s['repo'],'sha':info.sha,'license':str(lic or s['license'])} + +def prefetch(api): + ROOT.mkdir(parents=True,exist_ok=True); PREFETCH.mkdir(parents=True,exist_ok=True) + state={'created':time.time(),'destination':DEST_REPO,'models':{}} + for s in all_specs(): + k=s['key']; print(f'[prefetch] {k}',flush=True) + try: + if s['route']=='rtmdet': + ext=PREFETCH/k; ext.mkdir(parents=True,exist_ok=True) + ck=ext/'model.pth' + if not ck.exists(): run(['curl','-L','--fail','--retry','4','-o',str(ck),s['checkpoint_url']]) + state['models'][k]={'local_dir':str(ext),'checkpoint':str(ck),'sha256':sha256(ck),'license':s['license'],'external_url':s['checkpoint_url']} + continue + up=resolve_hf(api,s); loc=PREFETCH/k + snapshot_download(repo_id=s['repo'],revision=up['sha'],local_dir=str(loc),token=token(),ignore_patterns=['*.ckpt','last_ema.pth','checkpoint_best_ema.pth','checkpoints/*']) + state['models'][k]={**up,'local_dir':str(loc)} + if s['route']=='dinov3_fireviewer': + base=PREFETCH/(k+'_base') + snapshot_download(repo_id=s['base_repo'],revision=s['base_revision'],local_dir=str(base),token=token()) + state['models'][k]['base_local_dir']=str(base) + except Exception as e: + state['models'][k]={'error':f'{type(e).__name__}: {e}'} + print(' !!',state['models'][k]['error'],flush=True) + write_json(ROOT/'prefetch_manifest.json',state); return state + +def prefetched(k): + p=ROOT/'prefetch_manifest.json' + if not p.exists(): raise RuntimeError('run setup/prefetch first') + e=read_json(p)['models'].get(k,{}) + if e.get('error'): raise RuntimeError(e['error']) + return e + +class TimmWrap(nn.Module): + def __init__(self,m): super().__init__(); self.m=m + def forward(self,x): + f=self.m.forward_features(x); emb=self.m.forward_head(f,pre_logits=True); logits=self.m.forward_head(f) + return logits,emb +class ImageODWrap(nn.Module): + def __init__(self,m): super().__init__(); self.m=m + def forward(self,pixel_values,pixel_mask): + o=self.m(pixel_values=pixel_values,pixel_mask=pixel_mask,return_dict=True) + return o.logits,o.pred_boxes +class GroundWrap(nn.Module): + def __init__(self,m): super().__init__(); self.m=m + def forward(self,pixel_values,pixel_mask,input_ids,attention_mask,token_type_ids): + o=self.m(pixel_values=pixel_values,pixel_mask=pixel_mask,input_ids=input_ids.long(),attention_mask=attention_mask.long(),token_type_ids=token_type_ids.long(),return_dict=True) + return o.logits,o.pred_boxes +class OwlWrap(nn.Module): + def __init__(self,m): super().__init__(); self.m=m + def forward(self,pixel_values,input_ids,attention_mask): + o=self.m(pixel_values=pixel_values,input_ids=input_ids.long(),attention_mask=attention_mask.long(),return_dict=True) + return o.logits,o.pred_boxes +class DepthWrap(nn.Module): + def __init__(self,m): super().__init__(); self.m=m + def forward(self,pixel_values): return self.m(pixel_values=pixel_values,return_dict=True).predicted_depth +class SegWrap(nn.Module): + def __init__(self,m): super().__init__(); self.m=m + def forward(self,pixel_values): return self.m(pixel_values=pixel_values,return_dict=True).logits +class DictOutWrap(nn.Module): + def __init__(self,m): super().__init__(); self.m=m + def forward(self,x): + o=self.m(x) + if isinstance(o,dict): return tuple(o[k] for k in sorted(o)) + if isinstance(o,(tuple,list)): return tuple(o) + return o + +def processor_support_files(local: Path): + names={'preprocessor_config.json','processor_config.json','tokenizer.json','tokenizer_config.json','special_tokens_map.json','vocab.json','merges.txt','vocab.txt','sentencepiece.bpe.model','spiece.model','config.json'} + return [p for p in local.iterdir() if p.is_file() and p.name in names] + +def convert_timm(s,e,out): + import timm + local=Path(e['local_dir']); cfg=read_json(local/'config.json'); arch=cfg.get('architecture') or cfg.get('model_name') + weights=next(iter(local.glob('*.safetensors')),None) or next(iter(local.glob('pytorch_model.bin')),None) + if not arch or not weights: raise RuntimeError('missing timm architecture/weights') + m=timm.create_model(arch,pretrained=False,checkpoint_path=str(weights)).eval() + inp=cfg.get('pretrained_cfg',{}).get('input_size',[3,224,224]); x=torch.zeros(1,int(inp[0]),int(inp[1]),int(inp[2])) + p=out/'model.tflite'; chk=direct_export(TimmWrap(m),(x,),p) + return [p],{'route':'timm','structural':chk,'outputs':['classification_logits','embedding']} + +def convert_object_detection(s,e,out): + from transformers import AutoImageProcessor,AutoModelForObjectDetection + local=e['local_dir']; proc=AutoImageProcessor.from_pretrained(local,local_files_only=True,trust_remote_code=True) + m=AutoModelForObjectDetection.from_pretrained(local,local_files_only=True,trust_remote_code=True).eval() + # Use a static 640 square unless processor fixes another size. + img=np.zeros((640,640,3),dtype=np.uint8); d=proc(images=img,return_tensors='pt') + pv=d['pixel_values'].float(); pm=d.get('pixel_mask',torch.ones((pv.shape[0],pv.shape[-2],pv.shape[-1]),dtype=torch.int64)).to(torch.int32) + p=out/'model.tflite'; chk=direct_export(ImageODWrap(m),(pv,pm),p) + return [p],{'route':'transformers_object_detection','structural':chk,'outputs':['logits','pred_boxes']} + +def convert_grounding(s,e,out): + from transformers import AutoProcessor,AutoModelForZeroShotObjectDetection + local=e['local_dir']; proc=AutoProcessor.from_pretrained(local,local_files_only=True); m=AutoModelForZeroShotObjectDetection.from_pretrained(local,local_files_only=True).eval() + d=proc(images=np.zeros((800,800,3),dtype=np.uint8),text='person. car. animal.',padding='max_length',max_length=32,truncation=True,return_tensors='pt') + pv=d['pixel_values'].float(); pm=d.get('pixel_mask',torch.ones((1,pv.shape[-2],pv.shape[-1]),dtype=torch.int32)).to(torch.int32) + ids=d['input_ids'].to(torch.int32); att=d['attention_mask'].to(torch.int32); tt=d.get('token_type_ids',torch.zeros_like(ids)).to(torch.int32) + p=out/'model.tflite'; chk=direct_export(GroundWrap(m),(pv,pm,ids,att,tt),p) + return [p],{'route':'grounding_dino','structural':chk,'outputs':['logits','pred_boxes'],'text_max_length':32} + +def convert_owl(s,e,out): + from transformers import AutoProcessor,AutoModelForZeroShotObjectDetection + local=e['local_dir']; proc=AutoProcessor.from_pretrained(local,local_files_only=True); m=AutoModelForZeroShotObjectDetection.from_pretrained(local,local_files_only=True).eval() + d=proc(images=np.zeros((960,960,3),dtype=np.uint8),text=[['person','car','animal']],padding='max_length',max_length=16,truncation=True,return_tensors='pt') + p=out/'model.tflite'; chk=direct_export(OwlWrap(m),(d['pixel_values'].float(),d['input_ids'].to(torch.int32),d['attention_mask'].to(torch.int32)),p) + return [p],{'route':'owlv2','structural':chk,'outputs':['logits','pred_boxes'],'text_max_length':16} + +def convert_depth(s,e,out): + from transformers import AutoImageProcessor,AutoModelForDepthEstimation + local=e['local_dir']; proc=AutoImageProcessor.from_pretrained(local,local_files_only=True); m=AutoModelForDepthEstimation.from_pretrained(local,local_files_only=True).eval() + d=proc(images=np.zeros((518,518,3),dtype=np.uint8),return_tensors='pt'); p=out/'model.tflite'; chk=direct_export(DepthWrap(m),(d['pixel_values'].float(),),p) + return [p],{'route':'depth','structural':chk,'outputs':['predicted_depth']} + +def convert_seg(s,e,out): + from transformers import AutoImageProcessor,AutoModelForSemanticSegmentation + local=e['local_dir']; proc=AutoImageProcessor.from_pretrained(local,local_files_only=True); m=AutoModelForSemanticSegmentation.from_pretrained(local,local_files_only=True).eval() + d=proc(images=np.zeros((512,512,3),dtype=np.uint8),return_tensors='pt'); p=out/'model.tflite'; chk=direct_export(SegWrap(m),(d['pixel_values'].float(),),p) + return [p],{'route':'segmentation','structural':chk,'outputs':['segmentation_logits']} + +def convert_onnx_existing(s,e,out): + local=Path(e['local_dir']); onnx=next(iter(local.rglob('*.onnx')),None) + if onnx is None: raise RuntimeError('no ONNX artifact upstream') + size=960 if s['key']=='fireviewer_yolo11m_strict_v1' else 704 + p=out/'model.tflite'; chk=onnx_to_tflite(onnx,p,spatial_size=size); return [p],{'route':'onnx2tf','structural':chk,'source_onnx':onnx.name,'source_sha256':sha256(onnx),'image_size':size} + +def convert_picodet(s,e,out): + local=Path(e['local_dir']); model=local/'inference.json'; params=local/'inference.pdiparams' + if not model.exists() or not params.exists(): raise RuntimeError('Paddle inference.json/pdiparams missing') + onnx=out/'model.onnx' + run([ROOT/'.venv-onnx/bin/paddle2onnx','--model_dir',str(local),'--model_filename','inference.json','--params_filename','inference.pdiparams','--save_file',str(onnx),'--opset_version','11','--enable_onnx_checker','True']) + p=out/'model.tflite'; chk=onnx_to_tflite(onnx,p); return [p],{'route':'paddle2onnx+onnx2tf','structural':chk} + +def convert_rtmdet(s,e,out): + repo=ROOT/'vendor'/'mmyolo'; ck=Path(e['checkpoint']); cfg=repo/s['config']; od=out/'onnx'; od.mkdir(parents=True,exist_ok=True) + run([ROOT/'.venv-mm/bin/python',str(repo/'projects/easydeploy/tools/export.py'),str(cfg),str(ck),'--work-dir',str(od),'--img-size','640','640','--batch','1','--device','cpu','--opset','11','--model-only'],cwd=repo) + ons=sorted(od.rglob('*.onnx'),key=lambda p:p.stat().st_size,reverse=True) + if not ons: raise RuntimeError('MMYOLO EasyDeploy produced no ONNX') + p=out/'model.tflite'; chk=onnx_to_tflite(ons[0],p); return [p],{'route':'mmyolo-easydeploy+onnx2tf','structural':chk,'postprocess':'external RTMDet decode/NMS'} + +def convert_rfdetr_medium(s,e,out): + from rfdetr import RFDETRMedium + local=Path(e['local_dir']); cands=list(local.rglob('checkpoint_best_total.pth')) or list(local.rglob('*.pth')) + if not cands: raise RuntimeError('RF-DETR checkpoint missing') + training=read_json(local/'training_config.json') + m=RFDETRMedium(pretrain_weights=str(cands[0]),num_classes=int(training['num_classes']),resolution=int(training['model_config']['resolution']),device='cpu') + res=m.export(format='litert',output_dir=str(out),quantization=None) + p=Path(res) if isinstance(res,(str,Path)) else None + if p is None or not p.is_file(): + cs=list(out.rglob('*.tflite')); p=cs[0] if len(cs)==1 else None + if p is None: raise RuntimeError('RF-DETR produced no unique LiteRT file') + final=out/'model.tflite'; + if p.resolve()!=final.resolve(): shutil.copy2(p,final) + return [final],{'route':'rfdetr-direct-litert','structural':inspect_tflite(final)} + +def convert_dinov3_fv(s,e,out): + local=Path(e['local_dir']); adapter=local/'dinov3_adapter.py'; weights=local/'model.safetensors' + if not adapter.exists() or not weights.exists(): raise RuntimeError('FireViewer DINOv3 adapter/safetensors missing') + spec=importlib.util.spec_from_file_location('fv_dinov3_adapter',adapter); mod=importlib.util.module_from_spec(spec); assert spec and spec.loader + sys.modules[spec.name]=mod; spec.loader.exec_module(mod) + cls=getattr(mod,'DinoV3MultiTaskModel'); base=e.get('base_local_dir') or s['base_repo'] + # Transformers 5.17 nests the existing ViT encoder under model; keep strict load. + import safetensors.torch as st + original_load=st.load_file + def load_compatible(*args,**kwargs): + state=original_load(*args,**kwargs) + return {(k.replace('backbone.layer.','backbone.model.layer.',1) if k.startswith('backbone.layer.') else k):v for k,v in state.items()} + st.load_file=load_compatible + try:obj=cls.from_safetensors(str(weights),model_id=str(base),revision=None,image_size=448,token=False) + finally:st.load_file=original_load + net=getattr(obj,'network',obj).eval(); p=out/'model.tflite'; chk=direct_export(DictOutWrap(net),(torch.zeros(1,3,448,448),),p) + return [p],{'route':'fireviewer-dinov3-adapter','structural':chk,'warning':'pilot; no public-release approval inherited'} + +CONV={'timm':convert_timm,'object_detection':convert_object_detection,'grounding_dino':convert_grounding,'owlv2':convert_owl,'depth':convert_depth,'segmentation':convert_seg,'onnx_existing':convert_onnx_existing,'picodet':convert_picodet,'rtmdet':convert_rtmdet,'rfdetr_medium':convert_rfdetr_medium,'dinov3_fireviewer':convert_dinov3_fv} + +def support_files(s,e): + if not e.get('local_dir'): return [] + local=Path(e['local_dir']) + notices=[p for p in local.iterdir() if p.is_file() and (p.name.lower().startswith(('license','copying','notice')) or p.name in {'CHECKPOINT_LICENSE.md','CHECKPOINT_RIGHTS.md','CITATION.md','COMMERCIAL_USE.md','MODEL_RIGHTS_POLICY.md','RIGHTS_AND_ATTRIBUTION.md','THIRD_PARTY_NOTICES.md','labels.json'})] + return list(dict.fromkeys(processor_support_files(local)+notices)) + +def publish(api,s,e,out,arts,report): + group='models' + dest=DEST_REPO if s['group']=='general_models' else FIREVIEWER_DEST_REPO + if s['group']=='fireviewer_models' and not s.get('publish_default',False) and os.environ.get('PUBLISH_RESTRICTED_DINOV3','0')!='1': + return 'converted_local_only_rights_gate' + stage=out/'publish'; shutil.rmtree(stage,ignore_errors=True); stage.mkdir(parents=True) + for a in arts: shutil.copy2(a,stage/a.name) + for sf in support_files(s,e): shutil.copy2(sf,stage/sf.name) + source_readme=Path(e['local_dir'])/'README.md' + if source_readme.is_file(): shutil.copy2(source_readme,stage/'UPSTREAM_README.md') + contract={'schema':1,'key':s['key'],'title':s['title'],'task':s['task'],'route':report.get('route',s['route']),'upstream':s.get('repo') or s.get('checkpoint_url'),'upstream_revision':e.get('sha'),'license':e.get('license',s['license']),'tflite':[a.name for a in arts],'structural_only':not report.get('dynamic_validated',False),'benchmark':False,'report':report} + write_json(stage/'runtime_contract.json',contract) + write_json(stage/'artifact_manifest.json',{p.name:{'sha256':sha256(p),'bytes':p.stat().st_size} for p in stage.iterdir() if p.is_file()}) + (stage/'README.md').write_text(f"---\nlicense: {str(e.get('license',s['license'])).lower()}\ntags:\n- litert\n- tflite\n- android\n- dataset-preparation\n---\n\n# {s['title']} — LiteRT conversion\n\nSource: `{s.get('repo') or s.get('checkpoint_url')}`.\n\nTask: {s['task']}.\n\nThis conversion is prepared for human-reviewed dataset preprocessing. No accuracy or speed benchmark is claimed by this batch. The serialized LiteRT files were only structurally loaded and tensor allocation was checked.\n\nConversion route: `{s['route']}`.\n",encoding='utf-8') + if report.get('dynamic_validated'): + (stage/'README.md').write_text(f"# {s['title']} — dynamic LiteRT conversion\n\nSource: `{s.get('repo')}`, revision `{e.get('sha')}`.\n\nThe image height and width are dynamic. No image resampling is embedded in this graph. The supported bounds and any alignment constraints are recorded in `runtime_contract.json`. Batch size is one. Inputs must be normalized as required by the upstream configuration.\n\nValidation includes real LiteRT inference at several square and rectangular dimensions and numerical comparison against the source model. See `validation.json` for exact shapes, tolerances and measured errors. These are conversion checks, not accuracy or performance benchmarks. Android execution has not been tested. The required CPU interpreter/delegate configuration, when applicable, is recorded in `runtime_contract.json`.\n\nPreserved upstream license and attribution files apply.\n",encoding='utf-8') + write_json(stage/'validation.json',report) + write_json(stage/'artifact_manifest.json',{p.name:{'sha256':sha256(p),'bytes':p.stat().st_size} for p in stage.iterdir() if p.is_file() and p.name!='artifact_manifest.json'}) + with (ROOT/'.publish.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX) + parent=api.repo_info(repo_id=dest,repo_type='model',revision='main').sha + commit=api.upload_folder(repo_id=dest,repo_type='model',folder_path=str(stage),path_in_repo=f'{group}/{s["key"]}',revision='main',parent_commit=parent,commit_message=f'Add LiteRT conversion: {s["title"]}') + paths=[f'{group}/{s["key"]}/{a.name}' for a in arts] + remote=api.get_paths_info(repo_id=dest,paths=paths,revision=commit.oid,repo_type='model') + indexed={p.path:p for p in remote} + for a,path in zip(arts,paths): + entry=indexed[path] + if entry.size!=a.stat().st_size or (entry.lfs and entry.lfs.sha256!=sha256(a)): + raise RuntimeError(f'Remote artifact mismatch: {path}') + write_json(out/'publication_receipt.json',{'repo':dest,'commit':commit.oid,'verified_paths':paths}) + return f'{dest}/tree/{commit.oid}/{group}/{s["key"]}' + +def process(api,s): + k=s['key']; base=WORK/k; base.mkdir(parents=True,exist_ok=True); attempt=Path(tempfile.mkdtemp(prefix='attempt-',dir=base)); st={'key':k,'title':s['title'],'phase':'starting'} + try: + e=prefetched(k); st['upstream']=e; st['phase']='converting'; write_json(base/'status.json',st) + arts,rep=CONV[s['route']](s,e,attempt); rep['artifacts']=[{'name':p.name,'bytes':p.stat().st_size,'sha256':sha256(p),'tensors':inspect_tflite(p)} for p in arts] + st['report']=rep; st['phase']='publishing'; write_json(base/'status.json',st) + st['destination']=publish(api,s,e,attempt,arts,rep); st['phase']='pushed' if '/tree/' in st['destination'] else 'converted_local_only' + except Exception as ex: + st.update(phase='failed',error=f'{type(ex).__name__}: {ex}',traceback=traceback.format_exc()) + print(st['traceback'],flush=True) + finally: + write_json(base/'status.json',st); gc.collect(); + if torch.cuda.is_available(): + try: torch.cuda.empty_cache() + except Exception: pass + return st + +def main(): + ap=argparse.ArgumentParser(); ap.add_argument('--prefetch-only',action='store_true'); ap.add_argument('--general-only',action='store_true'); ap.add_argument('--fireviewer-only',action='store_true'); ap.add_argument('models',nargs='*'); a=ap.parse_args() + api=HfApi(token=token()) + if a.prefetch_only: + api.repo_info(repo_id=DEST_REPO,repo_type='model'); prefetch(api); return + specs=all_specs() + if a.general_only: specs=[s for s in specs if s['group']=='general_models'] + if a.fireviewer_only: specs=[s for s in specs if s['group']=='fireviewer_models'] + if a.models: specs=[s for s in specs if s['key'] in set(a.models)] + results={} + for i,s in enumerate(specs,1): + print(f'\n[{i}/{len(specs)}] {s["key"]}',flush=True); results[s['key']]=process(api,s); print(' ->',results[s['key']]['phase'],flush=True) + write_json(WORK/'summary.json',results) + print('\nPUSHED:',', '.join(k for k,v in results.items() if v['phase']=='pushed') or 'none') + print('FAILED:',', '.join(k for k,v in results.items() if v['phase']=='failed') or 'none') + print('LOCAL ONLY:',', '.join(k for k,v in results.items() if v['phase']=='converted_local_only') or 'none') + +if __name__=='__main__': main() diff --git a/conversion/patch_onnx2tf_gather_signature.py b/conversion/patch_onnx2tf_gather_signature.py new file mode 100644 index 0000000000000000000000000000000000000000..1de228b405117c283f8e22b0be58eeebb93f65fb --- /dev/null +++ b/conversion/patch_onnx2tf_gather_signature.py @@ -0,0 +1,19 @@ +"""Repair empty static shape signatures in onnx2tf 2.6.9 GatherElements. + +An absent optional shape_signature means the static tensor.shape, not rank zero. +The final D-FINE graph still has to match the original ONNX numerically. +""" +from pathlib import Path +import importlib.util +p=Path(importlib.util.find_spec('onnx2tf').origin).parent/'tflite_builder/op_builders/index.py' +text=p.read_text();start=text.index('def build_gather_elements_op(');end=text.find('\ndef ',start+5) +part=text[start:end if end!=-1 else len(text)] +for variable in ('data','indices','output'): + old=f'if {variable}_tensor is not None and {variable}_tensor.shape_signature is not None' + new=f'if {variable}_tensor is not None and {variable}_tensor.shape_signature is not None and len({variable}_tensor.shape_signature) == len({variable}_shape)' + if new not in part: + assert old in part;part=part.replace(old,new) +updated=text[:start]+part+(text[end:] if end!=-1 else '') +if updated!=text: + p.with_suffix('.py.vds-original').write_text(text);p.write_text(updated) +print('GatherElements static signature fallback installed') diff --git a/conversion/prefetch_manifest.json b/conversion/prefetch_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..064568efade65e22c7bbf19c7628a4283c67769e --- /dev/null +++ b/conversion/prefetch_manifest.json @@ -0,0 +1,40 @@ +{ + "created": 1789851716.286558, + "destination": "Charlbi/Lite_rt_prepared_for_android_dataset_builder", + "models": { + "fireviewer_dfine_m_strict_v1": { + "repo_id": "fireviewer/fire-smoke-dfine-m-strict-v1", + "sha": "a4d15bae27f8deaeb9f89e7e118b4643cddfecad", + "license": "other", + "local_dir": "/workspace/vds-litert-batch2/models/upstream/fireviewer_dfine_m_strict_v1" + }, + "fireviewer_rtdetr_v2_r50": { + "repo_id": "fireviewer/rtdetr-v2-r50-fire-smoke", + "sha": "b16bc79df05a69bfd90a7130d77fdbbd0d556465", + "license": "other", + "local_dir": "/workspace/vds-litert-batch2/models/upstream/fireviewer_rtdetr_v2_r50" + }, + "fireviewer_yolo11m_strict_v1": { + "repo_id": "fireviewer/fire-smoke-yolo11m-strict-v1", + "sha": "065979cd2300c27eeb355ab1e1524d8223d0b276", + "license": "agpl-3.0", + "local_dir": "/workspace/vds-litert-batch2/models/upstream/fireviewer_yolo11m_strict_v1" + }, + "fireviewer_rfdetr_medium_v110": { + "repo_id": "fireviewer/fire-smoke-rfdetr-medium-v110-strict-v1", + "sha": "901382d9c8eef2af8b4c13640413c03dbfdbeec2", + "license": "other", + "local_dir": "/workspace/vds-litert-batch2/models/upstream/fireviewer_rfdetr_medium_v110" + }, + "fireviewer_dinov3_pointing_pilot_v1": { + "repo_id": "fireviewer/dinov3-pointing-pilot-v1", + "sha": "bd4cc7e15e296fa95b1cd9478f6952a21a4201ec", + "license": "other", + "local_dir": "/workspace/vds-litert-batch2/models/upstream/fireviewer_dinov3_pointing_pilot_v1", + "base_local_dir": "/workspace/vds-litert-batch2/models/upstream/fireviewer_dinov3_pointing_pilot_v1_base" + }, + "fireviewer_segformer_b2_baseline_v1": { + "error": "RepositoryNotFoundError: 404 Client Error. (Request ID: Root=1-6aaef886-7a05968f44ee08895dcd3a44;e2f3be59-92ac-4cb9-9b39-b321c8349160)\n\nRepository Not Found for url: https://huggingface.co/api/models/fireviewer/segformer-b2-fire-smoke-baseline-v1.\nPlease make sure you specified the correct `repo_id` and `repo_type`.\nIf you are trying to access a private or gated repo, make sure you are authenticated and your token has the required permissions.\nFor more details, see https://huggingface.co/docs/huggingface_hub/authentication" + } + } +} diff --git a/conversion/prepare_learning_backbone.py b/conversion/prepare_learning_backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..dea70d420e460ebeb62a6cfc1c4730096b056848 --- /dev/null +++ b/conversion/prepare_learning_backbone.py @@ -0,0 +1,17 @@ +"""Prepare frozen inference components; never publish incomplete conversions.""" +import os,sys,json,traceback +from pathlib import Path +os.environ.update(VDS_BATCH2_ROOT='/workspace/vds-litert-batch2',HF_TOKEN_FILE='/workspace/.secrets/hf_token',TORCH_NUM_THREADS='6') +root=Path('/workspace/vds-litert-batch2');sys.path.insert(0,str(root/'package/scripts')) +import convert_batch2 as worker +key=sys.argv[1];out=root/'learning-backbones'/key;out.mkdir(parents=True,exist_ok=True) +status={'key':key,'phase':'converting_frozen_component','published':False} +(out/'status.json').write_text(json.dumps(status)) +try: + spec=next(s for s in worker.all_specs() if s['key']==key);entry=worker.prefetched(key) + artifacts,report=worker.CONV[spec['route']](spec,entry,out) + status.update(phase='converted_frozen_component',artifacts=[str(p) for p in artifacts],report=report) +except Exception as e: + status.update(phase='failed',error=str(e),traceback=traceback.format_exc());print(status['traceback'],flush=True) +(out/'status.json').write_text(json.dumps(status,indent=2)+'\n') +print(json.dumps(status),flush=True) diff --git a/conversion/protect_dfine_arena.py b/conversion/protect_dfine_arena.py new file mode 100644 index 0000000000000000000000000000000000000000..3704f75db5e8cd7ba91e5f2e42683b568747288b --- /dev/null +++ b/conversion/protect_dfine_arena.py @@ -0,0 +1,36 @@ +"""Preserve selected decoder activations until a final, tiny read operation. + +No prediction arithmetic or weight changes. These reads prevent premature +in-place reuse in the LiteRT arena; the guard values are not model predictions. +""" +import os +os.environ.update(CUDA_VISIBLE_DEVICES='',TF_CPP_MIN_LOG_LEVEL='3') +import json,flatbuffers,numpy as np +from pathlib import Path +from tensorflow.lite.python import schema_py_generated as schema +r=Path('/workspace/vds-litert-batch2/learning-backbones/fireviewer_dfine_m_strict_v1') +p=r/'simplified-converted/model-cropped.tflite';m=schema.ModelT.InitFromObj(schema.Model.GetRootAsModel(p.read_bytes(),0));g=m.subgraphs[0] +protection=json.loads((r/'arena-protection.json').read_text());lookup={t.name.decode():i for i,t in enumerate(g.tensors)} +def tensor(name,shape,dtype,data=None): + t=schema.TensorT();t.name=name.encode();t.shape=np.array(shape,np.int32);t.type=dtype;t.hasRank=True;t.buffer=0 + if data is not None: + b=schema.BufferT();b.data=np.frombuffer(data.tobytes(),np.uint8);t.buffer=len(m.buffers);m.buffers.append(b) + g.tensors.append(t);return len(g.tensors)-1 +def opcode(code): + o=schema.OperatorCodeT();o.builtinCode=code;o.deprecatedBuiltinCode=min(code,127);o.version=1;m.operatorCodes.append(o);return len(m.operatorCodes)-1 +reshape=opcode(schema.BuiltinOperator.RESHAPE);gather=opcode(schema.BuiltinOperator.GATHER) +flat_shape=tensor('arena_guard_shape',[1],schema.TensorType.INT32,np.array([-1],np.int32)) +zero=tensor('arena_guard_zero',[],schema.TensorType.INT32,np.array(0,np.int32)) +for n,item in enumerate(protection['tensors']): + selected=lookup[item['name']] + producer=next(o for o in g.operators if selected in o.outputs) + assert m.operatorCodes[producer.opcodeIndex].builtinCode==schema.BuiltinOperator.SOFTMAX + # The softmax input aliases corner logits later read by the LQE branch. + # Retaining this input prevents softmax from overwriting that shared storage. + source=int(producer.inputs[0]);original=g.tensors[source];count=int(np.prod(original.shape));assert count>0 + flat=tensor('arena_guard_flat_'+str(n),[count],original.type) + op=schema.OperatorT();op.opcodeIndex=reshape;op.inputs=np.array([source,flat_shape],np.int32);op.outputs=np.array([flat],np.int32);op.builtinOptionsType=schema.BuiltinOptions.ReshapeOptions;op.builtinOptions=schema.ReshapeOptionsT();op.builtinOptions.newShape=np.array([-1],np.int32);g.operators.append(op) + scalar=tensor('arena_guard_read_'+str(n),[],original.type) + op=schema.OperatorT();op.opcodeIndex=gather;op.inputs=np.array([flat,zero],np.int32);op.outputs=np.array([scalar],np.int32);op.builtinOptionsType=schema.BuiltinOptions.GatherOptions;op.builtinOptions=schema.GatherOptionsT();op.builtinOptions.axis=0;op.builtinOptions.batchDims=0;g.operators.append(op) +b=flatbuffers.Builder(1024);b.Finish(m.Pack(b),file_identifier=b'TFL3');dest=r/'model-protected.tflite';dest.write_bytes(b.Output()) +print(json.dumps({'artifact':str(dest),'protected_tensors':len(protection['tensors']),'protected_elements':sum(int(np.prod(t['shape'])) for t in protection['tensors'])}),flush=True) diff --git a/conversion/publish_learning.py b/conversion/publish_learning.py new file mode 100644 index 0000000000000000000000000000000000000000..84d4690648f8a097b9b0c1436813926b085aa004 --- /dev/null +++ b/conversion/publish_learning.py @@ -0,0 +1,82 @@ +"""Publish executed trainable variants beside immutable inference families.""" +import json,sys,hashlib,shutil,fcntl +from pathlib import Path +from huggingface_hub import HfApi +root=Path('/workspace/vds-litert-batch2') +api=HfApi(token=(Path('/workspace/.secrets/hf_token').read_text().strip())) +manifest=json.loads((root/'package/model_manifest.json').read_text());specs={s['key']:s for g in ('general_models','fireviewer_models') for s in manifest[g]} +upstreams=json.loads((root/'prefetch_manifest.json').read_text())['models'] +upstreams.update(json.loads((root/'previous-prefetch.json').read_text())) +for s in json.loads((root/'previous-models.json').read_text()):specs[s['key']]={**s,'title':s['key']} +def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest() +for key in sys.argv[1:]: + directory=root/'learning'/key;report=json.loads((directory/'training_validation.json').read_text());source=json.loads((directory/'source.json').read_text()) + assert report['validation_passed'] and report['optimizer_resume_exact'] and report['restart_logits_exact'] + assert sha(directory/'model.tflite')==report['sha256'] + spec=specs[key];upstream=upstreams[key] + destination='fireviewer/litert-models' if key.startswith('fireviewer_') else 'Charlbi/Lite_rt_prepared_for_android_dataset_builder' + if key.startswith('fireviewer_') and not spec.get('publish_default',False): + # User explicitly requested FireViewer delivery; the pilot permits a + # private research copy, never a public release under this workflow. + assert source.get('private_pilot_only') and api.repo_info(destination).private + stage=directory/'publish';stage.mkdir(exist_ok=True) + shutil.copy2(directory/'model.tflite',stage/'model.tflite') + labels=source.get('labels') or json.loads((root/'learning/imagenet-labels.json').read_text()) + labels=[label if labels.count(label)==1 else f'{label} [class {i}]' for i,label in enumerate(labels)] + prep=source.get('pretrained_cfg',{}) + config={'task':'classification' if prep else 'object_detection','adapter':'classification' if prep else 'yolo','inputWidth':224,'inputHeight':224,'inputLayout':'NCHW','inputType':'FLOAT32','mean':0.,'std':1.,'resizeMode':'stretch','labels':labels,'threads':2,'outputIndex':0,'threshold':.1,'scoreActivation':'softmax' if prep else 'none','dynamicMinSize':128,'dynamicMaxSize':1280,'dynamicStride':32 if key=='convformer_s18' or not prep else 1} + if prep: + config.update(channelMean=[v*255 for v in prep['mean']],channelStd=[v*255 for v in prep['std']]) + training={'targetEncoding':'one_hot','targetShape':[1,len(labels)],'inferOutputs':['scores']} + else: + config.update(inputWidth=source.get('app_size',320),inputHeight=source.get('app_size',320),mean=0.,std=255.,resizeMode='letterbox' if source['kind']=='yolo' else 'stretch',outputLayout='BCN',coordinates='pixels',outputIndexBoxes=0,outputIndexScores=1) + if source.get('channelMean'):config.update(channelMean=source['channelMean'],channelStd=source['channelStd']) + training={'targetEncoding':'boxes_xyxy_class_mask','targetShape':[1,100,6],'inferOutputs':['detections']} + training.update(trainSignature='train',inferSignature='infer',saveSignature='save',restoreSignature='restore',imageInput='x',targetInput='y',lossOutput='loss',checkpointInput='checkpoint_path',learningRateInput='learning_rate',scope=report['training_scope']) + config['training']=training + config['isRgb']=source.get('isRgb',True) + config['dynamicStride']=report.get('variable_image_preprocessing',{}).get('stride',32 if key=='convformer_s18' else 1) + config['dynamicMinSize']=source.get('dynamic_min_size',128) + if (directory/'android_model_config.json').exists():config=json.loads((directory/'android_model_config.json').read_text()) + # The default graph's output order is not necessarily the named detection + # signature order. Resolve actual tensor indices for the Android adapter. + import tensorflow as tf + engine=tf.lite.Interpreter(model_path=str(directory/'model.tflite'),experimental_op_resolver_type=tf.lite.experimental.OpResolverType.BUILTIN_WITHOUT_DEFAULT_DELEGATES) + named=engine.get_signature_runner('infer').get_output_details() + main=engine.get_output_details() + output_indices={name:next(j for j,d in enumerate(main) if d['index']==details['index']) for name,details in named.items()} + config['namedOutputIndices']=output_indices + if config['adapter']=='yolo': + config['outputIndex']=output_indices['detections'] + config.pop('outputIndexBoxes',None);config.pop('outputIndexScores',None) + del engine + (stage/'android_model_config.json').write_text(json.dumps(config,indent=2)+'\n') + report['runtime_requirement']='LiteRT builtin CPU; disable default XNNPACK delegates; Select TF Ops required for checkpoint Save/Restore' + report['android_runtime_dependencies']=['org.tensorflow:tensorflow-lite:2.16.1','org.tensorflow:tensorflow-lite-select-tf-ops:2.16.1'] + report['frozen_backbone_sha256']=sha(Path(source['backbone'])) + report['head_initialization']=source.get('head_initialization','source_model_or_identity_adapter') + builder='build_trainable_fire_multitask.py' if source.get('kind')=='multitask' else 'build_trainable_classifier.py' if prep else 'build_trainable_heatmap.py' if source.get('kind')=='heatmap' else 'build_trainable_detector.py' + report['conversion_source_sha256']={n:sha(root/n) for n in ['merge_training_graph.py',builder]} + if source.get('parity_report'): + parity=Path(source['parity_report']);shutil.copy2(parity,stage/'source_parity.json') + report['source_parity_report']='source_parity.json' + if (directory/'runtime216_validation.json').exists(): + compatible=json.loads((directory/'runtime216_validation.json').read_text());assert compatible['sha256']==report['sha256'] + shutil.copy2(directory/'runtime216_validation.json',stage/'runtime216_validation.json') + (stage/'training_validation.json').write_text(json.dumps(report,indent=2)+'\n') + contract={'schema':2,'key':key+'_learning','source_model':key,'task':config['task'],'upstream':spec.get('repo') or spec.get('checkpoint_url'),'upstream_revision':upstream.get('sha') or upstream.get('sha256'),'backbone_frozen':True,'training_scope':report['training_scope'],'report':report,'optimizer':'momentum SGD, momentum=0.9','checkpoint_contains':['trainable parameters','momentum buffers','global step'],'requires_select_tf_ops':True,'android_configuration':'android_model_config.json'} + (stage/'runtime_contract.json').write_text(json.dumps(contract,indent=2)+'\n') + local=Path(upstream['local_dir']) + for p in local.iterdir(): + if p.is_file() and (p.name.lower().startswith(('license','notice','copying')) or p.name in ('CHECKPOINT_LICENSE.md','CHECKPOINT_RIGHTS.md','CITATION.md','COMMERCIAL_USE.md','MODEL_RIGHTS_POLICY.md','RIGHTS_AND_ATTRIBUTION.md','THIRD_PARTY_NOTICES.md')):shutil.copy2(p,stage/p.name) + if (local/'README.md').is_file():shutil.copy2(local/'README.md',stage/'UPSTREAM_README.md') + if key=='rtmdet_tiny':shutil.copy2(root/'vendor/mmyolo/LICENSE',stage/'LICENSE') + (stage/'README.md').write_text(f"# {spec['title']} — Android learning variant\n\nSource: `{spec.get('repo')}`, revision `{upstream.get('sha')}`.\n\nThis is a real mutable LiteRT model with `infer`, `train`, `save`, and `restore` signatures. Training executes on Android using the image and reviewed targets. Trainable scope: **{report['training_scope']}**. The visual backbone remains frozen. {report.get('limitation','')}\n\nThis learning variant is stored separately under `models/{key}_learning` and has its own explicit Android configuration.\n\nVariable image dimensions: `{report.get('variable_image_preprocessing', {'mode':'native_dynamic'})}`. The Android configuration chooses a bounded input canvas; the image may have any original size. Images are processed individually, so a dataset need not have homogeneous dimensions.\n\nCheckpoints include parameters, momentum and step. Validation executed actual updates, changed parameters, closed/reopened the interpreter, restored identical predictions and reproduced the next optimizer update. See `training_validation.json`. Tests use synthetic supervision to verify execution and persistence; no accuracy or forgetting benchmark is claimed. Android device execution: `{report.get('android_device_tested',False)}`.\n\nUse LiteRT CPU with XNNPACK disabled and Select TF Ops for checkpoint operations. The default label space is retained; changing class count requires rebuilding the head with the supplied conversion sources. Use only human-reviewed labels for training. Retain replay examples and validate a candidate before activating it.\n\nUpstream licenses and attributions continue to apply.\n") + (stage/'artifact_manifest.json').write_text(json.dumps({p.name:{'sha256':sha(p),'bytes':p.stat().st_size} for p in stage.iterdir() if p.is_file() and p.name!='artifact_manifest.json'},indent=2)+'\n') + with (root/'.publish.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX);parent=api.repo_info(destination,repo_type='model').sha + commit=api.upload_folder(repo_id=destination,repo_type='model',folder_path=str(stage),path_in_repo=f'models/{key}_learning',parent_commit=parent,commit_message=f'Add on-device learning and optimizer resume: {key}') + path=f'models/{key}_learning/model.tflite';remote=api.get_paths_info(destination,[path],revision=commit.oid)[0] + assert remote.size==report['bytes'] and remote.lfs.sha256==report['sha256'] + receipt={'repo':destination,'commit':commit.oid,'path':path,'sha256':report['sha256'],'bytes':report['bytes'],'remote_verified':True} + (directory/'publication_receipt.json').write_text(json.dumps(receipt,indent=2)+'\n');print(json.dumps(receipt),flush=True) diff --git a/conversion/static_topk_lowering.py b/conversion/static_topk_lowering.py new file mode 100644 index 0000000000000000000000000000000000000000..fc5f40d26e5cbc8a377147613403aaa749ee3fd2 --- /dev/null +++ b/conversion/static_topk_lowering.py @@ -0,0 +1,35 @@ +"""Keep detector proposal TopK as a native TFLite operation, not StableHLO sort.""" +import torch +from litert_torch import fx_infra +from litert_torch.backend.lowerings import registry, utils +from litert_torch.backend.experimental.torch_tfl._lowerings import _ir_operation +from litert_converter.mlir import ir + +@torch.library.custom_op('vds_static::topk', mutates_args=()) +def native_topk(x: torch.Tensor, k: int) -> tuple[torch.Tensor, torch.Tensor]: + return torch.topk(x, k, dim=-1, largest=True, sorted=True) + +@native_topk.register_fake +def fake_topk(x, k): + shape = (*x.shape[:-1], k) + return x.new_empty(shape), x.new_empty(shape, dtype=torch.int64) + +@registry.lower(torch.ops.vds_static.topk.default) +def lower_topk(ctx, x, k): + values, indices = utils.node_meta_to_ir_types(ctx.node) + i32 = ir.IntegerType.get_signless(32) + index_type = ir.RankedTensorType.get(indices.shape, i32) + result = _ir_operation('tfl.topk_v2', results=[values, index_type], + operands=[x, utils.splat(k, i32, [])]) + return result[0], _ir_operation('tfl.cast', results=[indices], operands=[result[1]]) + +def topk(x, k, dim=-1, largest=True, sorted=True): + dim = dim % x.ndim + moved = x.movedim(dim, -1) + values, indices = torch.ops.vds_static.topk(moved if largest else -moved, k) + if not largest: + values = -values + return values.movedim(-1, dim), indices.movedim(-1, dim) + +fx_infra.decomp.update_pre_convert_decomp({torch.ops.aten.topk.default: topk}) +fx_infra.decomp.update_pre_lower_decomp({torch.ops.aten.topk.default: topk}) diff --git a/models/fireviewer_dinov3_pointing_pilot_v1_learning/README.md b/models/fireviewer_dinov3_pointing_pilot_v1_learning/README.md index be69dc02526415423d791b62031eecc7677842b8..ad1f3c9000be9c3fd04eed50d281100ed7c3f316 100644 --- a/models/fireviewer_dinov3_pointing_pilot_v1_learning/README.md +++ b/models/fireviewer_dinov3_pointing_pilot_v1_learning/README.md @@ -4,7 +4,7 @@ Source: `fireviewer/dinov3-pointing-pilot-v1`, revision `bd4cc7e15e296fa95b1cd94 This is a real mutable LiteRT model with `infer`, `train`, `save`, and `restore` signatures. Training executes on Android using the image and reviewed targets. Trainable scope: **four_task_output_adaptation_only**. The visual backbone remains frozen. Frozen DINOv3 and DPT; only task-output calibration/mixing adapts. Pilot, private research use. -The original inference family is retained under `models/fireviewer_dinov3_pointing_pilot_v1`. This variant has its own explicit Android configuration. +This learning variant is stored separately under `models/fireviewer_dinov3_pointing_pilot_v1_learning` and has its own explicit Android configuration. Variable image dimensions: `{'mode': 'bilinear_stretch_inside_graph', 'native_backbone_shape': [1, 3, 448, 448]}`. The Android configuration chooses a bounded input canvas; the image may have any original size. Images are processed individually, so a dataset need not have homogeneous dimensions. diff --git a/models/fireviewer_dinov3_pointing_pilot_v1_learning/artifact_manifest.json b/models/fireviewer_dinov3_pointing_pilot_v1_learning/artifact_manifest.json index 13ca23656c5f3e03452f56b5443ca06b8668b6f1..7fa436fdc5f2a1551ae15b4d8c12e52af1f05ccc 100644 --- a/models/fireviewer_dinov3_pointing_pilot_v1_learning/artifact_manifest.json +++ b/models/fireviewer_dinov3_pointing_pilot_v1_learning/artifact_manifest.json @@ -4,7 +4,7 @@ "bytes": 368 }, "README.md": { - "sha256": "f0972394a0439cc51556179a5d109a9b5612cdbe2eed3fd2a6be46330d49e1e4", + "sha256": "bba9d42bf0a747f2e46c1390c218d87e6826a7fdf400ea5c26b818cc26eed3c4", "bytes": 1742 }, "UPSTREAM_README.md": { diff --git a/models/fireviewer_rfdetr_medium_v110_learning/README.md b/models/fireviewer_rfdetr_medium_v110_learning/README.md index 419fd3512df1ab32652bb7950aabbd7e08577b43..46b9c8b3b03bae75c58ccec0d5578ec24b064f57 100644 --- a/models/fireviewer_rfdetr_medium_v110_learning/README.md +++ b/models/fireviewer_rfdetr_medium_v110_learning/README.md @@ -4,7 +4,7 @@ Source: `fireviewer/fire-smoke-rfdetr-medium-v110-strict-v1`, revision `901382d9 This is a real mutable LiteRT model with `infer`, `train`, `save`, and `restore` signatures. Training executes on Android using the image and reviewed targets. Trainable scope: **candidate_classification_and_box_adaptation_only**. The visual backbone remains frozen. Does not create proposals absent from frozen detector -The original inference family is retained under `models/fireviewer_rfdetr_medium_v110`. This variant has its own explicit Android configuration. +This learning variant is stored separately under `models/fireviewer_rfdetr_medium_v110_learning` and has its own explicit Android configuration. Variable image dimensions: `{'mode': 'bilinear_stretch_inside_graph', 'native_backbone_shape': [1, 3, 576, 576]}`. The Android configuration chooses a bounded input canvas; the image may have any original size. Images are processed individually, so a dataset need not have homogeneous dimensions. diff --git a/models/fireviewer_rfdetr_medium_v110_learning/artifact_manifest.json b/models/fireviewer_rfdetr_medium_v110_learning/artifact_manifest.json index 956e6dcd88fd50908cc4997363e1a71fc955137d..e89e0e45d3191b5d8d0bdd616749677c7fb578c3 100644 --- a/models/fireviewer_rfdetr_medium_v110_learning/artifact_manifest.json +++ b/models/fireviewer_rfdetr_medium_v110_learning/artifact_manifest.json @@ -8,7 +8,7 @@ "bytes": 253 }, "README.md": { - "sha256": "d9bff0f5082a744063b9a3fee74e970fa76a2b4ec26ca8ccd958216480df1104", + "sha256": "114bdc9b3998b19f6a6314092f3ac6fb54e0e9f4b03705347ab65017c20bd6ca", "bytes": 1730 }, "UPSTREAM_README.md": { diff --git a/models/fireviewer_rtdetr_v2_r50_learning/README.md b/models/fireviewer_rtdetr_v2_r50_learning/README.md index 7450410d24a88268cf8f346cfae4b52239731bec..df3fc5d4e3ecfb7bcd0f3076c9df96837a570a18 100644 --- a/models/fireviewer_rtdetr_v2_r50_learning/README.md +++ b/models/fireviewer_rtdetr_v2_r50_learning/README.md @@ -4,7 +4,7 @@ Source: `fireviewer/rtdetr-v2-r50-fire-smoke`, revision `b16bc79df05a69bfd90a713 This is a real mutable LiteRT model with `infer`, `train`, `save`, and `restore` signatures. Training executes on Android using the image and reviewed targets. Trainable scope: **candidate_classification_and_box_adaptation_only**. The visual backbone remains frozen. Does not create proposals absent from frozen detector -The original inference family is retained under `models/fireviewer_rtdetr_v2_r50`. This variant has its own explicit Android configuration. +This learning variant is stored separately under `models/fireviewer_rtdetr_v2_r50_learning` and has its own explicit Android configuration. Variable image dimensions: `{'mode': 'bilinear_stretch_inside_graph', 'native_backbone_shape': [1, 3, 768, 768]}`. The Android configuration chooses a bounded input canvas; the image may have any original size. Images are processed individually, so a dataset need not have homogeneous dimensions. diff --git a/models/fireviewer_rtdetr_v2_r50_learning/artifact_manifest.json b/models/fireviewer_rtdetr_v2_r50_learning/artifact_manifest.json index bfa68db436cf033b87dee84ae492a468ca383810..ba9601b4600979c604821533fde9b97c64906483 100644 --- a/models/fireviewer_rtdetr_v2_r50_learning/artifact_manifest.json +++ b/models/fireviewer_rtdetr_v2_r50_learning/artifact_manifest.json @@ -4,7 +4,7 @@ "bytes": 355 }, "README.md": { - "sha256": "0ce1d205407bdd4ec4345c3a998e3b8bb5f93924e56af1284784ca41bb10acd4", + "sha256": "5d72d8cb0d250e44207149f8f4b3cce47affb5c64f40fc57d4bfc13c0a4aab90", "bytes": 1694 }, "UPSTREAM_README.md": { diff --git a/reports/android/fire-build-final.log b/reports/android/fire-build-final.log new file mode 100644 index 0000000000000000000000000000000000000000..c7069978709ce38b98a36c8e8e1632027c94a6eb --- /dev/null +++ b/reports/android/fire-build-final.log @@ -0,0 +1,3 @@ +warning: [options] system modules path not set in conjunction with -source 11 +1 warning +/workspace/qa/vds-model-training/training-check.jar diff --git a/reports/android/fireviewer_yolo11m_strict_v1-final.log b/reports/android/fireviewer_yolo11m_strict_v1-final.log new file mode 100644 index 0000000000000000000000000000000000000000..c7ebeff5d8be1bc1ce1a5db2ee3145e31033f711 --- /dev/null +++ b/reports/android/fireviewer_yolo11m_strict_v1-final.log @@ -0,0 +1,25 @@ +CANNOT LINK EXECUTABLE "/system/bin/dex2oat": "/system/lib64/libart-compiler.so" is 64-bit instead of 32-bit +FIRE_CHECK opening yolo-final.tflite +INFO: Initialized TensorFlow Lite runtime. +INFO: Created TensorFlow Lite delegate for select TF ops. +native : core/platform/cpu_feature_guard.cc:210 This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: SSE3 SSE4.1 SSE4.2, in other operations, rebuild TensorFlow with the appropriate compiler flags. +INFO: TfLiteFlexDelegate delegate: 0 nodes delegated out of 1270 nodes with 0 partitions. + +VERBOSE: Replacing 9 out of 29 node(s) with delegate (TfLiteFlexDelegate) node, yielding 3 partitions for the whole graph. +native : core/framework/node_def_util.cc:676 NodeDef mentions attribute use_inter_op_parallelism which is not in the op definition: Op tensor:dt; attr=dt:type; attr=preferred_shard:int,default=-1; is_stateful=true> This may be expected if your graph generating binary is newer than this binary. Unknown attributes will be ignored. NodeDef: {{node Restore}} +VERBOSE: Replacing 1 out of 20 node(s) with delegate (TfLiteFlexDelegate) node, yielding 2 partitions for the whole graph. +VERBOSE: Replacing 9 out of 1537 node(s) with delegate (TfLiteFlexDelegate) node, yielding 5 partitions for the whole graph. +FIRE_CHECK opened +FIRE_CHECK infer 504 +FIRE_CHECK saved loss=7.3072906 +FIRE_CHECK continued loss=7.0748734 +VERBOSE: Replacing 9 out of 29 node(s) with delegate (TfLiteFlexDelegate) node, yielding 3 partitions for the whole graph. +native : core/framework/node_def_util.cc:676 NodeDef mentions attribute use_inter_op_parallelism which is not in the op definition: Op tensor:dt; attr=dt:type; attr=preferred_shard:int,default=-1; is_stateful=true> This may be expected if your graph generating binary is newer than this binary. Unknown attributes will be ignored. NodeDef: {{node Restore}} +VERBOSE: Replacing 1 out of 20 node(s) with delegate (TfLiteFlexDelegate) node, yielding 2 partitions for the whole graph. +VERBOSE: Replacing 9 out of 1537 node(s) with delegate (TfLiteFlexDelegate) node, yielding 5 partitions for the whole graph. +VERBOSE: Replacing 9 out of 29 node(s) with delegate (TfLiteFlexDelegate) node, yielding 3 partitions for the whole graph. +native : core/framework/node_def_util.cc:676 NodeDef mentions attribute use_inter_op_parallelism which is not in the op definition: Op tensor:dt; attr=dt:type; attr=preferred_shard:int,default=-1; is_stateful=true> This may be expected if your graph generating binary is newer than this binary. Unknown attributes will be ignored. NodeDef: {{node Restore}} +VERBOSE: Replacing 1 out of 20 node(s) with delegate (TfLiteFlexDelegate) node, yielding 2 partitions for the whole graph. +VERBOSE: Replacing 9 out of 1537 node(s) with delegate (TfLiteFlexDelegate) node, yielding 5 partitions for the whole graph. +ANDROID_FIRE_TRAINING_PASS {"runtime":"2.16.1","threads":1,"mixed_shapes":[[64,64],[64,96]],"train":true,"restore_exact":true,"optimizer_resume_exact":true,"incompatible_schema_rejected":true,"first_loss":7.3072906,"second_loss":7.0748734} diff --git a/reports/android/fireviewer_yolo11m_strict_v1.log b/reports/android/fireviewer_yolo11m_strict_v1.log new file mode 100644 index 0000000000000000000000000000000000000000..86101f5e373342bb09e7e833fb72f1010dae066b --- /dev/null +++ b/reports/android/fireviewer_yolo11m_strict_v1.log @@ -0,0 +1,25 @@ +CANNOT LINK EXECUTABLE "/system/bin/dex2oat": "/system/lib64/libart-compiler.so" is 64-bit instead of 32-bit +FIRE_CHECK opening yolo.tflite +INFO: Initialized TensorFlow Lite runtime. +INFO: Created TensorFlow Lite delegate for select TF ops. +native : core/platform/cpu_feature_guard.cc:210 This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: SSE3 SSE4.1 SSE4.2, in other operations, rebuild TensorFlow with the appropriate compiler flags. +INFO: TfLiteFlexDelegate delegate: 0 nodes delegated out of 1270 nodes with 0 partitions. + +VERBOSE: Replacing 9 out of 29 node(s) with delegate (TfLiteFlexDelegate) node, yielding 3 partitions for the whole graph. +native : core/framework/node_def_util.cc:676 NodeDef mentions attribute use_inter_op_parallelism which is not in the op definition: Op tensor:dt; attr=dt:type; attr=preferred_shard:int,default=-1; is_stateful=true> This may be expected if your graph generating binary is newer than this binary. Unknown attributes will be ignored. NodeDef: {{node Restore}} +VERBOSE: Replacing 1 out of 20 node(s) with delegate (TfLiteFlexDelegate) node, yielding 2 partitions for the whole graph. +VERBOSE: Replacing 9 out of 1532 node(s) with delegate (TfLiteFlexDelegate) node, yielding 5 partitions for the whole graph. +FIRE_CHECK opened +FIRE_CHECK infer 2016 +FIRE_CHECK saved loss=7.079305 +FIRE_CHECK continued loss=7.035917 +VERBOSE: Replacing 9 out of 29 node(s) with delegate (TfLiteFlexDelegate) node, yielding 3 partitions for the whole graph. +native : core/framework/node_def_util.cc:676 NodeDef mentions attribute use_inter_op_parallelism which is not in the op definition: Op tensor:dt; attr=dt:type; attr=preferred_shard:int,default=-1; is_stateful=true> This may be expected if your graph generating binary is newer than this binary. Unknown attributes will be ignored. NodeDef: {{node Restore}} +VERBOSE: Replacing 1 out of 20 node(s) with delegate (TfLiteFlexDelegate) node, yielding 2 partitions for the whole graph. +VERBOSE: Replacing 9 out of 1532 node(s) with delegate (TfLiteFlexDelegate) node, yielding 5 partitions for the whole graph. +VERBOSE: Replacing 9 out of 29 node(s) with delegate (TfLiteFlexDelegate) node, yielding 3 partitions for the whole graph. +native : core/framework/node_def_util.cc:676 NodeDef mentions attribute use_inter_op_parallelism which is not in the op definition: Op tensor:dt; attr=dt:type; attr=preferred_shard:int,default=-1; is_stateful=true> This may be expected if your graph generating binary is newer than this binary. Unknown attributes will be ignored. NodeDef: {{node Restore}} +VERBOSE: Replacing 1 out of 20 node(s) with delegate (TfLiteFlexDelegate) node, yielding 2 partitions for the whole graph. +VERBOSE: Replacing 9 out of 1532 node(s) with delegate (TfLiteFlexDelegate) node, yielding 5 partitions for the whole graph. +ANDROID_FIRE_TRAINING_PASS {"runtime":"2.16.1","mixed_shapes":[[128,128],[128,192]],"train":true,"restore_exact":true,"optimizer_resume_exact":true,"incompatible_schema_rejected":true,"first_loss":7.079305,"second_loss":7.035917} diff --git a/reports/android/runtime-libraries.json b/reports/android/runtime-libraries.json new file mode 100644 index 0000000000000000000000000000000000000000..ddfce81b05086df788955432f6b983f2908dad85 --- /dev/null +++ b/reports/android/runtime-libraries.json @@ -0,0 +1,12 @@ +[ + { + "url": "https://repo.maven.apache.org/maven2/org/tensorflow/tensorflow-lite/2.16.1/tensorflow-lite-2.16.1.aar", + "file": "libtensorflowlite_jni.so", + "sha256": "21f2570e650fc120ad28a379d927392c1883f9738895a3fd5efa0a32ae784507" + }, + { + "url": "https://repo.maven.apache.org/maven2/org/tensorflow/tensorflow-lite-select-tf-ops/2.16.1/tensorflow-lite-select-tf-ops-2.16.1.aar", + "file": "libtensorflowlite_flex_jni.so", + "sha256": "481de882bef6cc4389c974c5e4b718f77f230bdd697d56dd9937e55430727494" + } +] \ No newline at end of file diff --git a/reports/environments/litert-environment.txt b/reports/environments/litert-environment.txt new file mode 100644 index 0000000000000000000000000000000000000000..4336deb55027e5e9eb6e1c85b092ab60885022a2 --- /dev/null +++ b/reports/environments/litert-environment.txt @@ -0,0 +1,108 @@ +absl-py==2.5.0 +accelerate==1.15.0 +ai-edge-litert==2.2.0 +ai-edge-quantizer==0.9.0 +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.14.2 +av==18.1.0 +backports-strenum==1.3.1 +certifi==2026.7.22 +charset-normalizer==3.5.1 +click==8.5.0 +contourpy==1.4.0 +cuda-bindings==13.4.2 +cuda-pathfinder==1.8.2 +cuda-toolkit==13.0.3.0 +cycler==0.12.1 +defusedxml==0.7.1 +filelock==4.0.1 +fire==0.7.1 +flatbuffers==25.12.19 +fonttools==4.65.0 +fsspec==2026.9.0 +h11==0.16.0 +hf-xet==1.6.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface-hub==1.32.0 +idna==3.20 +immutabledict==4.2.1 +jax==0.11.2 +jaxlib==0.11.2 +jaxtyping==0.3.11 +jinja2==3.1.6 +kagglehub==1.0.2 +kagglesdk==0.1.37 +kiwisolver==1.5.1 +lark==1.3.1 +litert-converter==0.4.0 +litert-lm-builder==0.17.1 +litert-torch==0.9.4 +markdown-it-py==4.2.0 +markupsafe==3.0.3 +matplotlib==3.11.2 +mdurl==0.1.2 +ml-dtypes==0.6.0 +mpmath==1.3.0 +multipledispatch==1.0.0 +networkx==3.6.1 +numpy==2.5.3 +nvidia-cublas==13.1.1.3 +nvidia-cuda-cupti==13.0.85 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-runtime==13.0.96 +nvidia-cudnn-cu13==9.20.0.48 +nvidia-cufft==12.0.0.61 +nvidia-cufile==1.15.1.6 +nvidia-curand==10.4.0.35 +nvidia-cusolver==12.0.4.66 +nvidia-cusparse==12.6.3.3 +nvidia-cusparselt-cu13==0.8.1 +nvidia-nccl-cu13==2.29.7 +nvidia-nvjitlink==13.4.92 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvtx==13.0.85 +opt-einsum==3.4.0 +ordered-set==4.1.0 +packaging==26.3 +pillow==12.3.0 +pip==26.2.1 +protobuf==7.36.2 +psutil==7.2.2 +pydantic==2.11.10 +pydantic-core==2.33.2 +pydeprecate==0.9.0 +pygments==2.21.0 +pyparsing==3.3.2 +python-dateutil==2.9.0.post0 +pyyaml==6.0.3 +regex==2026.9.10 +requests==2.34.2 +rfdetr @ git+https://github.com/roboflow/rf-detr.git@5b39cbc0a0ef75ab9c9571ccb28b780c7da51432 +rich==15.0.0 +safetensors==0.8.0 +scipy==1.18.1 +sentencepiece==0.2.2 +setuptools==84.0.0 +shellingham==1.5.4 +six==1.17.0 +supervision==0.30.4 +sympy==1.14.0 +tabulate==0.10.0 +termcolor==3.3.0 +timm==1.0.29 +tokenizers==0.23.2 +tomli==2.4.1 +torch==2.13.0 +torchao==0.18.0 +torchvision==0.28.0 +tqdm==4.70.1 +transformers==5.17.0 +triton==3.7.1 +typer==0.27.2 +typing-extensions==4.12.2 +typing-inspection==0.4.2 +urllib3==2.8.0 +wadler-lindig==0.1.7 +xdsl==0.28.0 diff --git a/reports/environments/mm-environment.txt b/reports/environments/mm-environment.txt new file mode 100644 index 0000000000000000000000000000000000000000..18c0a9bb1721c63f28ff29f9cfd0da38758bd2da --- /dev/null +++ b/reports/environments/mm-environment.txt @@ -0,0 +1,68 @@ +addict==2.4.0 +certifi==2026.7.22 +charset-normalizer==3.5.1 +cmake==4.4.3 +contourpy==1.3.2 +cycler==0.12.1 +filelock==4.0.1 +fonttools==4.65.0 +fsspec==2026.9.0 +idna==3.20 +jinja2==3.1.6 +kiwisolver==1.5.1 +lit==23.1.1 +markdown-it-py==4.2.0 +markupsafe==3.0.3 +matplotlib==3.10.9 +mdurl==0.1.2 +mmcv @ https://download.openmmlab.com/mmcv/dist/cu118/torch2.0.0/mmcv-2.0.1-cp310-cp310-manylinux1_x86_64.whl +mmdet==3.3.0 +mmengine==0.10.7 +-e file:///workspace/vds-litert-batch2/vendor/mmyolo +mpmath==1.3.0 +networkx==3.4.2 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==8.9.2.26 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.18.1 +nvidia-nvjitlink-cu12==12.9.86 +nvidia-nvtx-cu12==12.1.105 +onnx==1.16.2 +onnxsim==0.7.3 +opencv-python==4.11.0.86 +packaging==26.3 +pillow==12.3.0 +platformdirs==4.11.11 +prettytable==3.18.0 +protobuf==7.36.2 +pycocotools==2.0.11 +pygments==2.21.0 +pyparsing==3.3.2 +python-dateutil==2.9.0.post0 +pyyaml==6.0.3 +requests==2.34.2 +rich==15.0.0 +scipy==1.15.3 +setuptools==69.5.1 +shapely==2.1.2 +six==1.17.0 +sympy==1.14.0 +termcolor==3.3.0 +terminaltables==3.1.10 +tomli==2.4.1 +torch @ https://download.pytorch.org/whl/cu118/torch-2.0.1%2Bcu118-cp310-cp310-linux_x86_64.whl +torchvision @ https://download.pytorch.org/whl/cu118/torchvision-0.15.2%2Bcu118-cp310-cp310-linux_x86_64.whl +tqdm==4.70.1 +triton==2.0.0 +typing-extensions==4.16.0 +urllib3==2.8.0 +wcwidth==0.8.4 +wheel==0.48.0 +yapf==0.43.0 diff --git a/reports/environments/onnx-environment.txt b/reports/environments/onnx-environment.txt new file mode 100644 index 0000000000000000000000000000000000000000..2e87f9ac4c4d9904e7c52018a2e5bf0bb897d711 --- /dev/null +++ b/reports/environments/onnx-environment.txt @@ -0,0 +1,43 @@ +ai-edge-litert==2.1.2 +anyio==4.15.1 +backports-strenum==1.3.1 +certifi==2026.7.22 +flatbuffers==25.12.19 +h11==0.16.0 +h5py==3.14.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.20 +iniconfig==2.3.0 +markdown-it-py==4.2.0 +mdurl==0.1.2 +ml-dtypes==0.5.4 +mpmath==1.3.0 +networkx==3.6.1 +numpy==2.2.6 +onnx==1.20.1 +onnx-ir==1.0.0 +onnx2tf==2.6.9 +onnxoptimizer==0.4.2 +onnxruntime==1.26.0 +onnxscript==0.6.2 +onnxsim==0.6.5 +opencv-python==4.13.0.92 +opt-einsum==3.3.0 +packaging==26.3 +paddle2onnx==1.3.1 +paddlepaddle==3.3.1 +pillow==12.3.0 +pluggy==1.6.0 +protobuf==7.35.1 +psutil==5.9.5 +pygments==2.21.0 +pytest==9.0.2 +rich==15.0.0 +safetensors==0.8.0 +setuptools==81.0.0 +sne4onnx==2.0.1 +sng4onnx==2.0.1 +sympy==1.14.0 +tqdm==4.67.1 +typing-extensions==4.16.0 diff --git a/reports/environments/runtime-environment.json b/reports/environments/runtime-environment.json new file mode 100644 index 0000000000000000000000000000000000000000..36f847bd5d22d47b927ea495dca722575aa56f64 --- /dev/null +++ b/reports/environments/runtime-environment.json @@ -0,0 +1,14 @@ +{ + "python": "3.12.3", + "torch": "2.13.0+cu130", + "cuda_available": true, + "torch_cuda": "13.0", + "device": "NVIDIA RTX 4000 Ada Generation", + "cuda_sum": 499500.0, + "litert-torch": "0.9.4", + "ai-edge-litert": "2.2.0", + "transformers": "5.17.0", + "timm": "1.0.29", + "huggingface_hub": "1.32.0", + "rfdetr": "1.11.0.dev0" +} \ No newline at end of file diff --git a/reports/environments/tf-current-freeze.txt b/reports/environments/tf-current-freeze.txt new file mode 100644 index 0000000000000000000000000000000000000000..e5a0d6252a704e4d7a6fc40b8fb286f85ba37e6e --- /dev/null +++ b/reports/environments/tf-current-freeze.txt @@ -0,0 +1,52 @@ +absl-py==2.5.0 +ai-edge-litert==2.1.2 +astunparse==1.6.3 +backports-strenum==1.3.1 +certifi==2026.7.22 +charset-normalizer==3.5.1 +flatbuffers==25.12.19 +gast==0.7.0 +google-pasta==0.2.0 +grpcio==1.84.0 +h5py==3.14.0 +idna==3.20 +iniconfig==2.3.0 +keras==3.15.0 +libclang==18.1.1 +markdown-it-py==4.2.0 +mdurl==0.1.2 +ml-dtypes==0.5.4 +mpmath==1.3.0 +namex==0.1.0 +numpy==2.2.6 +onnx==1.20.1 +onnx-ir==1.0.0 +onnx2tf==2.6.9 +onnxoptimizer==0.4.2 +onnxruntime==1.26.0 +onnxscript==0.6.2 +onnxsim==0.6.5 +opencv-python==4.13.0.92 +opt-einsum==3.4.0 +optree==0.20.0 +packaging==26.3 +pluggy==1.6.0 +protobuf==7.35.1 +psutil==5.9.5 +pygments==2.21.0 +pytest==9.0.2 +requests==2.34.2 +rich==15.0.0 +setuptools==81.0.0 +six==1.17.0 +sne4onnx==2.0.1 +sng4onnx==2.0.1 +sympy==1.14.0 +tensorflow==2.21.0 +termcolor==3.3.0 +tf-keras==2.21.0 +tqdm==4.67.1 +typing-extensions==4.16.0 +urllib3==2.8.0 +wheel==0.48.0 +wrapt==2.4.1 diff --git a/reports/environments/tf-probe-freeze.txt b/reports/environments/tf-probe-freeze.txt new file mode 100644 index 0000000000000000000000000000000000000000..89d03ddea1f48cfd5640ec3564e8702060ba17a1 --- /dev/null +++ b/reports/environments/tf-probe-freeze.txt @@ -0,0 +1,49 @@ +absl-py==2.5.0 +anyio==4.15.1 +astunparse==1.6.3 +certifi==2026.7.22 +charset-normalizer==3.5.1 +click==8.5.0 +filelock==4.0.1 +flatbuffers==25.12.19 +fsspec==2026.9.0 +gast==0.7.0 +google-pasta==0.2.0 +grpcio==1.84.0 +h11==0.16.0 +h5py==3.16.0 +hf-xet==1.6.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface-hub==1.32.0 +idna==3.20 +keras==3.15.1 +libclang==18.1.1 +markdown==3.10.3 +markdown-it-py==4.2.0 +markupsafe==3.0.3 +mdurl==0.1.2 +ml-dtypes==0.6.0 +namex==0.1.0 +numpy==2.1.3 +opt-einsum==3.4.0 +optree==0.20.0 +packaging==26.3 +protobuf==5.29.6 +pygments==2.21.0 +pyyaml==6.0.3 +requests==2.34.2 +rich==15.0.0 +setuptools==84.0.0 +six==1.17.0 +tensorboard==2.19.0 +tensorboard-data-server==0.7.2 +tensorflow==2.19.0 +termcolor==3.3.0 +tf-keras==2.19.0 +tqdm==4.70.1 +typing-extensions==4.16.0 +urllib3==2.8.0 +werkzeug==3.1.8 +wheel==0.48.0 +wrapt==2.4.1 diff --git a/reports/environments/tf216-freeze.txt b/reports/environments/tf216-freeze.txt new file mode 100644 index 0000000000000000000000000000000000000000..f470dc4ce64cac249b471fec6e9a02048603110c --- /dev/null +++ b/reports/environments/tf216-freeze.txt @@ -0,0 +1,37 @@ +absl-py==2.5.0 +astunparse==1.6.3 +certifi==2026.7.22 +charset-normalizer==3.5.1 +flatbuffers==25.12.19 +gast==0.7.0 +google-pasta==0.2.0 +grpcio==1.84.0 +h5py==3.16.0 +idna==3.20 +keras==3.15.1 +libclang==18.1.1 +markdown==3.10.3 +markdown-it-py==4.2.0 +markupsafe==3.0.3 +mdurl==0.1.2 +ml-dtypes==0.3.2 +namex==0.1.0 +numpy==1.26.4 +opt-einsum==3.4.0 +optree==0.20.0 +packaging==26.3 +protobuf==4.25.9 +pygments==2.21.0 +requests==2.34.2 +rich==15.0.0 +setuptools==84.0.0 +six==1.17.0 +tensorboard==2.16.2 +tensorboard-data-server==0.7.2 +tensorflow==2.16.1 +termcolor==3.3.0 +typing-extensions==4.16.0 +urllib3==2.8.0 +werkzeug==3.1.8 +wheel==0.48.0 +wrapt==2.4.1 diff --git a/reports/environments/tf219-freeze.txt b/reports/environments/tf219-freeze.txt new file mode 100644 index 0000000000000000000000000000000000000000..89d03ddea1f48cfd5640ec3564e8702060ba17a1 --- /dev/null +++ b/reports/environments/tf219-freeze.txt @@ -0,0 +1,49 @@ +absl-py==2.5.0 +anyio==4.15.1 +astunparse==1.6.3 +certifi==2026.7.22 +charset-normalizer==3.5.1 +click==8.5.0 +filelock==4.0.1 +flatbuffers==25.12.19 +fsspec==2026.9.0 +gast==0.7.0 +google-pasta==0.2.0 +grpcio==1.84.0 +h11==0.16.0 +h5py==3.16.0 +hf-xet==1.6.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface-hub==1.32.0 +idna==3.20 +keras==3.15.1 +libclang==18.1.1 +markdown==3.10.3 +markdown-it-py==4.2.0 +markupsafe==3.0.3 +mdurl==0.1.2 +ml-dtypes==0.6.0 +namex==0.1.0 +numpy==2.1.3 +opt-einsum==3.4.0 +optree==0.20.0 +packaging==26.3 +protobuf==5.29.6 +pygments==2.21.0 +pyyaml==6.0.3 +requests==2.34.2 +rich==15.0.0 +setuptools==84.0.0 +six==1.17.0 +tensorboard==2.19.0 +tensorboard-data-server==0.7.2 +tensorflow==2.19.0 +termcolor==3.3.0 +tf-keras==2.19.0 +tqdm==4.70.1 +typing-extensions==4.16.0 +urllib3==2.8.0 +werkzeug==3.1.8 +wheel==0.48.0 +wrapt==2.4.1 diff --git a/reports/fireviewer-completion.json b/reports/fireviewer-completion.json new file mode 100644 index 0000000000000000000000000000000000000000..f84e827e130f7253462f0a632840f874ab475d04 --- /dev/null +++ b/reports/fireviewer-completion.json @@ -0,0 +1,169 @@ +{ + "generated_at": "2026-09-20T00:35:11.670200+00:00", + "repo": "fireviewer/litert-models", + "models": [ + { + "key": "fireviewer_dfine_m_strict_v1", + "title": "FireViewer D-FINE-M strict v1", + "source": "fireviewer/fire-smoke-dfine-m-strict-v1", + "status": "published_verified", + "publication": { + "repo": "fireviewer/litert-models", + "commit": "dee2a0f6204cafff559244d3555a79c49023c57e", + "path": "models/fireviewer_dfine_m_strict_v1_learning/model.tflite", + "sha256": "b66acda95a5eaf5154031cf80c1ed906b8cb7838617424afa9b837c5cd44064a", + "bytes": 79120652, + "remote_verified": true + }, + "training_scope": "candidate_classification_and_box_adaptation_only", + "backbone_frozen": true, + "variable_image_preprocessing": { + "mode": "bilinear_stretch_inside_graph", + "native_backbone_shape": [ + 1, + 3, + 704, + 704 + ] + }, + "host_training_passed": true, + "host_runtime216_passed": true, + "android_emulator_passed": false, + "physical_arm_phone_tested": false, + "accuracy_evaluated": false + }, + { + "key": "fireviewer_rtdetr_v2_r50", + "title": "FireViewer RT-DETRv2-R50", + "source": "fireviewer/rtdetr-v2-r50-fire-smoke", + "status": "published_verified", + "publication": { + "repo": "fireviewer/litert-models", + "commit": "df60095dbaed66e8c0906b69bb0561c7ad71b3ac", + "path": "models/fireviewer_rtdetr_v2_r50_learning/model.tflite", + "sha256": "cd2c38cf21c139cf1f9fac45b4ccd6570b8cc3d8d4e5af21561e81c4e13ca565", + "bytes": 177870964, + "remote_verified": true + }, + "training_scope": "candidate_classification_and_box_adaptation_only", + "backbone_frozen": true, + "variable_image_preprocessing": { + "mode": "bilinear_stretch_inside_graph", + "native_backbone_shape": [ + 1, + 3, + 768, + 768 + ] + }, + "host_training_passed": true, + "host_runtime216_passed": true, + "android_emulator_passed": false, + "physical_arm_phone_tested": false, + "accuracy_evaluated": false + }, + { + "key": "fireviewer_yolo11m_strict_v1", + "title": "FireViewer YOLO11-M strict v1", + "source": "fireviewer/fire-smoke-yolo11m-strict-v1", + "status": "published_verified", + "publication": { + "repo": "fireviewer/litert-models", + "commit": "ae756eaffe4c953e1ad6a6252830f661c9e11e1e", + "path": "models/fireviewer_yolo11m_strict_v1_learning/model.tflite", + "sha256": "f05130795e14427402d1f6f0c28b777d43bd0909be9888b776a3c958e74e222b", + "bytes": 80800592, + "remote_verified": true + }, + "training_scope": "candidate_classification_and_box_adaptation_only", + "backbone_frozen": true, + "variable_image_preprocessing": { + "mode": "native_dynamic", + "stride": 32 + }, + "host_training_passed": true, + "host_runtime216_passed": true, + "android_emulator_passed": true, + "physical_arm_phone_tested": false, + "accuracy_evaluated": false + }, + { + "key": "fireviewer_rfdetr_medium_v110", + "title": "FireViewer RF-DETR Medium 1.10 strict v1", + "source": "fireviewer/fire-smoke-rfdetr-medium-v110-strict-v1", + "status": "published_verified", + "publication": { + "repo": "fireviewer/litert-models", + "commit": "22e5e0f48a2996e85accf851e4fb7ce59b3cf469", + "path": "models/fireviewer_rfdetr_medium_v110_learning/model.tflite", + "sha256": "dba3ade58743b422d492448605e1cb7b709de66323dd0fbdbd930f12af7bad20", + "bytes": 124411016, + "remote_verified": true + }, + "training_scope": "candidate_classification_and_box_adaptation_only", + "backbone_frozen": true, + "variable_image_preprocessing": { + "mode": "bilinear_stretch_inside_graph", + "native_backbone_shape": [ + 1, + 3, + 576, + 576 + ] + }, + "host_training_passed": true, + "host_runtime216_passed": true, + "android_emulator_passed": false, + "physical_arm_phone_tested": false, + "accuracy_evaluated": false + }, + { + "key": "fireviewer_dinov3_pointing_pilot_v1", + "title": "FireViewer DINOv3 pointing pilot v1", + "source": "fireviewer/dinov3-pointing-pilot-v1", + "status": "published_verified", + "publication": { + "repo": "fireviewer/litert-models", + "commit": "eb97d601b6287b64e7f2a192f0072c2282b40547", + "path": "models/fireviewer_dinov3_pointing_pilot_v1_learning/model.tflite", + "sha256": "6843e9dd525b76678c53e310939f0f63ccae5df084e012e7bc6442eb7a5b7061", + "bytes": 378204504, + "remote_verified": true + }, + "training_scope": "four_task_output_adaptation_only", + "backbone_frozen": true, + "variable_image_preprocessing": { + "mode": "bilinear_stretch_inside_graph", + "native_backbone_shape": [ + 1, + 3, + 448, + 448 + ] + }, + "host_training_passed": true, + "host_runtime216_passed": true, + "android_emulator_passed": false, + "physical_arm_phone_tested": false, + "accuracy_evaluated": false + }, + { + "key": "fireviewer_segformer_b2_baseline_v1", + "title": "FireViewer SegFormer-B2 baseline v1", + "source": "fireviewer/segformer-b2-fire-smoke-baseline-v1", + "status": "source_unavailable", + "reason": "Repository absent from authenticated FireViewer organization listing" + } + ], + "published_count": 5, + "training": "On-device adaptation heads; all visual backbones frozen", + "android_runtime": [ + "org.tensorflow:tensorflow-lite:2.16.1", + "org.tensorflow:tensorflow-lite-select-tf-ops:2.16.1" + ], + "app_source_changed": false, + "other_catalog_work": "paused at user request", + "conversion_pod": "7iz8rjvy5uld83", + "conversion_gpu": "RTX 4000 Ada", + "pod_stop": "performed after final archive and publication verification; local stop receipt is separate" +} diff --git a/reports/fireviewer_dfine_m_strict_v1/publication_receipt.json b/reports/fireviewer_dfine_m_strict_v1/publication_receipt.json new file mode 100644 index 0000000000000000000000000000000000000000..ce4b49402577fd8dc2c385810453c44cec91ff0a --- /dev/null +++ b/reports/fireviewer_dfine_m_strict_v1/publication_receipt.json @@ -0,0 +1,8 @@ +{ + "repo": "fireviewer/litert-models", + "commit": "dee2a0f6204cafff559244d3555a79c49023c57e", + "path": "models/fireviewer_dfine_m_strict_v1_learning/model.tflite", + "sha256": "b66acda95a5eaf5154031cf80c1ed906b8cb7838617424afa9b837c5cd44064a", + "bytes": 79120652, + "remote_verified": true +} diff --git a/reports/fireviewer_dfine_m_strict_v1/runtime216_validation.json b/reports/fireviewer_dfine_m_strict_v1/runtime216_validation.json new file mode 100644 index 0000000000000000000000000000000000000000..f1c990d0ce433121df0e02519b36d4c851a036ce --- /dev/null +++ b/reports/fireviewer_dfine_m_strict_v1/runtime216_validation.json @@ -0,0 +1,11 @@ +{ + "key": "fireviewer_dfine_m_strict_v1", + "runtime": "2.16.1", + "platform": "Linux x86_64; same LiteRT release as Android, different ABI", + "sha256": "b66acda95a5eaf5154031cf80c1ed906b8cb7838617424afa9b837c5cd44064a", + "train": true, + "loss": 2.902190685272217, + "restart_exact": true, + "optimizer_resume_exact": true, + "two_object_fixture": true +} diff --git a/reports/fireviewer_dfine_m_strict_v1/source.json b/reports/fireviewer_dfine_m_strict_v1/source.json new file mode 100644 index 0000000000000000000000000000000000000000..e71493d383ae76abdbb444ed68833cabb690a21a --- /dev/null +++ b/reports/fireviewer_dfine_m_strict_v1/source.json @@ -0,0 +1,25 @@ +{ + "key": "fireviewer_dfine_m_strict_v1", + "kind": "detr", + "class_count": 2, + "labels": [ + "smoke_visible", + "flame_visible" + ], + "class_indices": [ + 0, + 1 + ], + "backbone": "/workspace/vds-litert-batch2/learning-backbones/fireviewer_dfine_m_strict_v1/model-protected.tflite", + "parity_report": "/workspace/vds-litert-batch2/learning-backbones/fireviewer_dfine_m_strict_v1/parity.json", + "native_dynamic": false, + "bindings": { + "features_logits": 0, + "features_boxes": 1 + }, + "app_size": 704, + "test_size": [ + 193, + 317 + ] +} diff --git a/reports/fireviewer_dfine_m_strict_v1/source_parity.json b/reports/fireviewer_dfine_m_strict_v1/source_parity.json new file mode 100644 index 0000000000000000000000000000000000000000..0431dd84b0dfd007ef13d394fe6dc360776fbe54 --- /dev/null +++ b/reports/fireviewer_dfine_m_strict_v1/source_parity.json @@ -0,0 +1,84 @@ +{ + "key": "fireviewer_dfine_m_strict_v1", + "phase": "validated_frozen_component", + "upstream_onnx_parity_passed": true, + "backbone": "/workspace/vds-litert-batch2/learning-backbones/fireviewer_dfine_m_strict_v1/model-protected.tflite", + "source_sha256": "999ef94cc0419cc70a3c136b6a1ad30fb8ed66570dcad005308279342abcc77e", + "sha256": "9dac6e73396d0e949da5b6b7960966b2ec971b89739b961b3e732b8830f8e63a", + "input_shape": [ + 1, + 3, + 704, + 704 + ], + "outputs": [ + { + "index": 0, + "shape": [ + 1, + 300, + 2 + ] + }, + { + "index": 1, + "shape": [ + 1, + 300, + 4 + ] + } + ], + "max_abs_errors": [ + 3.337860107421875e-05, + 9.834766387939453e-07, + 3.4332275390625e-05, + 4.172325134277344e-07, + 5.6743621826171875e-05, + 7.152557373046875e-07 + ], + "seeds": [ + 842, + 843, + 844 + ], + "atol": 0.002, + "rtol": 0.002, + "preserve_all_tensors": false, + "arena_protection": { + "tensors": [ + { + "index": 1354, + "name": "wa/model/decoder/decoder/integral/Softmax_output_0", + "shape": [ + 1200, + 33 + ] + }, + { + "index": 1857, + "name": "wa/model/decoder/decoder/integral_1/Softmax_output_0", + "shape": [ + 1200, + 33 + ] + }, + { + "index": 2338, + "name": "wa/model/decoder/decoder/integral_2/Softmax_output_0", + "shape": [ + 1200, + 33 + ] + }, + { + "index": 2816, + "name": "wa/model/decoder/decoder/integral_3/Softmax_output_0", + "shape": [ + 1200, + 33 + ] + } + ] + } +} diff --git a/reports/fireviewer_dfine_m_strict_v1/training_validation.json b/reports/fireviewer_dfine_m_strict_v1/training_validation.json new file mode 100644 index 0000000000000000000000000000000000000000..94236d6e72d2e38c9235bf8610b73580f742c8ca --- /dev/null +++ b/reports/fireviewer_dfine_m_strict_v1/training_validation.json @@ -0,0 +1,86 @@ +{ + "key": "fireviewer_dfine_m_strict_v1", + "validation_passed": true, + "training_scope": "candidate_classification_and_box_adaptation_only", + "backbone_frozen": true, + "limitation": "Does not create proposals absent from frozen detector", + "initial_detector_parity": true, + "parameters_changed": true, + "initial_loss": 2.38586688041687, + "final_loss": 1.0897295475006104, + "mixed_image_shapes": [ + [ + 193, + 317 + ], + [ + 193, + 381 + ], + [ + 257, + 317 + ] + ], + "restart_logits_exact": true, + "optimizer_resume_exact": true, + "variable_image_preprocessing": { + "mode": "bilinear_stretch_inside_graph", + "native_backbone_shape": [ + 1, + 3, + 704, + 704 + ] + }, + "android_device_tested": false, + "accuracy_evaluated": false, + "sha256": "b66acda95a5eaf5154031cf80c1ed906b8cb7838617424afa9b837c5cd44064a", + "bytes": 79120652, + "signatures": { + "infer": { + "inputs": [ + "x" + ], + "outputs": [ + "boxes", + "detections", + "logits" + ] + }, + "restore": { + "inputs": [ + "checkpoint_path" + ], + "outputs": [ + "step" + ] + }, + "save": { + "inputs": [ + "checkpoint_path" + ], + "outputs": [ + "step" + ] + }, + "train": { + "inputs": [ + "learning_rate", + "x", + "y" + ], + "outputs": [ + "loss", + "step" + ] + }, + "training_state": { + "inputs": [], + "outputs": [ + "head_kernel", + "step" + ] + } + } +} diff --git a/reports/fireviewer_dinov3_pointing_pilot_v1/publication_receipt.json b/reports/fireviewer_dinov3_pointing_pilot_v1/publication_receipt.json new file mode 100644 index 0000000000000000000000000000000000000000..234822c94d878be46a33ddbf0354f8101c8b3004 --- /dev/null +++ b/reports/fireviewer_dinov3_pointing_pilot_v1/publication_receipt.json @@ -0,0 +1,8 @@ +{ + "repo": "fireviewer/litert-models", + "commit": "eb97d601b6287b64e7f2a192f0072c2282b40547", + "path": "models/fireviewer_dinov3_pointing_pilot_v1_learning/model.tflite", + "sha256": "6843e9dd525b76678c53e310939f0f63ccae5df084e012e7bc6442eb7a5b7061", + "bytes": 378204504, + "remote_verified": true +} diff --git a/reports/fireviewer_dinov3_pointing_pilot_v1/runtime216_validation.json b/reports/fireviewer_dinov3_pointing_pilot_v1/runtime216_validation.json new file mode 100644 index 0000000000000000000000000000000000000000..a420f074f4361eded3984c1d4facb64deb43539b --- /dev/null +++ b/reports/fireviewer_dinov3_pointing_pilot_v1/runtime216_validation.json @@ -0,0 +1,11 @@ +{ + "key": "fireviewer_dinov3_pointing_pilot_v1", + "runtime": "2.16.1", + "platform": "Linux x86_64; same LiteRT release as Android, different ABI", + "sha256": "6843e9dd525b76678c53e310939f0f63ccae5df084e012e7bc6442eb7a5b7061", + "train": true, + "loss": 3.8938868045806885, + "restart_exact": true, + "optimizer_resume_exact": true, + "two_object_fixture": false +} diff --git a/reports/fireviewer_dinov3_pointing_pilot_v1/source.json b/reports/fireviewer_dinov3_pointing_pilot_v1/source.json new file mode 100644 index 0000000000000000000000000000000000000000..19ba784fa21083fcba2922383f44dd0f787a955a --- /dev/null +++ b/reports/fireviewer_dinov3_pointing_pilot_v1/source.json @@ -0,0 +1,11 @@ +{ + "key": "fireviewer_dinov3_pointing_pilot_v1", + "kind": "multitask", + "labels": [ + "flame_visible", + "smoke_visible" + ], + "backbone": "/workspace/vds-litert-batch2/learning-backbones/fireviewer_dinov3_pointing_pilot_v1/model.tflite", + "private_pilot_only": true, + "parity_report": "/workspace/vds-litert-batch2/learning-backbones/fireviewer_dinov3_pointing_pilot_v1/parity.json" +} \ No newline at end of file diff --git a/reports/fireviewer_dinov3_pointing_pilot_v1/source_parity.json b/reports/fireviewer_dinov3_pointing_pilot_v1/source_parity.json new file mode 100644 index 0000000000000000000000000000000000000000..2042f613768810cdd979063a89bbaee8d3d81501 --- /dev/null +++ b/reports/fireviewer_dinov3_pointing_pilot_v1/source_parity.json @@ -0,0 +1,17 @@ +{ + "source_pytorch_parity_passed": true, + "outputs": [ + "abstention_logits", + "point_logits", + "presence_logits", + "segmentation_logits" + ], + "max_abs_errors": [ + 7.152557373046875e-07, + 5.435943603515625e-05, + 6.198883056640625e-06, + 9.5367431640625e-05 + ], + "atol": 0.002, + "rtol": 0.002 +} diff --git a/reports/fireviewer_dinov3_pointing_pilot_v1/training_validation.json b/reports/fireviewer_dinov3_pointing_pilot_v1/training_validation.json new file mode 100644 index 0000000000000000000000000000000000000000..f86cfaaae2bc01b4b83749ea5380982c35fd3483 --- /dev/null +++ b/reports/fireviewer_dinov3_pointing_pilot_v1/training_validation.json @@ -0,0 +1,81 @@ +{ + "key": "fireviewer_dinov3_pointing_pilot_v1", + "validation_passed": true, + "training_scope": "four_task_output_adaptation_only", + "backbone_frozen": true, + "limitation": "Frozen DINOv3 and DPT; only task-output calibration/mixing adapts. Pilot, private research use.", + "initial_detector_parity": true, + "parameters_changed": true, + "all_four_tasks_changed": true, + "missing_supervision_no_update": true, + "initial_loss": 4.2297587394714355, + "final_loss": 3.050323009490967, + "mixed_image_shapes": [ + [ + 193, + 317 + ], + [ + 321, + 257 + ] + ], + "restart_logits_exact": true, + "optimizer_resume_exact": true, + "variable_image_preprocessing": { + "mode": "bilinear_stretch_inside_graph", + "native_backbone_shape": [ + 1, + 3, + 448, + 448 + ] + }, + "android_device_tested": false, + "accuracy_evaluated": false, + "sha256": "6843e9dd525b76678c53e310939f0f63ccae5df084e012e7bc6442eb7a5b7061", + "bytes": 378204504, + "signatures": { + "infer": { + "inputs": [ + "x" + ], + "outputs": [ + "abstention_logits", + "point_logits", + "presence_logits", + "segmentation_logits" + ] + }, + "restore": { + "inputs": [ + "checkpoint_path" + ], + "outputs": [ + "step" + ] + }, + "save": { + "inputs": [ + "checkpoint_path" + ], + "outputs": [ + "step" + ] + }, + "train": { + "inputs": [ + "abstention", + "learning_rate", + "presence", + "supervision", + "x", + "y" + ], + "outputs": [ + "loss", + "step" + ] + } + } +} diff --git a/reports/fireviewer_rfdetr_medium_v110/publication_receipt.json b/reports/fireviewer_rfdetr_medium_v110/publication_receipt.json new file mode 100644 index 0000000000000000000000000000000000000000..b092150adf68dbac6ef94bad522af73618f19863 --- /dev/null +++ b/reports/fireviewer_rfdetr_medium_v110/publication_receipt.json @@ -0,0 +1,8 @@ +{ + "repo": "fireviewer/litert-models", + "commit": "22e5e0f48a2996e85accf851e4fb7ce59b3cf469", + "path": "models/fireviewer_rfdetr_medium_v110_learning/model.tflite", + "sha256": "dba3ade58743b422d492448605e1cb7b709de66323dd0fbdbd930f12af7bad20", + "bytes": 124411016, + "remote_verified": true +} diff --git a/reports/fireviewer_rfdetr_medium_v110/runtime216_validation.json b/reports/fireviewer_rfdetr_medium_v110/runtime216_validation.json new file mode 100644 index 0000000000000000000000000000000000000000..d4235a227a43a5408132371570008651d38bde54 --- /dev/null +++ b/reports/fireviewer_rfdetr_medium_v110/runtime216_validation.json @@ -0,0 +1,11 @@ +{ + "key": "fireviewer_rfdetr_medium_v110", + "runtime": "2.16.1", + "platform": "Linux x86_64; same LiteRT release as Android, different ABI", + "sha256": "dba3ade58743b422d492448605e1cb7b709de66323dd0fbdbd930f12af7bad20", + "train": true, + "loss": 2.0823371410369873, + "restart_exact": true, + "optimizer_resume_exact": true, + "two_object_fixture": true +} diff --git a/reports/fireviewer_rfdetr_medium_v110/source.json b/reports/fireviewer_rfdetr_medium_v110/source.json new file mode 100644 index 0000000000000000000000000000000000000000..4094d6cd6020b18522480525d26d16505e931f7f --- /dev/null +++ b/reports/fireviewer_rfdetr_medium_v110/source.json @@ -0,0 +1,35 @@ +{ + "key": "fireviewer_rfdetr_medium_v110", + "kind": "detr", + "class_count": 3, + "backbone": "/workspace/vds-litert-batch2/learning-backbones/fireviewer_rfdetr_medium_v110/model.tflite", + "native_dynamic": false, + "bindings": { + "features_boxes": 0, + "features_logits": 1 + }, + "app_size": 576, + "test_size": [ + 224, + 224 + ], + "channelMean": [ + 123.675, + 116.28, + 103.53 + ], + "channelStd": [ + 58.395, + 57.12, + 57.375 + ], + "labels": [ + "smoke_visible", + "flame_visible" + ], + "class_indices": [ + 0, + 1 + ], + "parity_report": "/workspace/vds-litert-batch2/learning-backbones/fireviewer_rfdetr_medium_v110/parity.json" +} \ No newline at end of file diff --git a/reports/fireviewer_rfdetr_medium_v110/source_parity.json b/reports/fireviewer_rfdetr_medium_v110/source_parity.json new file mode 100644 index 0000000000000000000000000000000000000000..640c90f6fc5491750917c0963451aa9cae86e573 --- /dev/null +++ b/reports/fireviewer_rfdetr_medium_v110/source_parity.json @@ -0,0 +1,16 @@ +{ + "key": "fireviewer_rfdetr_medium_v110", + "source_pytorch_parity_passed": true, + "input_shape": [ + 1, + 3, + 576, + 576 + ], + "max_abs_errors": [ + 6.617605686187744e-05, + 3.62396240234375e-05 + ], + "atol": 0.002, + "rtol": 0.002 +} diff --git a/reports/fireviewer_rfdetr_medium_v110/training_validation.json b/reports/fireviewer_rfdetr_medium_v110/training_validation.json new file mode 100644 index 0000000000000000000000000000000000000000..2a30460e522bd8aaef0ac88fd27fb9038b5abf77 --- /dev/null +++ b/reports/fireviewer_rfdetr_medium_v110/training_validation.json @@ -0,0 +1,86 @@ +{ + "key": "fireviewer_rfdetr_medium_v110", + "validation_passed": true, + "training_scope": "candidate_classification_and_box_adaptation_only", + "backbone_frozen": true, + "limitation": "Does not create proposals absent from frozen detector", + "initial_detector_parity": true, + "parameters_changed": true, + "initial_loss": 0.8691307902336121, + "final_loss": 0.19148443639278412, + "mixed_image_shapes": [ + [ + 224, + 224 + ], + [ + 224, + 288 + ], + [ + 288, + 224 + ] + ], + "restart_logits_exact": true, + "optimizer_resume_exact": true, + "variable_image_preprocessing": { + "mode": "bilinear_stretch_inside_graph", + "native_backbone_shape": [ + 1, + 3, + 576, + 576 + ] + }, + "android_device_tested": false, + "accuracy_evaluated": false, + "sha256": "dba3ade58743b422d492448605e1cb7b709de66323dd0fbdbd930f12af7bad20", + "bytes": 124411016, + "signatures": { + "infer": { + "inputs": [ + "x" + ], + "outputs": [ + "boxes", + "detections", + "logits" + ] + }, + "restore": { + "inputs": [ + "checkpoint_path" + ], + "outputs": [ + "step" + ] + }, + "save": { + "inputs": [ + "checkpoint_path" + ], + "outputs": [ + "step" + ] + }, + "train": { + "inputs": [ + "learning_rate", + "x", + "y" + ], + "outputs": [ + "loss", + "step" + ] + }, + "training_state": { + "inputs": [], + "outputs": [ + "head_kernel", + "step" + ] + } + } +} diff --git a/reports/fireviewer_rtdetr_v2_r50/publication_receipt.json b/reports/fireviewer_rtdetr_v2_r50/publication_receipt.json new file mode 100644 index 0000000000000000000000000000000000000000..770f6a9f014b603d50c60f4ba38674250a9e3033 --- /dev/null +++ b/reports/fireviewer_rtdetr_v2_r50/publication_receipt.json @@ -0,0 +1,8 @@ +{ + "repo": "fireviewer/litert-models", + "commit": "df60095dbaed66e8c0906b69bb0561c7ad71b3ac", + "path": "models/fireviewer_rtdetr_v2_r50_learning/model.tflite", + "sha256": "cd2c38cf21c139cf1f9fac45b4ccd6570b8cc3d8d4e5af21561e81c4e13ca565", + "bytes": 177870964, + "remote_verified": true +} diff --git a/reports/fireviewer_rtdetr_v2_r50/runtime216_validation.json b/reports/fireviewer_rtdetr_v2_r50/runtime216_validation.json new file mode 100644 index 0000000000000000000000000000000000000000..d5b2cf7dfa5c07276a3d07159eb9cb0a652ed386 --- /dev/null +++ b/reports/fireviewer_rtdetr_v2_r50/runtime216_validation.json @@ -0,0 +1,11 @@ +{ + "key": "fireviewer_rtdetr_v2_r50", + "runtime": "2.16.1", + "platform": "Linux x86_64; same LiteRT release as Android, different ABI", + "sha256": "cd2c38cf21c139cf1f9fac45b4ccd6570b8cc3d8d4e5af21561e81c4e13ca565", + "train": true, + "loss": 4.690672397613525, + "restart_exact": true, + "optimizer_resume_exact": true, + "two_object_fixture": true +} diff --git a/reports/fireviewer_rtdetr_v2_r50/source.json b/reports/fireviewer_rtdetr_v2_r50/source.json new file mode 100644 index 0000000000000000000000000000000000000000..9c99295b335fb9c5aac99d195b1e6e275363e7ae --- /dev/null +++ b/reports/fireviewer_rtdetr_v2_r50/source.json @@ -0,0 +1,25 @@ +{ + "key": "fireviewer_rtdetr_v2_r50", + "kind": "detr", + "class_count": 2, + "labels": [ + "smoke_visible", + "flame_visible" + ], + "class_indices": [ + 0, + 1 + ], + "backbone": "/workspace/vds-litert-batch2/learning-backbones/fireviewer_rtdetr_v2_r50/final/model.tflite", + "parity_report": "/workspace/vds-litert-batch2/learning-backbones/fireviewer_rtdetr_v2_r50/final/parity.json", + "native_dynamic": false, + "bindings": { + "features_logits": 0, + "features_boxes": 1 + }, + "app_size": 768, + "test_size": [ + 193, + 317 + ] +} \ No newline at end of file diff --git a/reports/fireviewer_rtdetr_v2_r50/source_parity.json b/reports/fireviewer_rtdetr_v2_r50/source_parity.json new file mode 100644 index 0000000000000000000000000000000000000000..edac32b816877223c63a715440389373029af688 --- /dev/null +++ b/reports/fireviewer_rtdetr_v2_r50/source_parity.json @@ -0,0 +1,37 @@ +{ + "key": "fireviewer_rtdetr_v2_r50", + "phase": "validated_frozen_component", + "backbone": "/workspace/vds-litert-batch2/learning-backbones/fireviewer_rtdetr_v2_r50/final/model.tflite", + "source_pytorch_parity_passed": true, + "input_shape": [ + 1, + 3, + 768, + 768 + ], + "outputs": [ + { + "shape": [ + 1, + 300, + 2 + ], + "index": 0 + }, + { + "shape": [ + 1, + 300, + 4 + ], + "index": 1 + } + ], + "max_abs_errors": [ + 0.00032138824462890625, + 0.00019538402557373047 + ], + "atol": 0.002, + "rtol": 0.002, + "fixed_shape_assertion_checked_before_export": true +} diff --git a/reports/fireviewer_rtdetr_v2_r50/training_validation.json b/reports/fireviewer_rtdetr_v2_r50/training_validation.json new file mode 100644 index 0000000000000000000000000000000000000000..cc996703b364651afeac6a27c5162108e325d0b7 --- /dev/null +++ b/reports/fireviewer_rtdetr_v2_r50/training_validation.json @@ -0,0 +1,86 @@ +{ + "key": "fireviewer_rtdetr_v2_r50", + "validation_passed": true, + "training_scope": "candidate_classification_and_box_adaptation_only", + "backbone_frozen": true, + "limitation": "Does not create proposals absent from frozen detector", + "initial_detector_parity": true, + "parameters_changed": true, + "initial_loss": 3.6428864002227783, + "final_loss": 0.4031541049480438, + "mixed_image_shapes": [ + [ + 193, + 317 + ], + [ + 193, + 381 + ], + [ + 257, + 317 + ] + ], + "restart_logits_exact": true, + "optimizer_resume_exact": true, + "variable_image_preprocessing": { + "mode": "bilinear_stretch_inside_graph", + "native_backbone_shape": [ + 1, + 3, + 768, + 768 + ] + }, + "android_device_tested": false, + "accuracy_evaluated": false, + "sha256": "cd2c38cf21c139cf1f9fac45b4ccd6570b8cc3d8d4e5af21561e81c4e13ca565", + "bytes": 177870964, + "signatures": { + "infer": { + "inputs": [ + "x" + ], + "outputs": [ + "boxes", + "detections", + "logits" + ] + }, + "restore": { + "inputs": [ + "checkpoint_path" + ], + "outputs": [ + "step" + ] + }, + "save": { + "inputs": [ + "checkpoint_path" + ], + "outputs": [ + "step" + ] + }, + "train": { + "inputs": [ + "learning_rate", + "x", + "y" + ], + "outputs": [ + "loss", + "step" + ] + }, + "training_state": { + "inputs": [], + "outputs": [ + "head_kernel", + "step" + ] + } + } +} diff --git a/reports/fireviewer_yolo11m_strict_v1/publication_receipt.json b/reports/fireviewer_yolo11m_strict_v1/publication_receipt.json new file mode 100644 index 0000000000000000000000000000000000000000..659b754144837073ea0a2ee61e3b89bd2db50b43 --- /dev/null +++ b/reports/fireviewer_yolo11m_strict_v1/publication_receipt.json @@ -0,0 +1,8 @@ +{ + "repo": "fireviewer/litert-models", + "commit": "ae756eaffe4c953e1ad6a6252830f661c9e11e1e", + "path": "models/fireviewer_yolo11m_strict_v1_learning/model.tflite", + "sha256": "f05130795e14427402d1f6f0c28b777d43bd0909be9888b776a3c958e74e222b", + "bytes": 80800592, + "remote_verified": true +} diff --git a/reports/fireviewer_yolo11m_strict_v1/runtime216_validation.json b/reports/fireviewer_yolo11m_strict_v1/runtime216_validation.json new file mode 100644 index 0000000000000000000000000000000000000000..010337b76b7bd9bd3d47e9fdbfe2547c02ac4c3e --- /dev/null +++ b/reports/fireviewer_yolo11m_strict_v1/runtime216_validation.json @@ -0,0 +1,11 @@ +{ + "key": "fireviewer_yolo11m_strict_v1", + "runtime": "2.16.1", + "platform": "Linux x86_64; same LiteRT release as Android, different ABI", + "sha256": "f05130795e14427402d1f6f0c28b777d43bd0909be9888b776a3c958e74e222b", + "train": true, + "loss": 6.923343181610107, + "restart_exact": true, + "optimizer_resume_exact": true, + "two_object_fixture": true +} diff --git a/reports/fireviewer_yolo11m_strict_v1/source.json b/reports/fireviewer_yolo11m_strict_v1/source.json new file mode 100644 index 0000000000000000000000000000000000000000..479674335128a6c6c7a4af300cda9b8abafa0eef --- /dev/null +++ b/reports/fireviewer_yolo11m_strict_v1/source.json @@ -0,0 +1,21 @@ +{ + "key": "fireviewer_yolo11m_strict_v1", + "kind": "yolo", + "class_count": 2, + "backbone": "/workspace/vds-litert-batch2/dynamic-work/fireviewer_yolo11m_strict_v1/onnx2tf/model_float32.tflite", + "native_dynamic": true, + "bindings": { + "features": 0 + }, + "labels": [ + "smoke_visible", + "flame_visible" + ], + "app_size": 320, + "class_indices": [ + 0, + 1 + ], + "dynamic_min_size": 64, + "parity_report": "/workspace/vds-litert-batch2/dynamic-work/fireviewer_yolo11m_strict_v1/validation.json" +} diff --git a/reports/fireviewer_yolo11m_strict_v1/source_parity.json b/reports/fireviewer_yolo11m_strict_v1/source_parity.json new file mode 100644 index 0000000000000000000000000000000000000000..d25f4b2fa7c51e2cb78fdf05c5f5ce20a4a0f4d5 --- /dev/null +++ b/reports/fireviewer_yolo11m_strict_v1/source_parity.json @@ -0,0 +1,78 @@ +{ + "key": "fireviewer_yolo11m_strict_v1", + "phase": "validated", + "route": "onnx2tf_dynamic", + "accuracy_evaluated": false, + "android_tested": false, + "shape_signature": [ + -1, + 3, + -1, + -1 + ], + "checks": [ + { + "input_shape": [ + 1, + 3, + 384, + 640 + ], + "output_shapes": [ + [ + 1, + 6, + 5040 + ] + ], + "max_abs_errors": [ + 0.001220703125 + ] + }, + { + "input_shape": [ + 1, + 3, + 640, + 384 + ], + "output_shapes": [ + [ + 1, + 6, + 5040 + ] + ], + "max_abs_errors": [ + 0.00177001953125 + ] + }, + { + "input_shape": [ + 1, + 3, + 960, + 960 + ], + "output_shapes": [ + [ + 1, + 6, + 18900 + ] + ], + "max_abs_errors": [ + 0.00341796875 + ] + } + ], + "dynamic_validated": true, + "atol": 0.001, + "rtol": 0.001, + "spatial_bounds": { + "alignment": 32, + "tested_max": 960 + }, + "input_layout": "NCHW", + "preprocessing": "RGB float32 in [0,1]; pad to multiples of 32; no resampling embedded" +} diff --git a/reports/fireviewer_yolo11m_strict_v1/training_validation.json b/reports/fireviewer_yolo11m_strict_v1/training_validation.json new file mode 100644 index 0000000000000000000000000000000000000000..a79ef4981e4a7dded0a6a2689680404628c5820c --- /dev/null +++ b/reports/fireviewer_yolo11m_strict_v1/training_validation.json @@ -0,0 +1,103 @@ +{ + "key": "fireviewer_yolo11m_strict_v1", + "validation_passed": true, + "training_scope": "candidate_classification_and_box_adaptation_only", + "backbone_frozen": true, + "limitation": "Does not create proposals absent from frozen detector", + "initial_detector_parity": true, + "parameters_changed": true, + "initial_loss": 5.981967926025391, + "final_loss": 0.14853930473327637, + "mixed_image_shapes": [ + [ + 320, + 320 + ], + [ + 320, + 384 + ], + [ + 384, + 320 + ] + ], + "restart_logits_exact": true, + "optimizer_resume_exact": true, + "variable_image_preprocessing": { + "mode": "native_dynamic", + "stride": 32 + }, + "android_device_tested": true, + "accuracy_evaluated": false, + "sha256": "f05130795e14427402d1f6f0c28b777d43bd0909be9888b776a3c958e74e222b", + "bytes": 80800592, + "signatures": { + "infer": { + "inputs": [ + "x" + ], + "outputs": [ + "detections" + ] + }, + "restore": { + "inputs": [ + "checkpoint_path" + ], + "outputs": [ + "step" + ] + }, + "save": { + "inputs": [ + "checkpoint_path" + ], + "outputs": [ + "step" + ] + }, + "train": { + "inputs": [ + "learning_rate", + "x", + "y" + ], + "outputs": [ + "loss", + "step" + ] + }, + "training_state": { + "inputs": [], + "outputs": [ + "head_kernel", + "step" + ] + } + }, + "android_validation": { + "runtime": "2.16.1", + "threads": 1, + "mixed_shapes": [ + [ + 64, + 64 + ], + [ + 64, + 96 + ] + ], + "train": true, + "restore_exact": true, + "optimizer_resume_exact": true, + "incompatible_schema_rejected": true, + "first_loss": 7.3072906, + "second_loss": 7.0748734, + "platform": "Android 9 API 28 emulator x86_64, CPU, XNNPACK disabled", + "model_sha256": "f05130795e14427402d1f6f0c28b777d43bd0909be9888b776a3c958e74e222b", + "log_sha256": "6f35591bdf03e4a7bccbcaf5713e0348f199128ea511078a0b391994d097fb51", + "physical_arm_phone_tested": false + } +}