Anuvani-LID / mobile_inference /android /LanguageIdentifier.kt

CIIL Anuvani-LID

bd48fd1 6 days ago

8.78 kB

// Copyright (c) 2026, CIIL Anuvani-LID. Apache-2.0.
//
// A starting point for an Android app: one class over the Kotlin API of the patched sherpa-onnx (LidPipeline for
// whole recordings, AudioTagging for one stretch of speech). Copy it into your app and change the package. It is
// compiled against android.jar and run in ART on an emulator by tests/android (docs/03-android.md).
package com.example.anuvanilid

import android.os.Build
import android.system.Os
import com.k2fsa.sherpa.onnx.AudioTagging
import com.k2fsa.sherpa.onnx.AudioTaggingConfig
import com.k2fsa.sherpa.onnx.AudioTaggingModelConfig
import com.k2fsa.sherpa.onnx.LidPipeline
import com.k2fsa.sherpa.onnx.LidPipelineConfig
import org.json.JSONObject
import java.io.File

/** The model file and execution provider to run (docs/01-formats-and-devices.md). */
enum class Accelerator {
    /** anuvani_lid.int8.onnx on the CPU: every phone; the default. */
    CPU_INT8,

    /** anuvani_lid.fp32.onnx on the CPU: the reference numbers, 2.7 times the file size of int8. */
    CPU_FP32,

    /** The 15 s accelerator format on the Qualcomm Hexagon NPU (Snapdragon with Hexagon v69 or newer; needs the
     *  QNN plugin in the app: docs/03-android.md). */
    QNN,

    /** The 15 s NNAPI encoder through NNAPI (Android 8.1 to 14; deprecated from Android 15). */
    NNAPI,
}

data class LanguageScore(val language: String, val probability: Double)

data class Segment(val startS: Double, val endS: Double, val language: String, val probability: Double)

/** The result for a recording: the anuvani-lid/1 JSON of the CIIL Anuvani-LID server, parsed. */
data class LidResult(
    /** The recording's language (null when there is no speech). */
    val language: String?,
    /** exp(duration-weighted mean log-probability) of that language. */
    val confidence: Double,
    val topK: List<LanguageScore>,
    /** The language with the most speech time. */
    val dominantByTime: String?,
    val timeShare: Map<String, Double>,
    /** One entry per classified piece of speech, in order. */
    val segments: List<Segment>,
    val speechSeconds: Double,
    val json: String,
)

class LanguageIdentifier(
    /** Directory holding the model files (stored as files, not compressed assets: docs/03-android.md). */
    private val modelDir: File,
    /** A writable directory of the app (e.g. context.filesDir): provider config files, the NPU-compiled graph. */
    private val workDir: File,
    val accelerator: Accelerator = Accelerator.CPU_INT8,
    private val threads: Int = 4,
    /** For QNN: the app's nativeLibraryDir (context.applicationInfo.nativeLibraryDir), where the plugin's
     *  libraries are; the NPU loads its libraries from there. */
    private val nativeLibraryDir: String? = null,
) : AutoCloseable {

    private val modelConfig: AudioTaggingModelConfig by lazy {
        val (file, provider) = when (accelerator) {
            Accelerator.CPU_INT8 -> "anuvani_lid.int8.onnx" to "cpu"
            Accelerator.CPU_FP32 -> "anuvani_lid.fp32.onnx" to "cpu"
            Accelerator.NNAPI -> "anuvani_lid.npu-15s-nnapi.encoder.onnx" to providerConfig("nnapi", "NNAPI.fp16=1\n")
            Accelerator.QNN -> "anuvani_lid.npu-15s.encoder.onnx" to providerConfig("qnn", qnnConfig())
        }
        AudioTaggingModelConfig(lid = File(modelDir, file).path, numThreads = threads, provider = provider)
    }

    // created on first use: an app that only identifies recordings, or only classifies segments, loads one model
    private val pipelineDelegate = lazy {
        LidPipeline(
            config = LidPipelineConfig(
                model = modelConfig,
                vadModel = File(modelDir, "silero_vad_6.2.1.onnx").path,
            ),
        )
    }
    private val taggerDelegate = lazy { AudioTagging(config = AudioTaggingConfig(model = modelConfig, topK = 23)) }

    /** The language(s) of a recording of any length: Silero VAD, speech pieces, one decision per piece, the
     *  duration-weighted decision; the procedure and results of the CIIL Anuvani-LID server. samples: mono in
     *  [-1, 1] at sampleRate (resampled to 16 kHz if different). Blocking; call it off the main thread. */
    fun identify(samples: FloatArray, sampleRate: Int): LidResult =
        parse(pipelineDelegate.value.process(samples, sampleRate))

    /** Probabilities of the languages for one stretch of speech (e.g. a VAD segment), largest first. Audio shorter
     *  than 0.1 s gives an empty list. Blocking. */
    fun classify(samples: FloatArray, sampleRate: Int, topK: Int = 23): List<LanguageScore> {
        val tagger = taggerDelegate.value
        val stream = tagger.createStream()
        stream.acceptWaveform(samples, sampleRate)
        val events = tagger.compute(stream, topK)
        stream.release()
        return events.map { LanguageScore(it.name, it.prob.toDouble()) }
    }

    override fun close() {
        if (pipelineDelegate.isInitialized()) pipelineDelegate.value.release()
        if (taggerDelegate.isInitialized()) taggerDelegate.value.release()
    }

    private fun providerConfig(name: String, text: String): String {
        val f = File(workDir, "anuvani-lid-$name.cfg")
        f.writeText(text)
        return "$name:${f.path}"
    }

    private fun qnnConfig(): String {
        nativeLibraryDir?.let {
            // the NPU side of QNN (FastRPC) finds the libQnnHtpV*Skel.so libraries through ADSP_LIBRARY_PATH
            Os.setenv("ADSP_LIBRARY_PATH", "$it;/system/lib/rfsa/adsp;/system/vendor/lib/rfsa/adsp;/dsp", true)
        }
        val cache = File(workDir, "qnn").apply { mkdirs() }
        val sb = StringBuilder()
        sb.append("QNN.library=libonnxruntime_providers_qnn.so\n")
        sb.append("QNN.backend_type=htp\n")
        sb.append("QNN.htp_performance_mode=burst\n")
        sb.append("QNN.cache_dir=${cache.path}\n")
        qnnSocModel()?.let {
            sb.append("QNN.soc_model=$it\n")
            sb.append("QNN.enable_htp_fp16_precision=1\n")
        }
        return sb.toString()
    }

    companion object {
        /** Qualcomm chips the QNN plugin knows, with their soc_model IDs and Hexagon generations (onnxruntime-qnn,
         *  onnxruntime/core/providers/qnn/soc_utility/soc_table.h). */
        val QNN_CHIPS: Map<String, Pair<Int, Int>> = mapOf(
            "SM8350" to (30 to 68), "SM8325" to (34 to 68), "SM8450" to (36 to 69), "SM8475" to (42 to 69),
            "SM8550" to (43 to 73), "SM8650" to (57 to 75), "SM8635" to (68 to 73), "SM8750" to (69 to 79),
            "SM7675" to (70 to 73), "SM8850" to (87 to 81),
        )

        /** The chip name Android reports (Android 12+), e.g. "SM8650"; "" before Android 12. */
        fun socModel(): String = if (Build.VERSION.SDK_INT >= 31) Build.SOC_MODEL ?: "" else ""

        /** soc_model of this phone's chip for the QNN plugin, or null if the table does not list it. */
        fun qnnSocModel(): Int? = QNN_CHIPS[socModel().uppercase()]?.first

        /** True on Snapdragon chips whose NPU builds the accelerator format in fp16: Hexagon v69 (Snapdragon 8 Gen 1)
         *  or newer. On Qualcomm's emulated NPU, v68 (Snapdragon 888) cannot compile it; if QNN is chosen there
         *  anyway, the model runs on the CPU (docs/01). */
        fun qnnFloatCapable(): Boolean = (QNN_CHIPS[socModel().uppercase()]?.second ?: 0) >= 69

        /** Parse the anuvani-lid/1 JSON of LidPipeline.process(). */
        fun parse(json: String): LidResult {
            val j = JSONObject(json)
            val overall = j.getJSONObject("overall")
            val topK = overall.getJSONArray("top_k").let { a ->
                (0 until a.length()).map { LanguageScore(a.getJSONObject(it).getString("label"), a.getJSONObject(it).getDouble("prob")) }
            }
            val share = j.getJSONObject("language_time_share").let { o -> o.keys().asSequence().associateWith { o.getDouble(it) } }
            val segments = j.getJSONArray("segments").let { a ->
                (0 until a.length()).map {
                    val s = a.getJSONObject(it)
                    Segment(s.getDouble("start_s"), s.getDouble("end_s"), s.getString("label"), s.getDouble("prob"))
                }
            }
            return LidResult(
                language = if (overall.isNull("label")) null else overall.getString("label"),
                confidence = overall.getDouble("confidence"),
                topK = topK,
                dominantByTime = if (j.isNull("dominant_by_time")) null else j.getString("dominant_by_time"),
                timeShare = share,
                segments = segments,
                speechSeconds = j.getJSONObject("stats").getDouble("speech_seconds"),
                json = json,
            )
        }
    }
}

	// Copyright (c) 2026, CIIL Anuvani-LID. Apache-2.0.
	//
	// A starting point for an Android app: one class over the Kotlin API of the patched sherpa-onnx (LidPipeline for
	// whole recordings, AudioTagging for one stretch of speech). Copy it into your app and change the package. It is
	// compiled against android.jar and run in ART on an emulator by tests/android (docs/03-android.md).
	package com.example.anuvanilid

	import android.os.Build
	import android.system.Os
	import com.k2fsa.sherpa.onnx.AudioTagging
	import com.k2fsa.sherpa.onnx.AudioTaggingConfig
	import com.k2fsa.sherpa.onnx.AudioTaggingModelConfig
	import com.k2fsa.sherpa.onnx.LidPipeline
	import com.k2fsa.sherpa.onnx.LidPipelineConfig
	import org.json.JSONObject
	import java.io.File

	/** The model file and execution provider to run (docs/01-formats-and-devices.md). */
	enum class Accelerator {
	/** anuvani_lid.int8.onnx on the CPU: every phone; the default. */
	CPU_INT8,

	/** anuvani_lid.fp32.onnx on the CPU: the reference numbers, 2.7 times the file size of int8. */
	CPU_FP32,

	/** The 15 s accelerator format on the Qualcomm Hexagon NPU (Snapdragon with Hexagon v69 or newer; needs the
	* QNN plugin in the app: docs/03-android.md). */
	QNN,

	/** The 15 s NNAPI encoder through NNAPI (Android 8.1 to 14; deprecated from Android 15). */
	NNAPI,
	}

	data class LanguageScore(val language: String, val probability: Double)

	data class Segment(val startS: Double, val endS: Double, val language: String, val probability: Double)

	/** The result for a recording: the anuvani-lid/1 JSON of the CIIL Anuvani-LID server, parsed. */
	data class LidResult(
	/** The recording's language (null when there is no speech). */
	val language: String?,
	/** exp(duration-weighted mean log-probability) of that language. */
	val confidence: Double,
	val topK: List<LanguageScore>,
	/** The language with the most speech time. */
	val dominantByTime: String?,
	val timeShare: Map<String, Double>,
	/** One entry per classified piece of speech, in order. */
	val segments: List<Segment>,
	val speechSeconds: Double,
	val json: String,
	)

	class LanguageIdentifier(
	/** Directory holding the model files (stored as files, not compressed assets: docs/03-android.md). */
	private val modelDir: File,
	/** A writable directory of the app (e.g. context.filesDir): provider config files, the NPU-compiled graph. */
	private val workDir: File,
	val accelerator: Accelerator = Accelerator.CPU_INT8,
	private val threads: Int = 4,
	/** For QNN: the app's nativeLibraryDir (context.applicationInfo.nativeLibraryDir), where the plugin's
	* libraries are; the NPU loads its libraries from there. */
	private val nativeLibraryDir: String? = null,
	) : AutoCloseable {

	private val modelConfig: AudioTaggingModelConfig by lazy {
	val (file, provider) = when (accelerator) {
	Accelerator.CPU_INT8 -> "anuvani_lid.int8.onnx" to "cpu"
	Accelerator.CPU_FP32 -> "anuvani_lid.fp32.onnx" to "cpu"
	Accelerator.NNAPI -> "anuvani_lid.npu-15s-nnapi.encoder.onnx" to providerConfig("nnapi", "NNAPI.fp16=1\n")
	Accelerator.QNN -> "anuvani_lid.npu-15s.encoder.onnx" to providerConfig("qnn", qnnConfig())
	}
	AudioTaggingModelConfig(lid = File(modelDir, file).path, numThreads = threads, provider = provider)
	}

	// created on first use: an app that only identifies recordings, or only classifies segments, loads one model
	private val pipelineDelegate = lazy {
	LidPipeline(
	config = LidPipelineConfig(
	model = modelConfig,
	vadModel = File(modelDir, "silero_vad_6.2.1.onnx").path,
	),
	)
	}
	private val taggerDelegate = lazy { AudioTagging(config = AudioTaggingConfig(model = modelConfig, topK = 23)) }

	/** The language(s) of a recording of any length: Silero VAD, speech pieces, one decision per piece, the
	* duration-weighted decision; the procedure and results of the CIIL Anuvani-LID server. samples: mono in
	* [-1, 1] at sampleRate (resampled to 16 kHz if different). Blocking; call it off the main thread. */
	fun identify(samples: FloatArray, sampleRate: Int): LidResult =
	parse(pipelineDelegate.value.process(samples, sampleRate))

	/** Probabilities of the languages for one stretch of speech (e.g. a VAD segment), largest first. Audio shorter
	* than 0.1 s gives an empty list. Blocking. */
	fun classify(samples: FloatArray, sampleRate: Int, topK: Int = 23): List<LanguageScore> {
	val tagger = taggerDelegate.value
	val stream = tagger.createStream()
	stream.acceptWaveform(samples, sampleRate)
	val events = tagger.compute(stream, topK)
	stream.release()
	return events.map { LanguageScore(it.name, it.prob.toDouble()) }
	}

	override fun close() {
	if (pipelineDelegate.isInitialized()) pipelineDelegate.value.release()
	if (taggerDelegate.isInitialized()) taggerDelegate.value.release()
	}

	private fun providerConfig(name: String, text: String): String {
	val f = File(workDir, "anuvani-lid-$name.cfg")
	f.writeText(text)
	return "$name:${f.path}"
	}

	private fun qnnConfig(): String {
	nativeLibraryDir?.let {
	// the NPU side of QNN (FastRPC) finds the libQnnHtpV*Skel.so libraries through ADSP_LIBRARY_PATH
	Os.setenv("ADSP_LIBRARY_PATH", "$it;/system/lib/rfsa/adsp;/system/vendor/lib/rfsa/adsp;/dsp", true)
	}
	val cache = File(workDir, "qnn").apply { mkdirs() }
	val sb = StringBuilder()
	sb.append("QNN.library=libonnxruntime_providers_qnn.so\n")
	sb.append("QNN.backend_type=htp\n")
	sb.append("QNN.htp_performance_mode=burst\n")
	sb.append("QNN.cache_dir=${cache.path}\n")
	qnnSocModel()?.let {
	sb.append("QNN.soc_model=$it\n")
	sb.append("QNN.enable_htp_fp16_precision=1\n")
	}
	return sb.toString()
	}

	companion object {
	/** Qualcomm chips the QNN plugin knows, with their soc_model IDs and Hexagon generations (onnxruntime-qnn,
	* onnxruntime/core/providers/qnn/soc_utility/soc_table.h). */
	val QNN_CHIPS: Map<String, Pair<Int, Int>> = mapOf(
	"SM8350" to (30 to 68), "SM8325" to (34 to 68), "SM8450" to (36 to 69), "SM8475" to (42 to 69),
	"SM8550" to (43 to 73), "SM8650" to (57 to 75), "SM8635" to (68 to 73), "SM8750" to (69 to 79),
	"SM7675" to (70 to 73), "SM8850" to (87 to 81),
	)

	/** The chip name Android reports (Android 12+), e.g. "SM8650"; "" before Android 12. */
	fun socModel(): String = if (Build.VERSION.SDK_INT >= 31) Build.SOC_MODEL ?: "" else ""

	/** soc_model of this phone's chip for the QNN plugin, or null if the table does not list it. */
	fun qnnSocModel(): Int? = QNN_CHIPS[socModel().uppercase()]?.first

	/** True on Snapdragon chips whose NPU builds the accelerator format in fp16: Hexagon v69 (Snapdragon 8 Gen 1)
	* or newer. On Qualcomm's emulated NPU, v68 (Snapdragon 888) cannot compile it; if QNN is chosen there
	* anyway, the model runs on the CPU (docs/01). */
	fun qnnFloatCapable(): Boolean = (QNN_CHIPS[socModel().uppercase()]?.second ?: 0) >= 69

	/** Parse the anuvani-lid/1 JSON of LidPipeline.process(). */
	fun parse(json: String): LidResult {
	val j = JSONObject(json)
	val overall = j.getJSONObject("overall")
	val topK = overall.getJSONArray("top_k").let { a ->
	(0 until a.length()).map { LanguageScore(a.getJSONObject(it).getString("label"), a.getJSONObject(it).getDouble("prob")) }
	}
	val share = j.getJSONObject("language_time_share").let { o -> o.keys().asSequence().associateWith { o.getDouble(it) } }
	val segments = j.getJSONArray("segments").let { a ->
	(0 until a.length()).map {
	val s = a.getJSONObject(it)
	Segment(s.getDouble("start_s"), s.getDouble("end_s"), s.getString("label"), s.getDouble("prob"))
	}
	}
	return LidResult(
	language = if (overall.isNull("label")) null else overall.getString("label"),
	confidence = overall.getDouble("confidence"),
	topK = topK,
	dominantByTime = if (j.isNull("dominant_by_time")) null else j.getString("dominant_by_time"),
	timeShare = share,
	segments = segments,
	speechSeconds = j.getJSONObject("stats").getDouble("speech_seconds"),
	json = json,
	)
	}
	}
	}