ailia Speech Live Mode Guide

Overview

With AILIA_SPEECH_FLAG_LIVE, speech recognition runs with low latency, without waiting for VAD or an audio segment boundary.

In normal mode, recognition does not run until 30 seconds of audio have accumulated or VAD detects a speech boundary. In Live mode, a tentative inference runs on the current buffer content even before a boundary is detected, and the result is passed to IntermediateCallback.

How to Push PCM

When using AILIA_SPEECH_FLAG_LIVE, do not run the Buffered check and Transcribe after every PCM chunk you push. Instead, first push all PCM received so far, then run the Buffered check and Transcribe.

In Live mode, every Transcribe call runs a tentative inference over the whole current buffer. Running it per chunk makes inference fall behind the incoming audio, and latency keeps growing.

// NG: running the Buffered check and Transcribe for every chunk
while (mic_queue_pop(&chunk)) {
    ailiaSpeechPushInputData(speech, chunk.pcm, chunk.channels, chunk.samples, sample_rate);
    unsigned int buffered = 0;
    ailiaSpeechBuffered(speech, &buffered);
    if (buffered == 1) ailiaSpeechTranscribe(speech);
}

Code Samples

C

int intermediate_callback(void *handle, const char *text){
    printf("%s\n", text); // tentative result
    return 0; // return 1 to interrupt
}

ailiaSpeechCreate(&speech, env_id, AILIA_MULTITHREAD_AUTO, memory_mode,
                  AILIA_SPEECH_TASK_TRANSCRIBE, AILIA_SPEECH_FLAG_LIVE,
                  ailiaSpeechUtilGetCallback(), AILIA_SPEECH_API_CALLBACK_VERSION);
ailiaSpeechOpenModelFileA(speech, encoder_path, decoder_path,
                          AILIA_SPEECH_MODEL_TYPE_WHISPER_MULTILINGUAL_SMALL);
ailiaSpeechOpenVadFileA(speech, "silero_vad.onnx", AILIA_SPEECH_VAD_TYPE_SILERO);
ailiaSpeechSetIntermediateCallback(speech, &intermediate_callback, NULL);

while (recording) {
    // 1. Push all PCM received so far
    while (mic_queue_pop(&chunk)) {
        ailiaSpeechPushInputData(speech, chunk.pcm, chunk.channels, chunk.samples, sample_rate);
    }

    // 2. Then run the Buffered check and Transcribe
    while (true) {
        unsigned int buffered = 0;
        ailiaSpeechBuffered(speech, &buffered);
        if (buffered == 0) break;
        ailiaSpeechTranscribe(speech);

        unsigned int n;
        ailiaSpeechGetTextCount(speech, &n);
        for (unsigned int i = 0; i < n; i++) {
            AILIASpeechText text;
            ailiaSpeechGetText(speech, &text, AILIA_SPEECH_TEXT_VERSION, i);
            printf("%s\n", text.text);
        }
    }
}

Python

Each call to transcribe_step() pushes PCM, runs the Buffered check and runs Transcribe. So instead of calling transcribe_step() for every chunk received from the microphone, concatenate all PCM received so far and pass it to a single transcribe_step() call.

import numpy as np
import ailia_speech

def on_intermediate(text):
    print(text)  # tentative result

speech = ailia_speech.Whisper(flags=ailia_speech.AILIA_SPEECH_FLAG_LIVE,
                              callback=on_intermediate)
speech.initialize_model(
    model_path="./models/",
    model_type=ailia_speech.AILIA_SPEECH_MODEL_TYPE_WHISPER_MULTILINGUAL_SMALL,
)

while recording:
    # 1. Take all PCM received so far and concatenate it
    chunks = []
    while not mic_queue.empty():
        chunks.append(mic_queue.get())
    if not chunks:
        continue
    audio = np.concatenate(chunks)

    # 2. Pass it to transcribe_step once
    for result in speech.transcribe_step(audio, sample_rate, complete=False):
        print(result["text"])

JNI (Kotlin)

val speech = AiliaSpeech(
    envId = -1,
    task = AiliaSpeech.AILIA_SPEECH_TASK_TRANSCRIBE,
    flags = AiliaSpeech.AILIA_SPEECH_FLAG_LIVE,
)
speech.openModel(encoderPath, decoderPath,
    AiliaSpeech.AILIA_SPEECH_MODEL_TYPE_WHISPER_MULTILINGUAL_SMALL)
speech.setIntermediateCallback(object : IntermediateCallback {
    override fun onIntermediateResult(text: String): Int {
        Log.d("ailia", text) // tentative result
        return 0
    }
})

while (recording) {
    // 1. Push all PCM received so far
    while (true) {
        val chunk = micQueue.poll() ?: break
        speech.pushInputData(chunk.pcm, chunk.channels, chunk.samples, sampleRate)
    }

    // 2. Then run the Buffered check and Transcribe
    while (speech.getBuffered() == 1) {
        speech.transcribe()
        for (i in 0 until speech.getTextCount()) {
            Log.d("ailia", speech.getText(i).text)
        }
    }
}

Flutter

final speech = AiliaSpeechModel();
speech.create(true, false, envId); // liveTranscribe = true
speech.open(File(encoderPath), File(decoderPath), File(vadPath), "auto",
    AILIA_SPEECH_MODEL_TYPE_WHISPER_MULTILINGUAL_SMALL);
speech.setIntermediateCallback((String text) {
  print(text); // tentative result
});

while (recording) {
  // 1. Push all PCM received so far
  while (micQueue.isNotEmpty) {
    final chunk = micQueue.removeFirst();
    speech.pushInputData(chunk.pcm, sampleRate, chunk.channels);
  }

  // 2. Then run the Buffered check and Transcribe
  while (speech.isBuffered()) {
    for (final text in speech.transcribe()) {
      print(text.text);
    }
  }
}

Unity

AiliaSpeechModel.Transcribe() adds the given PCM to a queue, and a worker thread pushes the PCM, runs the Buffered check and runs Transcribe. So instead of calling Transcribe() for every chunk received from the microphone, concatenate the PCM received in each frame and pass it to a single Transcribe() call. The tentative result can be obtained with GetIntermediateText().

AiliaSpeechModel ailia_speech = new AiliaSpeechModel();
List<float[]> waveQueue = new List<float[]>();

void OnEnable(){
    ailia_speech.Open(encoder_path, decoder_path, env_id, memory_mode,
        AiliaSpeech.AILIA_SPEECH_MODEL_TYPE_WHISPER_MULTILINGUAL_SMALL,
        AiliaSpeech.AILIA_SPEECH_TASK_TRANSCRIBE, AiliaSpeech.AILIA_SPEECH_FLAG_LIVE, "auto");
    ailia_speech.OpenVad(vad_path, AiliaSpeech.AILIA_SPEECH_VAD_TYPE_SILERO);
}

void Update(){
    // Tentative result (e.g. show it in the UI)
    ui_text.text = ailia_speech.GetIntermediateText();

    // Confirmed results
    foreach (string text in ailia_speech.GetResults()){
        Debug.Log(text);
    }

    // 1. Concatenate all PCM received from the microphone so far
    if (waveQueue.Count == 0){
        return;
    }
    float[] waveData = waveQueue.SelectMany(x => x).ToArray();
    waveQueue.Clear();

    // 2. Pass it to Transcribe once
    ailia_speech.Transcribe(waveData, frequency, channels, false);
}

void OnDisable(){
    ailia_speech.Close();
}