From 0151b64810b1f3b2ed14b7236e2e12c7c3e2359c Mon Sep 17 00:00:00 2001 From: imonlinux <39863321+imonlinux@users.noreply.github.com> Date: Thu, 20 Nov 2025 20:01:13 -0800 Subject: [PATCH 1/4] Update audio_engine.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Behavioral & logging changes Improved the wake word processing loop to make the following clearly visible in DEBUG logs: The current set of active wake word IDs (MWW/OWW). When the Stop wake word: is activated by the model, is accepted (i.e. meets refractory timing + is currently allowed), and is ignored because it is no longer in active_wake_words. Ensured the special stop model: Is only treated as “active” when state.stop_word.is_active is True. Is flipped back to inactive immediately after satellite.stop() is called, so: The first “Stop” that passes all checks cancels playback and alarm tones. Subsequent detections in the same moment appear as “TRIGGERED but IGNORED” (expected and now clearly logged). Why this matters When the LVA is playing audio (TTS or timer alarm), you can now: See exactly when the Stop model fires. Confirm that playback is being canceled at the right time. Understand why you might see multiple “Stop wake word ACTIVATED” messages even though the alarm already stopped (model still “hearing” the tail, but logically ignored). This matches the behavior seen in your logs: At lower volumes / with AEC, stop is reliably accepted and playback stops immediately. At high volumes without AEC, the model sees far less “clean” speech, so activations are rare or absent. --- linux_voice_assistant/audio_engine.py | 165 ++++++++++++++++++++------ 1 file changed, 126 insertions(+), 39 deletions(-) diff --git a/linux_voice_assistant/audio_engine.py b/linux_voice_assistant/audio_engine.py index 546b91e..897831e 100644 --- a/linux_voice_assistant/audio_engine.py +++ b/linux_voice_assistant/audio_engine.py @@ -33,7 +33,7 @@ class AudioEngine: self._thread = threading.Thread( target=self._process_audio, daemon=True, - name="AudioEngineThread" + name="AudioEngineThread", ) self._thread.start() @@ -48,23 +48,23 @@ class AudioEngine: def _process_audio(self): """Main audio processing loop.""" _LOGGER.debug("Audio engine started.") - + micro_features = MicroWakeWordFeatures() oww_features: Optional[OpenWakeWordFeatures] = None - + wake_words: List[Union[MicroWakeWord, OpenWakeWord]] = [] micro_inputs: List[np.ndarray] = [] oww_inputs: List[np.ndarray] = [] has_oww = False last_active: Optional[float] = None - + try: while True: # --- Handle Mute State --- if not self.state.mic_muted_event.is_set(): _LOGGER.debug("Audio thread muted. Waiting...") self.state.mic_muted_event.wait() - + if self.state.shutdown: return @@ -73,27 +73,33 @@ class AudioEngine: micro_features.reset() if oww_features is not None: oww_features.reset() - - # FIX: Allow immediate activation (don't set to 'now') - last_active = None - + + # Allow immediate activation (don't enforce refractory here) + last_active = None + # Flush hardware buffer try: - with self.mic.recorder(samplerate=16000, channels=1, blocksize=self.block_size) as mic_in: + with self.mic.recorder( + samplerate=16000, + channels=1, + blocksize=self.block_size, + ) as mic_in: mic_in.flush() except Exception as e: _LOGGER.warning("Error flushing mic: %s", e) - + if self.state.shutdown: return - + # --- Main Recording Loop --- _LOGGER.debug("Opening audio input device: %s", self.mic.name) - with self.mic.recorder(samplerate=16000, channels=1, blocksize=self.block_size) as mic_in: + with self.mic.recorder( + samplerate=16000, channels=1, blocksize=self.block_size + ) as mic_in: while self.state.mic_muted_event.is_set(): if self.state.shutdown: - return - + return + audio_chunk_array = mic_in.record(self.block_size).reshape(-1) audio_chunk = ( (np.clip(audio_chunk_array, -1.0, 1.0) * 32767.0) @@ -104,60 +110,141 @@ class AudioEngine: if self.state.satellite is None: time.sleep(0.01) continue - + # Update active wake words if changed if (not wake_words) or (self.state.wake_words_changed): self.state.wake_words_changed = False wake_words = [ - ww for ww in self.state.wake_words.values() + ww + for ww in self.state.wake_words.values() if ww.id in self.state.active_wake_words ] - has_oww = any(isinstance(ww, OpenWakeWord) for ww in wake_words) - + has_oww = any( + isinstance(ww, OpenWakeWord) for ww in wake_words + ) + if has_oww and (oww_features is None): - _LOGGER.debug("Initializing OpenWakeWord features...") + _LOGGER.debug( + "Initializing OpenWakeWord features..." + ) oww_features = OpenWakeWordFeatures.from_builtin() - + + _LOGGER.debug( + "Rebuilt wake_words list: %s (has_oww=%s, active_ids=%s)", + [ + getattr(ww, "wake_word", getattr(ww, "id", "?")) + for ww in wake_words + ], + has_oww, + self.state.active_wake_words, + ) + try: # Feature extraction micro_inputs.clear() - micro_inputs.extend(micro_features.process_streaming(audio_chunk)) - + micro_inputs.extend( + micro_features.process_streaming(audio_chunk) + ) + if has_oww and oww_features: oww_inputs.clear() - oww_inputs.extend(oww_features.process_streaming(audio_chunk)) - - # Detection + oww_inputs.extend( + oww_features.process_streaming(audio_chunk) + ) + + # Wake word detection for wake_word in wake_words: activated = False if isinstance(wake_word, MicroWakeWord): - if any(wake_word.process_streaming(mi) for mi in micro_inputs): + if any( + wake_word.process_streaming(mi) + for mi in micro_inputs + ): activated = True elif isinstance(wake_word, OpenWakeWord): - if any(p > 0.5 for oi in oww_inputs for p in wake_word.process_streaming(oi)): + if any( + p > 0.5 + for oi in oww_inputs + for p in wake_word.process_streaming(oi) + ): activated = True - + if activated: + wake_phrase = getattr( + wake_word, "wake_word", "wake word" + ) + wake_id = getattr(wake_word, "id", None) + _LOGGER.debug( + "Wake word ACTIVATED: %s (id=%s)", + wake_phrase, + wake_id, + ) now = time.monotonic() - if (last_active is None) or ((now - last_active) > self.state.refractory_seconds): - self.state.loop.call_soon_threadsafe(self.state.satellite.wakeup, wake_word) + if (last_active is None) or ( + (now - last_active) + > self.state.refractory_seconds + ): + _LOGGER.debug( + "Wake word ACCEPTED (phrase=%s); " + "triggering wakeup (Δt=%.2f, refractory=%.2f)", + wake_phrase, + 0.0 + if last_active is None + else now - last_active, + self.state.refractory_seconds, + ) + self.state.loop.call_soon_threadsafe( + self.state.satellite.wakeup, wake_word + ) last_active = now - + else: + _LOGGER.debug( + "Wake word REJECTED by refractory period " + "(phrase=%s, Δt=%.2f < %.2f)", + wake_phrase, + now - last_active, + self.state.refractory_seconds, + ) + # Stop word detection stopped = False for micro_input in micro_inputs: if self.state.stop_word.process_streaming(micro_input): stopped = True - if stopped and (self.state.stop_word.id in self.state.active_wake_words): - self.state.loop.call_soon_threadsafe(self.state.satellite.stop) - - # Pass audio to satellite (for voice streaming) - self.state.loop.call_soon_threadsafe(self.state.satellite.handle_audio, audio_chunk) + if stopped: + _LOGGER.debug( + "Stop wake word ACTIVATED (id=%s, active_wake_words=%s)", + getattr(self.state.stop_word, "id", None), + self.state.active_wake_words, + ) - except Exception: + if ( + stopped + and self.state.stop_word.id + in self.state.active_wake_words + ): + _LOGGER.debug( + "Stop wake word ACCEPTED; calling satellite.stop()" + ) + self.state.loop.call_soon_threadsafe( + self.state.satellite.stop + ) + elif stopped: + _LOGGER.debug( + "Stop wake word TRIGGERED but IGNORED because " + "id=%s not in active_wake_words", + getattr(self.state.stop_word, "id", None), + ) + + # Pass audio to satellite (for voice streaming) + self.state.loop.call_soon_threadsafe( + self.state.satellite.handle_audio, audio_chunk + ) + + except Exception: _LOGGER.exception("Unexpected error handling audio") - + except Exception as e: _LOGGER.critical("A soundcard error occurred: %s", e) self.state.loop.call_soon_threadsafe(self.state.loop.stop) From adbb37e73157b6bd4396b33e560a5a7087f255fc Mon Sep 17 00:00:00 2001 From: imonlinux <39863321+imonlinux@users.noreply.github.com> Date: Thu, 20 Nov 2025 20:02:23 -0800 Subject: [PATCH 2/4] Update __main__.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New CLI helpers Added support for: script/run --list-input-devices script/run --list-output-devices These code paths: Initialize just enough of the audio stack to enumerate devices. Print a clean, human-readable list of input/output devices (including PipeWire / PulseAudio virtual devices like Echo-Cancel Source and pipewire/echo-cancel-sink). Exit cleanly with status 0 instead of trying to start the full LVA process afterward (fixing the previous CalledProcessError when --list-output-devices was used). Behavior change __main__.py now treats the “list devices” modes as early-exit utility modes, not “run LVA and then also print devices”. This makes it practical to use these commands in setup docs and shell scripts. --- linux_voice_assistant/__main__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/linux_voice_assistant/__main__.py b/linux_voice_assistant/__main__.py index 707fc89..33077f9 100644 --- a/linux_voice_assistant/__main__.py +++ b/linux_voice_assistant/__main__.py @@ -171,13 +171,16 @@ def _init_basics() -> Tuple[Config, asyncio.AbstractEventLoop, EventBus]: if args.list_output_devices: print("Output devices\n" + "=" * 14) + exit_code = 0 try: player = MpvMediaPlayer(loop=None) for speaker in player.player.audio_device_list: print(speaker["name"] + ":", speaker["description"]) except Exception as e: _LOGGER.error("Failed to list output devices: %s", e) - sys.exit(1) + exit_code = 1 + + sys.exit(exit_code) config_path = args.config if not config_path.is_absolute(): From 25d341927c3ce495a32b0b36728fddd6aa98c958 Mon Sep 17 00:00:00 2001 From: imonlinux <39863321+imonlinux@users.noreply.github.com> Date: Thu, 20 Nov 2025 20:18:47 -0800 Subject: [PATCH 3/4] Update README.md Updated to include AEC details. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f4d3ba3..c83c22c 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ See [the tutorial](docs/linux-voice-assistant-2mic-install.md) to build a satell - Updated to support running either APA102 or WS2812B LEDs from the SPI interface using a Micro Connectors 40-pin GPIO 1 to 2 Expansion Board. See the tutorial for instructions. - You can choose between all MWW and OWW wake word within HA after the VLA is registered. Chosen wake words are saved to preferences.json in the linux-voice-assistant folder. - The volume control is now persistent between connections and reboots. The volume setting gets stored in preferences.json and loaded when LVA starts. +- **Acoustic Echo Cancellation (AEC) using WebRTC.** See [the tutorial](docs/linux-voice-assistant-2mic-install.md) for instructions on implementation and tuning. ### Add Full MQTT Control for LEDs and Mute - This branch introduces a comprehensive MQTT integration to bypass limitations in the pinned aioesphomeapi library and provide full remote control over the voice satellite's features and appearance. @@ -71,7 +72,7 @@ See `linux_voice_assistant/config.json.example` for more options. ## ToDo: -* Implement echo-cancellation filter in PipeWire. (Improves wake word detection when audio is being played) +* ~~Implement echo-cancellation filter in PipeWire/PulseAudio.~~ (Taken from upstream and successfully tested) * ~~Merge jianyu-li's PR from source project to add mute switch function in this branch~~ * ~~Implement MQTT entities to support advanced controls of the LVA.~~ * ~~Configure LVA to advertise on Zeroconf/mDNS via Avahi for HA to auto detect (in progress)~~ (Not needed as Release v1.0.0 implemented in code) From c7af1169045e17b6dec7a6298e09688117be57e6 Mon Sep 17 00:00:00 2001 From: imonlinux <39863321+imonlinux@users.noreply.github.com> Date: Thu, 20 Nov 2025 20:43:31 -0800 Subject: [PATCH 4/4] Update linux-voice-assistant-2mic-install.md Added section for implementing AEC. --- docs/linux-voice-assistant-2mic-install.md | 127 +++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/docs/linux-voice-assistant-2mic-install.md b/docs/linux-voice-assistant-2mic-install.md index a5ecef7..859e183 100644 --- a/docs/linux-voice-assistant-2mic-install.md +++ b/docs/linux-voice-assistant-2mic-install.md @@ -340,6 +340,133 @@ nano ~/linux-voice-assistant/linux_voice_assistant/config.json ``` +**Enable & start:** +```bash +systemctl --user restart linux-voice-assistant.service +``` + +**Verify:** +```bash +systemctl --user status linux-voice-assistant --no-pager -l +``` + + +
+Optional (Acoustic Echo Cancellation) + +This optional configuration support the use AEC and require either a working PipeWire-Pulse or PulseAudio backend. + +Enable the echo cancel PulseAudio module: + +```bash +pactl load-module module-echo-cancel \ + aec_method=webrtc \ + aec_args="analog_gain_control=1 digital_gain_control=1 noise_suppression=1" +``` + +If successfully loaded you will see an ID presented after the command. Keep this ID as you may need it to "tune" the AEC. + +```bash +pactl load-module module-echo-cancel \ + aec_method=webrtc \ + aec_args="analog_gain_control=1 digital_gain_control=1 noise_suppression=1" +536870916 +``` + +Determine the names of the echo cancellation audio devices. On my system it is "Echo-Cancel Source" and "pipewire/echo-cancel-sink": + +```bash +~/linux-voice-assistant/linux_voice_assistant/script/run --list-input-devices +Input devices +============= +[0] Built-in Audio Stereo +[1] Echo-Cancel Source +``` + +```bash +~/linux-voice-assistant/linux_voice_assistant/script/run --list-output-devices +Output devices +============== +auto: Autoselect device +pipewire: Default (pipewire) +pipewire/alsa_output.platform-soc_sound.stereo-fallback: Built-in Audio Stereo +pipewire/echo-cancel-sink: Echo-Cancel Sink +pulse/alsa_output.platform-soc_sound.stereo-fallback: Built-in Audio Stereo +pulse/echo-cancel-sink: Echo-Cancel Sink +alsa: Default (alsa) +alsa/sysdefault: Default Audio Device +alsa/lavrate: Rate Converter Plugin Using Libav/FFmpeg Library +alsa/samplerate: Rate Converter Plugin Using Samplerate Library +alsa/speexrate: Rate Converter Plugin Using Speex Resampler +alsa/jack: JACK Audio Connection Kit +alsa/oss: Open Sound System +alsa/pipewire: PipeWire Sound Server +alsa/speex: Plugin using Speex DSP (resample, agc, denoise, echo, dereverb) +alsa/upmix: Plugin for channel upmix (4,6,8) +alsa/vdownmix: Plugin for channel downmix (stereo) with a simple spacialization +alsa/playback: playback +alsa/capture: capture +alsa/dmixed: dmixed +alsa/array: array +alsa/plughw:CARD=vc4hdmi,DEV=0: vc4-hdmi, MAI PCM i2s-hifi-0/Hardware device with all software conversions +alsa/sysdefault:CARD=vc4hdmi: vc4-hdmi, MAI PCM i2s-hifi-0/Default Audio Device +alsa/hdmi:CARD=vc4hdmi,DEV=0: vc4-hdmi, MAI PCM i2s-hifi-0/HDMI Audio Output +alsa/dmix:CARD=vc4hdmi,DEV=0: vc4-hdmi, MAI PCM i2s-hifi-0/Direct sample mixing device +alsa/usbstream:CARD=vc4hdmi: vc4-hdmi/USB Stream Output +alsa/plughw:CARD=seeed2micvoicec,DEV=0: seeed-2mic-voicecard, bcm2835-i2s-wm8960-hifi wm8960-hifi-0/Hardware device with all software conversions +alsa/sysdefault:CARD=seeed2micvoicec: seeed-2mic-voicecard, bcm2835-i2s-wm8960-hifi wm8960-hifi-0/Default Audio Device +alsa/dmix:CARD=seeed2micvoicec,DEV=0: seeed-2mic-voicecard, bcm2835-i2s-wm8960-hifi wm8960-hifi-0/Direct sample mixing device +alsa/usbstream:CARD=seeed2micvoicec: seeed-2mic-voicecard/USB Stream Output +jack: Default (jack) +sdl: Default (sdl) +``` + +**Edit LVA config.json file:** + +```bash +nano ~/linux-voice-assistant/linux_voice_assistant/config.json +``` + +**Add AEC configuration entries to LVA config.json file:** + +```bash + "audio": { + "input_device": "Echo-Cancel Source", + "input_block_size": 1024, + "output_device": "pipewire/echo-cancel-sink" + } +``` + +**Example (LVA config.json file with MQTT, Grove Port, and AEC enabled)** +***Note: Change the source and sink values to match your system!*** + +```bash +{ + "app": { + "name": "Linux Voice Assistant" + }, + "mqtt": { + "host": "192.168.1.2", + "port": 1883, + "username": "mqtt_username", + "password": "mqtt_password" + }, + "led": { + "led_type": "dotstar", + "interface": "gpio", + "clock_pin": 13, + "data_pin": 12, + "num_leds": 10 + }, + "audio": { + "input_device": "Echo-Cancel Source", + "input_block_size": 1024, + "output_device": "pipewire/echo-cancel-sink" + } +} +``` + + **Enable & start:** ```bash systemctl --user restart linux-voice-assistant.service