diff --git a/README.md b/README.md index f4d3ba3..c83c22c 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ See [the tutorial](docs/linux-voice-assistant-2mic-install.md) to build a satell - Updated to support running either APA102 or WS2812B LEDs from the SPI interface using a Micro Connectors 40-pin GPIO 1 to 2 Expansion Board. See the tutorial for instructions. - You can choose between all MWW and OWW wake word within HA after the VLA is registered. Chosen wake words are saved to preferences.json in the linux-voice-assistant folder. - The volume control is now persistent between connections and reboots. The volume setting gets stored in preferences.json and loaded when LVA starts. +- **Acoustic Echo Cancellation (AEC) using WebRTC.** See [the tutorial](docs/linux-voice-assistant-2mic-install.md) for instructions on implementation and tuning. ### Add Full MQTT Control for LEDs and Mute - This branch introduces a comprehensive MQTT integration to bypass limitations in the pinned aioesphomeapi library and provide full remote control over the voice satellite's features and appearance. @@ -71,7 +72,7 @@ See `linux_voice_assistant/config.json.example` for more options. ## ToDo: -* Implement echo-cancellation filter in PipeWire. (Improves wake word detection when audio is being played) +* ~~Implement echo-cancellation filter in PipeWire/PulseAudio.~~ (Taken from upstream and successfully tested) * ~~Merge jianyu-li's PR from source project to add mute switch function in this branch~~ * ~~Implement MQTT entities to support advanced controls of the LVA.~~ * ~~Configure LVA to advertise on Zeroconf/mDNS via Avahi for HA to auto detect (in progress)~~ (Not needed as Release v1.0.0 implemented in code) diff --git a/docs/linux-voice-assistant-2mic-install.md b/docs/linux-voice-assistant-2mic-install.md index a5ecef7..859e183 100644 --- a/docs/linux-voice-assistant-2mic-install.md +++ b/docs/linux-voice-assistant-2mic-install.md @@ -340,6 +340,133 @@ nano ~/linux-voice-assistant/linux_voice_assistant/config.json ``` +**Enable & start:** +```bash +systemctl --user restart linux-voice-assistant.service +``` + +**Verify:** +```bash +systemctl --user status linux-voice-assistant --no-pager -l +``` + + +
+Optional (Acoustic Echo Cancellation) + +This optional configuration support the use AEC and require either a working PipeWire-Pulse or PulseAudio backend. + +Enable the echo cancel PulseAudio module: + +```bash +pactl load-module module-echo-cancel \ + aec_method=webrtc \ + aec_args="analog_gain_control=1 digital_gain_control=1 noise_suppression=1" +``` + +If successfully loaded you will see an ID presented after the command. Keep this ID as you may need it to "tune" the AEC. + +```bash +pactl load-module module-echo-cancel \ + aec_method=webrtc \ + aec_args="analog_gain_control=1 digital_gain_control=1 noise_suppression=1" +536870916 +``` + +Determine the names of the echo cancellation audio devices. On my system it is "Echo-Cancel Source" and "pipewire/echo-cancel-sink": + +```bash +~/linux-voice-assistant/linux_voice_assistant/script/run --list-input-devices +Input devices +============= +[0] Built-in Audio Stereo +[1] Echo-Cancel Source +``` + +```bash +~/linux-voice-assistant/linux_voice_assistant/script/run --list-output-devices +Output devices +============== +auto: Autoselect device +pipewire: Default (pipewire) +pipewire/alsa_output.platform-soc_sound.stereo-fallback: Built-in Audio Stereo +pipewire/echo-cancel-sink: Echo-Cancel Sink +pulse/alsa_output.platform-soc_sound.stereo-fallback: Built-in Audio Stereo +pulse/echo-cancel-sink: Echo-Cancel Sink +alsa: Default (alsa) +alsa/sysdefault: Default Audio Device +alsa/lavrate: Rate Converter Plugin Using Libav/FFmpeg Library +alsa/samplerate: Rate Converter Plugin Using Samplerate Library +alsa/speexrate: Rate Converter Plugin Using Speex Resampler +alsa/jack: JACK Audio Connection Kit +alsa/oss: Open Sound System +alsa/pipewire: PipeWire Sound Server +alsa/speex: Plugin using Speex DSP (resample, agc, denoise, echo, dereverb) +alsa/upmix: Plugin for channel upmix (4,6,8) +alsa/vdownmix: Plugin for channel downmix (stereo) with a simple spacialization +alsa/playback: playback +alsa/capture: capture +alsa/dmixed: dmixed +alsa/array: array +alsa/plughw:CARD=vc4hdmi,DEV=0: vc4-hdmi, MAI PCM i2s-hifi-0/Hardware device with all software conversions +alsa/sysdefault:CARD=vc4hdmi: vc4-hdmi, MAI PCM i2s-hifi-0/Default Audio Device +alsa/hdmi:CARD=vc4hdmi,DEV=0: vc4-hdmi, MAI PCM i2s-hifi-0/HDMI Audio Output +alsa/dmix:CARD=vc4hdmi,DEV=0: vc4-hdmi, MAI PCM i2s-hifi-0/Direct sample mixing device +alsa/usbstream:CARD=vc4hdmi: vc4-hdmi/USB Stream Output +alsa/plughw:CARD=seeed2micvoicec,DEV=0: seeed-2mic-voicecard, bcm2835-i2s-wm8960-hifi wm8960-hifi-0/Hardware device with all software conversions +alsa/sysdefault:CARD=seeed2micvoicec: seeed-2mic-voicecard, bcm2835-i2s-wm8960-hifi wm8960-hifi-0/Default Audio Device +alsa/dmix:CARD=seeed2micvoicec,DEV=0: seeed-2mic-voicecard, bcm2835-i2s-wm8960-hifi wm8960-hifi-0/Direct sample mixing device +alsa/usbstream:CARD=seeed2micvoicec: seeed-2mic-voicecard/USB Stream Output +jack: Default (jack) +sdl: Default (sdl) +``` + +**Edit LVA config.json file:** + +```bash +nano ~/linux-voice-assistant/linux_voice_assistant/config.json +``` + +**Add AEC configuration entries to LVA config.json file:** + +```bash + "audio": { + "input_device": "Echo-Cancel Source", + "input_block_size": 1024, + "output_device": "pipewire/echo-cancel-sink" + } +``` + +**Example (LVA config.json file with MQTT, Grove Port, and AEC enabled)** +***Note: Change the source and sink values to match your system!*** + +```bash +{ + "app": { + "name": "Linux Voice Assistant" + }, + "mqtt": { + "host": "192.168.1.2", + "port": 1883, + "username": "mqtt_username", + "password": "mqtt_password" + }, + "led": { + "led_type": "dotstar", + "interface": "gpio", + "clock_pin": 13, + "data_pin": 12, + "num_leds": 10 + }, + "audio": { + "input_device": "Echo-Cancel Source", + "input_block_size": 1024, + "output_device": "pipewire/echo-cancel-sink" + } +} +``` + + **Enable & start:** ```bash systemctl --user restart linux-voice-assistant.service diff --git a/linux_voice_assistant/__main__.py b/linux_voice_assistant/__main__.py index 707fc89..33077f9 100644 --- a/linux_voice_assistant/__main__.py +++ b/linux_voice_assistant/__main__.py @@ -171,13 +171,16 @@ def _init_basics() -> Tuple[Config, asyncio.AbstractEventLoop, EventBus]: if args.list_output_devices: print("Output devices\n" + "=" * 14) + exit_code = 0 try: player = MpvMediaPlayer(loop=None) for speaker in player.player.audio_device_list: print(speaker["name"] + ":", speaker["description"]) except Exception as e: _LOGGER.error("Failed to list output devices: %s", e) - sys.exit(1) + exit_code = 1 + + sys.exit(exit_code) config_path = args.config if not config_path.is_absolute(): diff --git a/linux_voice_assistant/audio_engine.py b/linux_voice_assistant/audio_engine.py index 546b91e..897831e 100644 --- a/linux_voice_assistant/audio_engine.py +++ b/linux_voice_assistant/audio_engine.py @@ -33,7 +33,7 @@ class AudioEngine: self._thread = threading.Thread( target=self._process_audio, daemon=True, - name="AudioEngineThread" + name="AudioEngineThread", ) self._thread.start() @@ -48,23 +48,23 @@ class AudioEngine: def _process_audio(self): """Main audio processing loop.""" _LOGGER.debug("Audio engine started.") - + micro_features = MicroWakeWordFeatures() oww_features: Optional[OpenWakeWordFeatures] = None - + wake_words: List[Union[MicroWakeWord, OpenWakeWord]] = [] micro_inputs: List[np.ndarray] = [] oww_inputs: List[np.ndarray] = [] has_oww = False last_active: Optional[float] = None - + try: while True: # --- Handle Mute State --- if not self.state.mic_muted_event.is_set(): _LOGGER.debug("Audio thread muted. Waiting...") self.state.mic_muted_event.wait() - + if self.state.shutdown: return @@ -73,27 +73,33 @@ class AudioEngine: micro_features.reset() if oww_features is not None: oww_features.reset() - - # FIX: Allow immediate activation (don't set to 'now') - last_active = None - + + # Allow immediate activation (don't enforce refractory here) + last_active = None + # Flush hardware buffer try: - with self.mic.recorder(samplerate=16000, channels=1, blocksize=self.block_size) as mic_in: + with self.mic.recorder( + samplerate=16000, + channels=1, + blocksize=self.block_size, + ) as mic_in: mic_in.flush() except Exception as e: _LOGGER.warning("Error flushing mic: %s", e) - + if self.state.shutdown: return - + # --- Main Recording Loop --- _LOGGER.debug("Opening audio input device: %s", self.mic.name) - with self.mic.recorder(samplerate=16000, channels=1, blocksize=self.block_size) as mic_in: + with self.mic.recorder( + samplerate=16000, channels=1, blocksize=self.block_size + ) as mic_in: while self.state.mic_muted_event.is_set(): if self.state.shutdown: - return - + return + audio_chunk_array = mic_in.record(self.block_size).reshape(-1) audio_chunk = ( (np.clip(audio_chunk_array, -1.0, 1.0) * 32767.0) @@ -104,60 +110,141 @@ class AudioEngine: if self.state.satellite is None: time.sleep(0.01) continue - + # Update active wake words if changed if (not wake_words) or (self.state.wake_words_changed): self.state.wake_words_changed = False wake_words = [ - ww for ww in self.state.wake_words.values() + ww + for ww in self.state.wake_words.values() if ww.id in self.state.active_wake_words ] - has_oww = any(isinstance(ww, OpenWakeWord) for ww in wake_words) - + has_oww = any( + isinstance(ww, OpenWakeWord) for ww in wake_words + ) + if has_oww and (oww_features is None): - _LOGGER.debug("Initializing OpenWakeWord features...") + _LOGGER.debug( + "Initializing OpenWakeWord features..." + ) oww_features = OpenWakeWordFeatures.from_builtin() - + + _LOGGER.debug( + "Rebuilt wake_words list: %s (has_oww=%s, active_ids=%s)", + [ + getattr(ww, "wake_word", getattr(ww, "id", "?")) + for ww in wake_words + ], + has_oww, + self.state.active_wake_words, + ) + try: # Feature extraction micro_inputs.clear() - micro_inputs.extend(micro_features.process_streaming(audio_chunk)) - + micro_inputs.extend( + micro_features.process_streaming(audio_chunk) + ) + if has_oww and oww_features: oww_inputs.clear() - oww_inputs.extend(oww_features.process_streaming(audio_chunk)) - - # Detection + oww_inputs.extend( + oww_features.process_streaming(audio_chunk) + ) + + # Wake word detection for wake_word in wake_words: activated = False if isinstance(wake_word, MicroWakeWord): - if any(wake_word.process_streaming(mi) for mi in micro_inputs): + if any( + wake_word.process_streaming(mi) + for mi in micro_inputs + ): activated = True elif isinstance(wake_word, OpenWakeWord): - if any(p > 0.5 for oi in oww_inputs for p in wake_word.process_streaming(oi)): + if any( + p > 0.5 + for oi in oww_inputs + for p in wake_word.process_streaming(oi) + ): activated = True - + if activated: + wake_phrase = getattr( + wake_word, "wake_word", "wake word" + ) + wake_id = getattr(wake_word, "id", None) + _LOGGER.debug( + "Wake word ACTIVATED: %s (id=%s)", + wake_phrase, + wake_id, + ) now = time.monotonic() - if (last_active is None) or ((now - last_active) > self.state.refractory_seconds): - self.state.loop.call_soon_threadsafe(self.state.satellite.wakeup, wake_word) + if (last_active is None) or ( + (now - last_active) + > self.state.refractory_seconds + ): + _LOGGER.debug( + "Wake word ACCEPTED (phrase=%s); " + "triggering wakeup (Δt=%.2f, refractory=%.2f)", + wake_phrase, + 0.0 + if last_active is None + else now - last_active, + self.state.refractory_seconds, + ) + self.state.loop.call_soon_threadsafe( + self.state.satellite.wakeup, wake_word + ) last_active = now - + else: + _LOGGER.debug( + "Wake word REJECTED by refractory period " + "(phrase=%s, Δt=%.2f < %.2f)", + wake_phrase, + now - last_active, + self.state.refractory_seconds, + ) + # Stop word detection stopped = False for micro_input in micro_inputs: if self.state.stop_word.process_streaming(micro_input): stopped = True - if stopped and (self.state.stop_word.id in self.state.active_wake_words): - self.state.loop.call_soon_threadsafe(self.state.satellite.stop) - - # Pass audio to satellite (for voice streaming) - self.state.loop.call_soon_threadsafe(self.state.satellite.handle_audio, audio_chunk) + if stopped: + _LOGGER.debug( + "Stop wake word ACTIVATED (id=%s, active_wake_words=%s)", + getattr(self.state.stop_word, "id", None), + self.state.active_wake_words, + ) - except Exception: + if ( + stopped + and self.state.stop_word.id + in self.state.active_wake_words + ): + _LOGGER.debug( + "Stop wake word ACCEPTED; calling satellite.stop()" + ) + self.state.loop.call_soon_threadsafe( + self.state.satellite.stop + ) + elif stopped: + _LOGGER.debug( + "Stop wake word TRIGGERED but IGNORED because " + "id=%s not in active_wake_words", + getattr(self.state.stop_word, "id", None), + ) + + # Pass audio to satellite (for voice streaming) + self.state.loop.call_soon_threadsafe( + self.state.satellite.handle_audio, audio_chunk + ) + + except Exception: _LOGGER.exception("Unexpected error handling audio") - + except Exception as e: _LOGGER.critical("A soundcard error occurred: %s", e) self.state.loop.call_soon_threadsafe(self.state.loop.stop)