diff --git a/.gitignore b/.gitignore index 3c1e29b..91fc0d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,19 +1,75 @@ -.DS_Store -.idea -*.log -tmp/ - +# Byte-compiled / optimized / DLL files +__pycache__/ *.py[cod] -*.egg -build -htmlcov +*$py.class -.projectile -.venv/ -venv/ -.tox/ -.mypy_cache/ -*.egg-info/ +# C extensions +*.so +# Distribution / packaging +.Python +build/ +develop-eggs/ dist/ -preferences.json +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a script, but we don't want them in .git +*.spec + + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Foreign VSCode settings +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json + +# Ruff cache +.ruff_cache/ + +# Pyright cache +.pyright_cache/ + +# mypy +.mypy_cache/ + +# venv +venv/ +.venv/ + +# App-specific +/dist/ +/preferences.json +/local/ diff --git a/README.md b/README.md index b1c7b6e..4228ea3 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ See [the tutorial](docs/linux-voice-assistant-2mic-install.md) to build a satell ### What's working: - This fork is from https://github.com/OHF-Voice/linux-voice-assistant Release v1.0.0 which introduces the ablity to use both MicroWakeWord and OpenWakeWord detections models. +- Refactor: Sync with Upstream Architectural Changes (Nov 2025) incudes the 7 commits since the release of v1.0.0. - Updated to support LED Events including GPIO based LED controls. Defaults to the ReSpeaker 2Mic Hat SPI leds, but you can use the Grove port GPIO12/13 by updating the config.json file. - Updated to support running either APA102 or WS2812B LEDs from the SPI interface using a Micro Connectors 40-pin GPIO 1 to 2 Expansion Board. See the tutorial for instructions. - Now supports **ALSA, PulseAudio and PipeWire** playback backends using the updated `linux_voice_assistant/mpv_player.py`. diff --git a/lib/SOURCE b/lib/SOURCE deleted file mode 100644 index cff51ab..0000000 --- a/lib/SOURCE +++ /dev/null @@ -1 +0,0 @@ -https://github.com/tphakala/tflite_c/ diff --git a/lib/linux_amd64/libtensorflowlite_c.so b/lib/linux_amd64/libtensorflowlite_c.so deleted file mode 100644 index 34090dd..0000000 Binary files a/lib/linux_amd64/libtensorflowlite_c.so and /dev/null differ diff --git a/lib/linux_arm64/libtensorflowlite_c.so b/lib/linux_arm64/libtensorflowlite_c.so deleted file mode 100644 index ea2b03c..0000000 Binary files a/lib/linux_arm64/libtensorflowlite_c.so and /dev/null differ diff --git a/linux_voice_assistant/__main__.py b/linux_voice_assistant/__main__.py index 30448dc..2b585d0 100644 --- a/linux_voice_assistant/__main__.py +++ b/linux_voice_assistant/__main__.py @@ -3,36 +3,31 @@ import argparse import asyncio import json import logging +import sys import threading import time from pathlib import Path -from queue import Queue -from typing import Dict, List, Optional, Union +from typing import Dict, List, Optional, Set, Union import numpy as np -import sounddevice as sd +import soundcard as sc +from pymicro_wakeword import MicroWakeWord, MicroWakeWordFeatures +from pyopen_wakeword import OpenWakeWord, OpenWakeWordFeatures -from .config import Config, load_config_from_json # <-- NEW +from .config import Config, load_config_from_json from .event_bus import EventBus, EventHandler, subscribe from .led_controller import LedController from .mqtt_controller import MqttController -from .microwakeword import MicroWakeWord, MicroWakeWordFeatures from .models import AvailableWakeWord, Preferences, ServerState, WakeWordType from .mpv_player import MpvMediaPlayer -from .openwakeword import OpenWakeWord, OpenWakeWordFeatures from .satellite import VoiceSatelliteProtocol -from .util import get_mac, is_arm +from .util import get_mac from .zeroconf import HomeAssistantZeroconf _LOGGER = logging.getLogger(__name__) _MODULE_DIR = Path(__file__).parent _REPO_DIR = _MODULE_DIR.parent -if is_arm(): - _LIB_DIR = _REPO_DIR / "lib" / "linux_arm64" -else: - _LIB_DIR = _REPO_DIR / "lib" / "linux_amd64" - # ----------------------------------------------------------------------------- class MicMuteHandler(EventHandler): @@ -54,12 +49,6 @@ class MicMuteHandler(EventHandler): if is_muted: self.state.event_bus.publish("mic_muted") else: - _LOGGER.debug("Clearing stale audio queue...") - while not self.state.audio_queue.empty(): - try: - self.state.audio_queue.get_nowait() - except Queue.Empty: - break self.state.event_bus.publish("mic_unmuted") # ----------------------------------------------------------------------------- @@ -71,21 +60,69 @@ async def main() -> None: "-c", "--config", type=Path, - required=True, - help="Path to configuration.json file", + required=False, + default=_MODULE_DIR / "config.json", + help="Path to configuration.json file (default: linux_voice_assistant/config.json)", + ) + parser.add_argument( + "--debug", + action="store_true", + help="Enable debug logging (overrides config file)", + ) + parser.add_argument( + "--list-input-devices", + action="store_true", + help="List audio input devices and exit", + ) + parser.add_argument( + "--list-output-devices", + action="store_true", + help="List audio output devices and exit", ) args = parser.parse_args() - # --- Load Configuration --- - config = load_config_from_json(args.config) + # --- Handle List Devices --- + if args.list_input_devices: + print("Input devices") + print("=" * 13) + for idx, mic in enumerate(sc.all_microphones(include_loopback=False)): + print(f"[{idx}]", mic.name) + return + + if args.list_output_devices: + player = MpvMediaPlayer(loop=None) + print("Output devices") + print("=" * 14) + try: + for speaker in player.player.audio_device_list: + print(speaker["name"] + ":", speaker["description"]) + except Exception as e: + _LOGGER.error("Failed to list output devices: %s", e) + return + + # --- Load Configuration --- + config_path = args.config + + if not config_path.is_absolute(): + config_path = _REPO_DIR / config_path + + config = load_config_from_json(config_path) + + if args.debug: + config.app.debug = True - # --- Setup Logging --- logging.basicConfig(level=logging.DEBUG if config.app.debug else logging.INFO) + _LOGGER.info("Loading configuration from: %s", config_path) _LOGGER.debug("Configuration loaded: %s", config) loop = asyncio.get_running_loop() event_bus = EventBus() + # --- Create Download Dir --- + download_dir = _REPO_DIR / config.wake_word.download_dir + download_dir.mkdir(parents=True, exist_ok=True) + _LOGGER.debug("Download directory: %s", download_dir) + # --- Load Preferences --- preferences_path = _REPO_DIR / config.app.preferences_file if preferences_path.exists(): @@ -95,51 +132,63 @@ async def main() -> None: else: preferences = Preferences() - # Set default num_leds from config, allowing preferences to override preferences.num_leds = getattr(preferences, 'num_leds', config.led.num_leds) # --- Load Wake Words --- available_wake_words: Dict[str, AvailableWakeWord] = {} - # Add default directories if none are specified if not config.wake_word.directories: config.wake_word.directories = ["wakewords", "wakewords/openWakeWord"] - for ww_dir_str in config.wake_word.directories: - wake_word_dir = _REPO_DIR / ww_dir_str + # Add download directory to search path + wake_word_dirs = [_REPO_DIR / d for d in config.wake_word.directories] + wake_word_dirs.append(download_dir / "external_wake_words") # <-- ADDED + + for wake_word_dir in wake_word_dirs: + if not wake_word_dir.exists(): + continue + for model_config_path in wake_word_dir.glob("*.json"): model_id = model_config_path.stem if model_id == config.wake_word.stop_model: continue with open(model_config_path, "r", encoding="utf-8") as model_config_file: model_config = json.load(model_config_file) - model_type = model_config["type"] + model_type = WakeWordType(model_config["type"]) + + if model_type == WakeWordType.OPEN_WAKE_WORD: + wake_word_path = model_config_path.parent / model_config["model"] + else: + wake_word_path = model_config_path + available_wake_words[model_id] = AvailableWakeWord( id=model_id, - type=WakeWordType(model_type), + type=model_type, wake_word=model_config["wake_word"], trained_languages=model_config.get("trained_languages", []), - config_path=model_config_path, + wake_word_path=wake_word_path, ) _LOGGER.debug("Available wake words: %s", list(sorted(available_wake_words.keys()))) - libtensorflowlite_c_path = _LIB_DIR / "libtensorflowlite_c.so" - # --- Load Active Wake Word Models --- + active_wake_words: Set[str] = set() wake_models: Dict[str, Union[MicroWakeWord, OpenWakeWord]] = {} if preferences.active_wake_words: for wake_word_id in preferences.active_wake_words: wake_word = available_wake_words.get(wake_word_id) if wake_word is None: continue - wake_models[wake_word_id] = wake_word.load(libtensorflowlite_c_path) + _LOGGER.debug("Loading wake model: %s", wake_word_id) + wake_models[wake_word_id] = wake_word.load() + active_wake_words.add(wake_word_id) - # Load default model if no preferences are set if not wake_models: wake_word_id = config.wake_word.model wake_word = available_wake_words.get(wake_word_id) if wake_word: - wake_models[wake_word_id] = wake_word.load(libtensorflowlite_c_path) + _LOGGER.debug("Loading wake model: %s", wake_word_id) + wake_models[wake_word_id] = wake_word.load() + active_wake_words.add(wake_word_id) # --- Load Stop Model --- stop_model: Optional[MicroWakeWord] = None @@ -147,10 +196,27 @@ async def main() -> None: wake_word_dir = _REPO_DIR / ww_dir_str stop_config_path = wake_word_dir / f"{config.wake_word.stop_model}.json" if not stop_config_path.exists(): continue - stop_model = MicroWakeWord.from_config(stop_config_path, libtensorflowlite_c_path) + _LOGGER.debug("Loading stop model: %s", stop_config_path) + stop_model = MicroWakeWord.from_config(stop_config_path) break assert stop_model is not None + + # --- Resolve Microphone --- + if config.audio.input_device is not None: + try: + input_device_idx = int(config.audio.input_device) + mic = sc.all_microphones(include_loopback=False)[input_device_idx] + except (ValueError, IndexError): + mic = sc.get_microphone(config.audio.input_device) + else: + mic = sc.default_microphone() + if mic is None: + _LOGGER.critical("No microphone found.") + sys.exit(1) + + _LOGGER.info("Using audio input device: %s", mic.name) + # --- Initialize Media Players --- music_player = MpvMediaPlayer( loop=loop, @@ -167,10 +233,10 @@ async def main() -> None: state = ServerState( name=config.app.name, mac_address=get_mac(), - audio_queue=Queue(), entities=[], available_wake_words=available_wake_words, wake_words=wake_models, + active_wake_words=active_wake_words, stop_word=stop_model, music_player=music_player, tts_player=tts_player, @@ -178,11 +244,9 @@ async def main() -> None: timer_finished_sound=str(_REPO_DIR / config.app.timer_finished_sound), preferences=preferences, preferences_path=preferences_path, - libtensorflowlite_c_path=libtensorflowlite_c_path, event_bus=event_bus, loop=loop, - oww_melspectrogram_path=(_REPO_DIR / config.wake_word.oww_melspectrogram_model), - oww_embedding_path=(_REPO_DIR / config.wake_word.oww_embedding_model), + download_dir=download_dir, # <-- ADDED refractory_seconds=config.wake_word.refractory_seconds, ) @@ -210,11 +274,12 @@ async def main() -> None: mic_mute_handler = MicMuteHandler(state, mqtt_controller) # --- Start Audio Processing Thread --- - process_audio_thread = threading.Thread(target=process_audio, args=(state,), daemon=True) + process_audio_thread = threading.Thread( + target=process_audio, + args=(state, mic, config.audio.input_block_size), + daemon=True + ) process_audio_thread.start() - - def sd_callback(indata, _frames, _time, _status): - state.audio_queue.put_nowait(bytes(indata)) # --- Start ESPHome Server --- server = await loop.create_server( @@ -227,31 +292,22 @@ async def main() -> None: # --- Run Forever --- try: - input_device = config.audio.input_device if config.audio.input_device else "default" - with sd.RawInputStream( - samplerate=16000, - blocksize=config.audio.input_block_size, - device=input_device, - dtype="int16", - channels=1, - callback=sd_callback - ): - async with server: - _LOGGER.info("Server started (host=%s, port=%s)", config.esphome.host, config.esphome.port) - await server.serve_forever() + async with server: + _LOGGER.info("Server started (host=%s, port=%s)", config.esphome.host, config.esphome.port) + await server.serve_forever() except KeyboardInterrupt: pass - except sd.PortAudioError as e: - _LOGGER.critical("Failed to open audio input device '%s': %s", input_device, e) finally: if mqtt_controller: mqtt_controller.stop() - state.audio_queue.put_nowait(None) + + state.mic_muted = True process_audio_thread.join() _LOGGER.debug("Server stopped") -def process_audio(state: ServerState): +def process_audio(state: ServerState, mic, block_size: int): + """Process audio chunks from the microphone in a separate thread.""" wake_words: List[Union[MicroWakeWord, OpenWakeWord]] = [] micro_features: Optional[MicroWakeWordFeatures] = None micro_inputs: List[np.ndarray] = [] @@ -260,43 +316,69 @@ def process_audio(state: ServerState): has_oww = False last_active: Optional[float] = None try: - while True: - if state.mic_muted: - time.sleep(0.1) - continue - audio_chunk = state.audio_queue.get() - if audio_chunk is None: break - if state.satellite is None: continue - if (not wake_words) or (state.wake_words_changed and state.wake_words): - state.wake_words_changed = False - wake_words = [ww for ww in state.wake_words.values() if ww.is_active] - has_oww = any(isinstance(ww, OpenWakeWord) for ww in wake_words) - if micro_features is None: - micro_features = MicroWakeWordFeatures(libtensorflowlite_c_path=state.libtensorflowlite_c_path) - if has_oww and (oww_features is None): - oww_features = OpenWakeWordFeatures(melspectrogram_model=state.oww_melspectrogram_path, embedding_model=state.oww_embedding_path, libtensorflowlite_c_path=state.libtensorflowlite_c_path) - try: - state.satellite.handle_audio(audio_chunk) - assert micro_features is not None - micro_inputs.clear(); micro_inputs.extend(micro_features.process_streaming(audio_chunk)) - if has_oww: - assert oww_features is not None - oww_inputs.clear(); oww_inputs.extend(oww_features.process_streaming(audio_chunk)) - for wake_word in wake_words: - activated = False - if isinstance(wake_word, MicroWakeWord): - if any(wake_word.process_streaming(mi) for mi in micro_inputs): activated = True - elif isinstance(wake_word, OpenWakeWord): - if any(p > 0.5 for oi in oww_inputs for p in wake_word.process_streaming(oi)): activated = True - if activated: - now = time.monotonic() - if (last_active is None) or ((now - last_active) > state.refractory_seconds): - state.satellite.wakeup(wake_word) - last_active = now - if any(state.stop_word.process_streaming(mi) for mi in micro_inputs) and state.stop_word.is_active: - state.satellite.stop() - except Exception: _LOGGER.exception("Unexpected error handling audio") - except Exception: _LOGGER.exception("Unexpected error processing audio") + _LOGGER.debug("Opening audio input device: %s", mic.name) + with mic.recorder(samplerate=16000, channels=1, blocksize=block_size) as mic_in: + while True: + if state.mic_muted: + time.sleep(0.1) + continue + + audio_chunk_array = mic_in.record(block_size).reshape(-1) + audio_chunk = ( + (np.clip(audio_chunk_array, -1.0, 1.0) * 32767.0) + .astype(" 0.5 for oi in oww_inputs for p in wake_word.process_streaming(oi)): activated = True + + if activated: + now = time.monotonic() + if (last_active is None) or ((now - last_active) > state.refractory_seconds): + state.satellite.wakeup(wake_word) + last_active = now + + stopped = False + for micro_input in micro_inputs: + if state.stop_word.process_streaming(micro_input): + stopped = True + + if stopped and (state.stop_word.id in state.active_wake_words): + state.satellite.stop() + except Exception: _LOGGER.exception("Unexpected error handling audio") + except Exception as e: + _LOGGER.critical("Error in audio processing thread: %s", e) + state.loop.call_soon_threadsafe(state.loop.stop) + if __name__ == "__main__": print("--- __main__ block executing ---") diff --git a/linux_voice_assistant/config.py b/linux_voice_assistant/config.py index 56a698f..74e453b 100644 --- a/linux_voice_assistant/config.py +++ b/linux_voice_assistant/config.py @@ -23,8 +23,7 @@ class WakeWordConfig: model: str = "okay_nabu" stop_model: str = "stop" refractory_seconds: float = 2.0 - oww_melspectrogram_model: str = "wakewords/openWakeWord/melspectrogram.tflite" - oww_embedding_model: str = "wakewords/openWakeWord/embedding_model.tflite" + download_dir: str = "local" @dataclass class ESPHomeConfig: @@ -88,10 +87,6 @@ def load_config_from_json(config_path: Path) -> Config: raise # --- Step 2: Create config objects from raw data --- - # This nested loading allows users to omit entire sections (like 'audio') - # and still get the defaults. - - # 'app' is the only required section if "app" not in raw_data: raise ValueError("Configuration file must contain an 'app' section with a 'name'.") @@ -103,7 +98,6 @@ def load_config_from_json(config_path: Path) -> Config: mqtt_config = MqttConfig(**raw_data.get("mqtt", {})) # --- Step 3: Set MQTT 'enabled' flag --- - # If host is provided, automatically enable MQTT if mqtt_config.host: mqtt_config.enabled = True @@ -117,6 +111,5 @@ def load_config_from_json(config_path: Path) -> Config: mqtt=mqtt_config, ) -# Logger is needed for the load function import logging _LOGGER = logging.getLogger(__name__) diff --git a/linux_voice_assistant/microwakeword.py b/linux_voice_assistant/microwakeword.py deleted file mode 100644 index 79b4067..0000000 --- a/linux_voice_assistant/microwakeword.py +++ /dev/null @@ -1,189 +0,0 @@ -import ctypes -import json -import statistics -from collections import deque -from collections.abc import Iterable -from pathlib import Path -from typing import Deque, List, Union - -import numpy as np -from pymicro_features import MicroFrontend - -from .wakeword import TfLiteWakeWord - -SAMPLES_PER_SECOND = 16000 -SAMPLES_PER_CHUNK = 160 # 10ms -BYTES_PER_SAMPLE = 2 # 16-bit -BYTES_PER_CHUNK = SAMPLES_PER_CHUNK * BYTES_PER_SAMPLE -SECONDS_PER_CHUNK = SAMPLES_PER_CHUNK / SAMPLES_PER_SECOND -STRIDE = 3 - - -class MicroWakeWord(TfLiteWakeWord): - def __init__( - self, - id: str, # pylint: disable=redefined-builtin - wake_word: str, - tflite_model: Union[str, Path], - probability_cutoff: float, - sliding_window_size: int, - trained_languages: List[str], - libtensorflowlite_c_path: Union[str, Path], - ) -> None: - TfLiteWakeWord.__init__(self, libtensorflowlite_c_path) - - self.id = id - self.wake_word = wake_word - self.tflite_model = tflite_model - self.probability_cutoff = probability_cutoff - self.sliding_window_size = sliding_window_size - self.trained_languages = trained_languages - - self.is_active = True - - # Load the model and create interpreter - self.model_path = str(Path(tflite_model).resolve()).encode("utf-8") - self._load_model() - - self._features: List[np.ndarray] = [] - self._probabilities: Deque[float] = deque(maxlen=self.sliding_window_size) - self._audio_buffer = bytes() - - def _load_model(self) -> None: - self.model = self.lib.TfLiteModelCreateFromFile(self.model_path) - self.interpreter = self.lib.TfLiteInterpreterCreate(self.model, None) - self.lib.TfLiteInterpreterAllocateTensors(self.interpreter) - - # Access input and output tensor - self.input_tensor = self.lib.TfLiteInterpreterGetInputTensor( - self.interpreter, 0 - ) - self.output_tensor = self.lib.TfLiteInterpreterGetOutputTensor( - self.interpreter, 0 - ) - - # Get quantization parameters - input_q = self.lib.TfLiteTensorQuantizationParams(self.input_tensor) - output_q = self.lib.TfLiteTensorQuantizationParams(self.output_tensor) - - self.input_scale, self.input_zero_point = input_q.scale, input_q.zero_point - self.output_scale, self.output_zero_point = output_q.scale, output_q.zero_point - - @staticmethod - def from_config( - config_path: Union[str, Path], - libtensorflowlite_c_path: Union[str, Path], - ) -> "MicroWakeWord": - """Load a microWakeWord model from a JSON config file. - - Parameters - ---------- - config_path: str or Path - Path to JSON configuration file - """ - config_path = Path(config_path) - with open(config_path, "r", encoding="utf-8") as config_file: - config = json.load(config_file) - - micro_config = config["micro"] - - return MicroWakeWord( - id=Path(config["model"]).stem, - wake_word=config["wake_word"], - tflite_model=config_path.parent / config["model"], - probability_cutoff=micro_config["probability_cutoff"], - sliding_window_size=micro_config["sliding_window_size"], - trained_languages=micro_config.get("trained_languages", []), - libtensorflowlite_c_path=libtensorflowlite_c_path, - ) - - def process_streaming(self, features: np.ndarray) -> bool: - self._features.append(features) - - if len(self._features) < STRIDE: - # Not enough windows - return False - - # Allocate and quantize input data - quant_features = np.round( - np.concatenate(self._features, axis=1) / self.input_scale - + self.input_zero_point - ).astype(np.uint8) - - # Stride instead of rolling - self._features.clear() - - # Set tensor - quant_ptr = quant_features.ctypes.data_as(ctypes.c_void_p) - self.lib.TfLiteTensorCopyFromBuffer( - self.input_tensor, quant_ptr, quant_features.nbytes - ) - - # Run inference - self.lib.TfLiteInterpreterInvoke(self.interpreter) - - # Read output - output_bytes = self.lib.TfLiteTensorByteSize(self.output_tensor) - output_data = np.empty(output_bytes, dtype=np.uint8) - self.lib.TfLiteTensorCopyToBuffer( - self.output_tensor, - output_data.ctypes.data_as(ctypes.c_void_p), - output_bytes, - ) - - # Dequantize output - result = ( - output_data.astype(np.float32) - self.output_zero_point - ) * self.output_scale - - self._probabilities.append(result.item()) - - if len(self._probabilities) < self.sliding_window_size: - # Not enough probabilities - return False - - if statistics.mean(self._probabilities) > self.probability_cutoff: - return True - - return False - - -# ----------------------------------------------------------------------------- - - -class MicroWakeWordFeatures(TfLiteWakeWord): - def __init__( - self, - libtensorflowlite_c_path: Union[str, Path], - ) -> None: - TfLiteWakeWord.__init__(self, libtensorflowlite_c_path) - - self._audio_buffer = bytes() - self._frontend = MicroFrontend() - - def process_streaming(self, audio_bytes: bytes) -> Iterable[np.ndarray]: - self._audio_buffer += audio_bytes - - if len(self._audio_buffer) < BYTES_PER_CHUNK: - # Not enough audio to get features - return - - audio_buffer_idx = 0 - while (audio_buffer_idx + BYTES_PER_CHUNK) <= len(self._audio_buffer): - # Process chunk - chunk_bytes = self._audio_buffer[ - audio_buffer_idx : audio_buffer_idx + BYTES_PER_CHUNK - ] - frontend_result = self._frontend.ProcessSamples(chunk_bytes) - audio_buffer_idx += frontend_result.samples_read * BYTES_PER_SAMPLE - - if not frontend_result.features: - # Not enough audio for a full window - continue - - yield np.array(frontend_result.features).reshape( - (1, 1, len(frontend_result.features)) - ) - - # Remove processed audio - self._audio_buffer = self._audio_buffer[audio_buffer_idx:] diff --git a/linux_voice_assistant/models.py b/linux_voice_assistant/models.py index 6ded847..2b585d0 100644 --- a/linux_voice_assistant/models.py +++ b/linux_voice_assistant/models.py @@ -1,111 +1,385 @@ -"""Shared models.""" - +#!/usr/bin/env python3 +import argparse import asyncio import json import logging -from dataclasses import asdict, dataclass, field -from enum import Enum +import sys +import threading +import time from pathlib import Path -from queue import Queue -from typing import TYPE_CHECKING, Dict, List, Optional, Union +from typing import Dict, List, Optional, Set, Union -from .event_bus import EventBus +import numpy as np +import soundcard as sc +from pymicro_wakeword import MicroWakeWord, MicroWakeWordFeatures +from pyopen_wakeword import OpenWakeWord, OpenWakeWordFeatures -if TYPE_CHECKING: - from .entity import ESPHomeEntity, MediaPlayerEntity - from .microwakeword import MicroWakeWord - from .mpv_player import MpvMediaPlayer - from .openwakeword import OpenWakeWord - from .satellite import VoiceSatelliteProtocol +from .config import Config, load_config_from_json +from .event_bus import EventBus, EventHandler, subscribe +from .led_controller import LedController +from .mqtt_controller import MqttController +from .models import AvailableWakeWord, Preferences, ServerState, WakeWordType +from .mpv_player import MpvMediaPlayer +from .satellite import VoiceSatelliteProtocol +from .util import get_mac +from .zeroconf import HomeAssistantZeroconf _LOGGER = logging.getLogger(__name__) +_MODULE_DIR = Path(__file__).parent +_REPO_DIR = _MODULE_DIR.parent + +# ----------------------------------------------------------------------------- + +class MicMuteHandler(EventHandler): + """Event handler for mic mute switch.""" + def __init__(self, state, mqtt_controller: Optional[MqttController]): + super().__init__(state) + self.mqtt_controller = mqtt_controller + + @subscribe + def set_mic_mute(self, data: dict): + is_muted = data.get("state", False) + if self.state.mic_muted != is_muted: + self.state.mic_muted = is_muted + _LOGGER.debug("Mic muted = %s", is_muted) + + if self.mqtt_controller: + self.mqtt_controller.publish_mute_state(is_muted) + + if is_muted: + self.state.event_bus.publish("mic_muted") + else: + self.state.event_bus.publish("mic_unmuted") + +# ----------------------------------------------------------------------------- -class SatelliteState(str, Enum): - """Voice satellite state.""" - STARTING = "starting" # <-- ADDED - IDLE = "idle" - LISTENING = "listening" # Wake word detected - THINKING = "thinking" # VAD, waiting for STT/intent - RESPONDING = "responding" # Playing TTS - ERROR = "error" +async def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "-c", + "--config", + type=Path, + required=False, + default=_MODULE_DIR / "config.json", + help="Path to configuration.json file (default: linux_voice_assistant/config.json)", + ) + parser.add_argument( + "--debug", + action="store_true", + help="Enable debug logging (overrides config file)", + ) + parser.add_argument( + "--list-input-devices", + action="store_true", + help="List audio input devices and exit", + ) + parser.add_argument( + "--list-output-devices", + action="store_true", + help="List audio output devices and exit", + ) + args = parser.parse_args() + + # --- Handle List Devices --- + if args.list_input_devices: + print("Input devices") + print("=" * 13) + for idx, mic in enumerate(sc.all_microphones(include_loopback=False)): + print(f"[{idx}]", mic.name) + return + + if args.list_output_devices: + player = MpvMediaPlayer(loop=None) + print("Output devices") + print("=" * 14) + try: + for speaker in player.player.audio_device_list: + print(speaker["name"] + ":", speaker["description"]) + except Exception as e: + _LOGGER.error("Failed to list output devices: %s", e) + return + + # --- Load Configuration --- + config_path = args.config + + if not config_path.is_absolute(): + config_path = _REPO_DIR / config_path + + config = load_config_from_json(config_path) + + if args.debug: + config.app.debug = True + + logging.basicConfig(level=logging.DEBUG if config.app.debug else logging.INFO) + _LOGGER.info("Loading configuration from: %s", config_path) + _LOGGER.debug("Configuration loaded: %s", config) + + loop = asyncio.get_running_loop() + event_bus = EventBus() + + # --- Create Download Dir --- + download_dir = _REPO_DIR / config.wake_word.download_dir + download_dir.mkdir(parents=True, exist_ok=True) + _LOGGER.debug("Download directory: %s", download_dir) + + # --- Load Preferences --- + preferences_path = _REPO_DIR / config.app.preferences_file + if preferences_path.exists(): + with open(preferences_path, "r", encoding="utf-8") as preferences_file: + preferences_dict = json.load(preferences_file) + preferences = Preferences(**preferences_dict) + else: + preferences = Preferences() + + preferences.num_leds = getattr(preferences, 'num_leds', config.led.num_leds) + + # --- Load Wake Words --- + available_wake_words: Dict[str, AvailableWakeWord] = {} + + if not config.wake_word.directories: + config.wake_word.directories = ["wakewords", "wakewords/openWakeWord"] + + # Add download directory to search path + wake_word_dirs = [_REPO_DIR / d for d in config.wake_word.directories] + wake_word_dirs.append(download_dir / "external_wake_words") # <-- ADDED + + for wake_word_dir in wake_word_dirs: + if not wake_word_dir.exists(): + continue + + for model_config_path in wake_word_dir.glob("*.json"): + model_id = model_config_path.stem + if model_id == config.wake_word.stop_model: + continue + with open(model_config_path, "r", encoding="utf-8") as model_config_file: + model_config = json.load(model_config_file) + model_type = WakeWordType(model_config["type"]) + + if model_type == WakeWordType.OPEN_WAKE_WORD: + wake_word_path = model_config_path.parent / model_config["model"] + else: + wake_word_path = model_config_path + + available_wake_words[model_id] = AvailableWakeWord( + id=model_id, + type=model_type, + wake_word=model_config["wake_word"], + trained_languages=model_config.get("trained_languages", []), + wake_word_path=wake_word_path, + ) + + _LOGGER.debug("Available wake words: %s", list(sorted(available_wake_words.keys()))) + + # --- Load Active Wake Word Models --- + active_wake_words: Set[str] = set() + wake_models: Dict[str, Union[MicroWakeWord, OpenWakeWord]] = {} + if preferences.active_wake_words: + for wake_word_id in preferences.active_wake_words: + wake_word = available_wake_words.get(wake_word_id) + if wake_word is None: continue + _LOGGER.debug("Loading wake model: %s", wake_word_id) + wake_models[wake_word_id] = wake_word.load() + active_wake_words.add(wake_word_id) + + if not wake_models: + wake_word_id = config.wake_word.model + wake_word = available_wake_words.get(wake_word_id) + if wake_word: + _LOGGER.debug("Loading wake model: %s", wake_word_id) + wake_models[wake_word_id] = wake_word.load() + active_wake_words.add(wake_word_id) + + # --- Load Stop Model --- + stop_model: Optional[MicroWakeWord] = None + for ww_dir_str in config.wake_word.directories: + wake_word_dir = _REPO_DIR / ww_dir_str + stop_config_path = wake_word_dir / f"{config.wake_word.stop_model}.json" + if not stop_config_path.exists(): continue + _LOGGER.debug("Loading stop model: %s", stop_config_path) + stop_model = MicroWakeWord.from_config(stop_config_path) + break + assert stop_model is not None + + # --- Resolve Microphone --- + if config.audio.input_device is not None: + try: + input_device_idx = int(config.audio.input_device) + mic = sc.all_microphones(include_loopback=False)[input_device_idx] + except (ValueError, IndexError): + mic = sc.get_microphone(config.audio.input_device) + else: + mic = sc.default_microphone() + + if mic is None: + _LOGGER.critical("No microphone found.") + sys.exit(1) + + _LOGGER.info("Using audio input device: %s", mic.name) + + # --- Initialize Media Players --- + music_player = MpvMediaPlayer( + loop=loop, + device=config.audio.output_device, + initial_volume=preferences.volume_level + ) + tts_player = MpvMediaPlayer( + loop=loop, + device=config.audio.output_device, + initial_volume=preferences.volume_level + ) + + # --- Create Global Server State --- + state = ServerState( + name=config.app.name, + mac_address=get_mac(), + entities=[], + available_wake_words=available_wake_words, + wake_words=wake_models, + active_wake_words=active_wake_words, + stop_word=stop_model, + music_player=music_player, + tts_player=tts_player, + wakeup_sound=str(_REPO_DIR / config.app.wakeup_sound), + timer_finished_sound=str(_REPO_DIR / config.app.timer_finished_sound), + preferences=preferences, + preferences_path=preferences_path, + event_bus=event_bus, + loop=loop, + download_dir=download_dir, # <-- ADDED + refractory_seconds=config.wake_word.refractory_seconds, + ) + + # --- Initialize Controllers --- + led_controller = LedController( + state, + led_type=config.led.led_type, + interface=config.led.interface, + clock_pin=config.led.clock_pin, + data_pin=config.led.data_pin, + num_leds=state.preferences.num_leds, + ) + + mqtt_controller: Optional[MqttController] = None + if config.mqtt.enabled: + mqtt_controller = MqttController( + state=state, + host=config.mqtt.host, + port=config.mqtt.port, + username=config.mqtt.username, + password=config.mqtt.password + ) + mqtt_controller.start() + + mic_mute_handler = MicMuteHandler(state, mqtt_controller) + + # --- Start Audio Processing Thread --- + process_audio_thread = threading.Thread( + target=process_audio, + args=(state, mic, config.audio.input_block_size), + daemon=True + ) + process_audio_thread.start() + + # --- Start ESPHome Server --- + server = await loop.create_server( + lambda: VoiceSatelliteProtocol(state), + host=config.esphome.host, + port=config.esphome.port + ) + discovery = HomeAssistantZeroconf(port=config.esphome.port, name=config.app.name) + await discovery.register_server() + + # --- Run Forever --- + try: + async with server: + _LOGGER.info("Server started (host=%s, port=%s)", config.esphome.host, config.esphome.port) + await server.serve_forever() + except KeyboardInterrupt: + pass + finally: + if mqtt_controller: + mqtt_controller.stop() + + state.mic_muted = True + process_audio_thread.join() + + _LOGGER.debug("Server stopped") + +def process_audio(state: ServerState, mic, block_size: int): + """Process audio chunks from the microphone in a separate thread.""" + wake_words: List[Union[MicroWakeWord, OpenWakeWord]] = [] + micro_features: Optional[MicroWakeWordFeatures] = None + micro_inputs: List[np.ndarray] = [] + oww_features: Optional[OpenWakeWordFeatures] = None + oww_inputs: List[np.ndarray] = [] + has_oww = False + last_active: Optional[float] = None + try: + _LOGGER.debug("Opening audio input device: %s", mic.name) + with mic.recorder(samplerate=16000, channels=1, blocksize=block_size) as mic_in: + while True: + if state.mic_muted: + time.sleep(0.1) + continue + + audio_chunk_array = mic_in.record(block_size).reshape(-1) + audio_chunk = ( + (np.clip(audio_chunk_array, -1.0, 1.0) * 32767.0) + .astype(" 0.5 for oi in oww_inputs for p in wake_word.process_streaming(oi)): activated = True + + if activated: + now = time.monotonic() + if (last_active is None) or ((now - last_active) > state.refractory_seconds): + state.satellite.wakeup(wake_word) + last_active = now + + stopped = False + for micro_input in micro_inputs: + if state.stop_word.process_streaming(micro_input): + stopped = True + + if stopped and (state.stop_word.id in state.active_wake_words): + state.satellite.stop() + except Exception: _LOGGER.exception("Unexpected error handling audio") + except Exception as e: + _LOGGER.critical("Error in audio processing thread: %s", e) + state.loop.call_soon_threadsafe(state.loop.stop) -class WakeWordType(str, Enum): - MICRO_WAKE_WORD = "micro" - OPEN_WAKE_WORD = "openWakeWord" - - -@dataclass -class AvailableWakeWord: - id: str - type: WakeWordType - wake_word: str - trained_languages: List[str] - config_path: Path - - def load( - self, libtensorflowlite_c_path: Path - ) -> "Union[MicroWakeWord, OpenWakeWord]": - if self.type == WakeWordType.MICRO_WAKE_WORD: - from .microwakeword import MicroWakeWord - - return MicroWakeWord.from_config( - config_path=self.config_path, - libtensorflowlite_c_path=libtensorflowlite_c_path, - ) - - if self.type == WakeWordType.OPEN_WAKE_WORD: - from .openwakeword import OpenWakeWord - - return OpenWakeWord.from_config( - config_path=self.config_path, - libtensorflowlite_c_path=libtensorflowlite_c_path, - ) - - raise ValueError(f"Unexpected wake word type: {self.type}") - - -@dataclass -class Preferences: - active_wake_words: List[str] = field(default_factory=list) - volume_level: float = 1.0 - num_leds: int = 3 - - -@dataclass -class ServerState: - # --- Fields WITHOUT default values --- - name: str - mac_address: str - audio_queue: "Queue[Optional[bytes]]" - entities: "List[ESPHomeEntity]" - available_wake_words: "Dict[str, AvailableWakeWord]" - wake_words: "Dict[str, Union[MicroWakeWord, OpenWakeWord]]" - stop_word: "MicroWakeWord" - music_player: "MpvMediaPlayer" - tts_player: "MpvMediaPlayer" - wakeup_sound: str - timer_finished_sound: str - preferences: Preferences - preferences_path: Path - libtensorflowlite_c_path: Path - event_bus: EventBus - loop: asyncio.AbstractEventLoop - oww_melspectrogram_path: Path - oww_embedding_path: Path - - # --- Fields WITH default values --- - satellite: "Optional[VoiceSatelliteProtocol]" = None - wake_words_changed: bool = False - refractory_seconds: float = 2.0 - mic_muted: bool = False - - def save_preferences(self) -> None: - """Save preferences as JSON.""" - _LOGGER.debug("Saving preferences: %s", self.preferences_path) - self.preferences_path.parent.mkdir(parents=True, exist_ok=True) - with open(self.preferences_path, "w", encoding="utf-8") as preferences_file: - json.dump( - asdict(self.preferences), preferences_file, ensure_ascii=False, indent=4 - ) +if __name__ == "__main__": + print("--- __main__ block executing ---") + asyncio.run(main()) diff --git a/linux_voice_assistant/openwakeword.py b/linux_voice_assistant/openwakeword.py deleted file mode 100644 index 317ea74..0000000 --- a/linux_voice_assistant/openwakeword.py +++ /dev/null @@ -1,293 +0,0 @@ -"""openWakeWord implementation.""" - -import ctypes as C -import json -from collections.abc import Iterable -from pathlib import Path -from typing import Final, Union - -import numpy as np - -from .wakeword import TfLiteWakeWord - -c_void_p = C.c_void_p -c_int32 = C.c_int32 -c_size_t = C.c_size_t - -BATCH_SIZE: Final = 1 - -AUTOFILL_SECONDS: Final = 8 -MAX_SECONDS: Final = 10 - -SAMPLE_RATE: Final = 16000 # 16Khz -_MAX_SAMPLES: Final = MAX_SECONDS * SAMPLE_RATE - -SAMPLES_PER_CHUNK: Final = 1280 # 80 ms @ 16Khz -MS_PER_CHUNK: Final = SAMPLES_PER_CHUNK // SAMPLE_RATE - -# window = 400, hop length = 160 -MELS_PER_SECOND: Final = 97 -MAX_MELS: Final = MAX_SECONDS * MELS_PER_SECOND -MEL_SAMPLES: Final = 1760 -NUM_MELS: Final = 32 - -EMB_FEATURES: Final = 76 # 775 ms -EMB_STEP: Final = 8 -MAX_EMB: Final = MAX_SECONDS * EMB_STEP -WW_FEATURES: Final = 96 - -MEL_SHAPE: Final = (BATCH_SIZE, MEL_SAMPLES) -EMB_SHAPE: Final = (BATCH_SIZE, EMB_FEATURES, NUM_MELS, 1) - -# melspec = [batch x samples (min: 1280)] => [batch x 1 x window x mels (32)] -# stft window size: 25ms (400) -# stft window step: 10ms (160) -# mel band limits: 60Hz - 3800Hz -# mel frequency bins: 32 -# -# embedding = [batch x window x mels (32) x 1] => [batch x 1 x 1 x features (96)] -# ww = [batch x window x features (96)] => [batch x probability] - - -class OpenWakeWord(TfLiteWakeWord): - def __init__( - self, - id: str, # pylint: disable=redefined-builtin - wake_word: str, - tflite_model: Union[str, Path], - libtensorflowlite_c_path: Union[str, Path], - ): - TfLiteWakeWord.__init__(self, libtensorflowlite_c_path) - - self.id = id - self.wake_word = wake_word - self.tflite_model = tflite_model - - self.is_active = True - - # Load the model and create interpreter - self.model_path = str(Path(tflite_model).resolve()).encode("utf-8") - self.model = self.lib.TfLiteModelCreateFromFile(self.model_path) - self.interpreter = self.lib.TfLiteInterpreterCreate(self.model, None) - self.lib.TfLiteInterpreterAllocateTensors(self.interpreter) - - self.input_tensor = self.lib.TfLiteInterpreterGetInputTensor( - self.interpreter, c_int32(0) - ) - self.output_tensor = self.lib.TfLiteInterpreterGetOutputTensor( - self.interpreter, c_int32(0) - ) - - num_input_dims = self.lib.TfLiteTensorNumDims(self.input_tensor) - input_shape = [ - self.lib.TfLiteTensorDim(self.input_tensor, i) - for i in range(num_input_dims) - ] - self.input_windows = input_shape[1] - - self.new_embeddings: int = 0 - self.embeddings: np.ndarray = np.zeros( - shape=(MAX_EMB, WW_FEATURES), dtype=np.float32 - ) - - def process_streaming(self, embeddings: np.ndarray) -> Iterable[float]: - """Generate probabilities from embeddings.""" - num_embedding_windows = embeddings.shape[2] - - # Shift - self.embeddings[:-num_embedding_windows] = self.embeddings[ - num_embedding_windows: - ] - - # Overwrite - self.embeddings[-num_embedding_windows:] = embeddings[0, 0, :, :] - self.new_embeddings = min( - len(self.embeddings), - self.new_embeddings + num_embedding_windows, - ) - - while self.new_embeddings >= self.input_windows: - emb_tensor = np.zeros( - shape=(1, self.input_windows, WW_FEATURES), - dtype=np.float32, - ) - - emb_tensor[0, :] = self.embeddings[ - -self.new_embeddings : len(self.embeddings) - - self.new_embeddings - + self.input_windows - ] - self.new_embeddings = max(0, self.new_embeddings - 1) - - # Run inference - emb_ptr = emb_tensor.ctypes.data_as(c_void_p) - self.lib.TfLiteTensorCopyFromBuffer( - self.input_tensor, emb_ptr, c_size_t(emb_tensor.nbytes) - ) - self.lib.TfLiteInterpreterInvoke(self.interpreter) - - output_bytes = self.lib.TfLiteTensorByteSize(self.output_tensor) - probs = np.empty( - output_bytes // np.dtype(np.float32).itemsize, dtype=np.float32 - ) - self.lib.TfLiteTensorCopyToBuffer( - self.output_tensor, - probs.ctypes.data_as(c_void_p), - c_size_t(output_bytes), - ) - - yield probs.item() - - @staticmethod - def from_config( - config_path: Union[str, Path], - libtensorflowlite_c_path: Union[str, Path], - ) -> "OpenWakeWord": - config_path = Path(config_path) - with open(config_path, "r", encoding="utf-8") as config_file: - config = json.load(config_file) - - return OpenWakeWord( - id=Path(config["model"]).stem, - wake_word=config["wake_word"], - tflite_model=config_path.parent / config["model"], - libtensorflowlite_c_path=libtensorflowlite_c_path, - ) - - -# ----------------------------------------------------------------------------- - - -class OpenWakeWordFeatures(TfLiteWakeWord): - def __init__( - self, - melspectrogram_model: Union[str, Path], - embedding_model: Union[str, Path], - libtensorflowlite_c_path: Union[str, Path], - ) -> None: - TfLiteWakeWord.__init__(self, libtensorflowlite_c_path) - - # Melspectrogram - mel_path = str(Path(melspectrogram_model).resolve()).encode("utf-8") - self.mel_model = self.lib.TfLiteModelCreateFromFile(mel_path) - self.mel_interpreter = self.lib.TfLiteInterpreterCreate(self.mel_model, None) - - mels_dims = (c_int32 * len(MEL_SHAPE))(*MEL_SHAPE) - self.lib.TfLiteInterpreterResizeInputTensor( - self.mel_interpreter, - c_int32(0), - mels_dims, - c_int32(len(MEL_SHAPE)), - ) - self.lib.TfLiteInterpreterAllocateTensors(self.mel_interpreter) - self.mel_input_tensor = self.lib.TfLiteInterpreterGetInputTensor( - self.mel_interpreter, c_int32(0) - ) - self.mel_output_tensor = self.lib.TfLiteInterpreterGetOutputTensor( - self.mel_interpreter, c_int32(0) - ) - - # Embedding - emb_path = str(Path(embedding_model).resolve()).encode("utf-8") - self.emb_model = self.lib.TfLiteModelCreateFromFile(emb_path) - self.emb_interpreter = self.lib.TfLiteInterpreterCreate(self.emb_model, None) - emb_dims = (c_int32 * len(EMB_SHAPE))(*EMB_SHAPE) - self.lib.TfLiteInterpreterResizeInputTensor( - self.emb_interpreter, - c_int32(0), - emb_dims, - c_int32(len(EMB_SHAPE)), - ) - self.lib.TfLiteInterpreterAllocateTensors(self.emb_interpreter) - self.emb_input_tensor = self.lib.TfLiteInterpreterGetInputTensor( - self.emb_interpreter, c_int32(0) - ) - self.emb_output_tensor = self.lib.TfLiteInterpreterGetOutputTensor( - self.emb_interpreter, c_int32(0) - ) - - # State - self.new_audio_samples: int = AUTOFILL_SECONDS * SAMPLE_RATE - self.audio: np.ndarray = np.zeros(shape=(_MAX_SAMPLES,), dtype=np.float32) - self.new_mels: int = 0 - self.mels: np.ndarray = np.zeros(shape=(MAX_MELS, NUM_MELS), dtype=np.float32) - - def process_streaming(self, audio_chunk: bytes) -> Iterable[np.ndarray]: - """Generate embeddings from audio.""" - chunk_array = np.frombuffer(audio_chunk, dtype=np.int16).astype(np.float32) - - # Shift samples left - self.audio[: -len(chunk_array)] = self.audio[len(chunk_array) :] - - # Add new samples to end - self.audio[-len(chunk_array) :] = chunk_array - self.new_audio_samples = min( - len(self.audio), - self.new_audio_samples + len(chunk_array), - ) - - while self.new_audio_samples >= MEL_SAMPLES: - audio_tensor = np.zeros(shape=(BATCH_SIZE, MEL_SAMPLES), dtype=np.float32) - audio_tensor[0, :] = self.audio[ - -self.new_audio_samples : len(self.audio) - - self.new_audio_samples - + MEL_SAMPLES - ] - audio_tensor = np.ascontiguousarray(audio_tensor) - self.new_audio_samples = max(0, self.new_audio_samples - SAMPLES_PER_CHUNK) - - # Run inference (mels) - audio_ptr = audio_tensor.ctypes.data_as(c_void_p) - self.lib.TfLiteTensorCopyFromBuffer( - self.mel_input_tensor, audio_ptr, c_size_t(audio_tensor.nbytes) - ) - self.lib.TfLiteInterpreterInvoke(self.mel_interpreter) - - mels_output_bytes = self.lib.TfLiteTensorByteSize(self.mel_output_tensor) - mels = np.empty( - mels_output_bytes // np.dtype(np.float32).itemsize, dtype=np.float32 - ) - self.lib.TfLiteTensorCopyToBuffer( - self.mel_output_tensor, - mels.ctypes.data_as(c_void_p), - c_size_t(mels_output_bytes), - ) - - mels = (mels / 10) + 2 # transform to fit embedding - mels = mels.reshape((1, 1, -1, NUM_MELS)) - - # Shift left - num_mel_windows = mels.shape[2] - self.mels[:-num_mel_windows] = self.mels[num_mel_windows:] - - # Overwrite - self.mels[-num_mel_windows:] = mels[0, 0, :, :] - self.new_mels = min(len(self.mels), self.new_mels + num_mel_windows) - - while self.new_mels >= EMB_FEATURES: - mels_tensor = np.ascontiguousarray( - np.zeros(shape=EMB_SHAPE, dtype=np.float32) - ) - mels_tensor[0, :, :, 0] = self.mels[ - -self.new_mels : len(self.mels) - self.new_mels + EMB_FEATURES, : - ] - self.new_mels = max(0, self.new_mels - EMB_STEP) - - # Run inference (embedding) - mels_ptr = mels_tensor.ctypes.data_as(c_void_p) - self.lib.TfLiteTensorCopyFromBuffer( - self.emb_input_tensor, mels_ptr, c_size_t(mels_tensor.nbytes) - ) - self.lib.TfLiteInterpreterInvoke(self.emb_interpreter) - - emb_output_bytes = self.lib.TfLiteTensorByteSize(self.emb_output_tensor) - emb = np.empty( - emb_output_bytes // np.dtype(np.float32).itemsize, dtype=np.float32 - ) - self.lib.TfLiteTensorCopyToBuffer( - self.emb_output_tensor, - emb.ctypes.data_as(c_void_p), - c_size_t(emb_output_bytes), - ) - emb = emb.reshape((1, 1, -1, WW_FEATURES)) - yield emb diff --git a/linux_voice_assistant/satellite.py b/linux_voice_assistant/satellite.py index 000c93e..3fe1ccc 100644 --- a/linux_voice_assistant/satellite.py +++ b/linux_voice_assistant/satellite.py @@ -1,12 +1,18 @@ """Voice satellite protocol.""" +import hashlib import logging +import posixpath +import shutil import time from collections.abc import Iterable +from pathlib import Path from typing import Dict, Optional, Set, Union +from urllib.parse import urlparse, urlunparse +from urllib.request import urlopen # pylint: disable=no-name-in-module -from aioesphomeapi.api_pb2 import ( # type: ignore[attr-defined] +from aioesphomeapi.api_pb2 import ( DeviceInfoRequest, DeviceInfoResponse, ListEntitiesDoneResponse, @@ -19,6 +25,7 @@ from aioesphomeapi.api_pb2 import ( # type: ignore[attr-defined] VoiceAssistantConfigurationRequest, VoiceAssistantConfigurationResponse, VoiceAssistantEventResponse, + VoiceAssistantExternalWakeWord, # <-- ADDED VoiceAssistantRequest, VoiceAssistantSetConfiguration, VoiceAssistantTimerEventResponse, @@ -30,12 +37,12 @@ from aioesphomeapi.model import ( VoiceAssistantTimerEventType, ) from google.protobuf import message +from pymicro_wakeword import MicroWakeWord +from pyopen_wakeword import OpenWakeWord from .api_server import APIServer from .entity import MediaPlayerEntity -from .microwakeword import MicroWakeWord -from .models import ServerState, SatelliteState -from .openwakeword import OpenWakeWord +from .models import AvailableWakeWord, ServerState, SatelliteState, WakeWordType # <-- ADDED from .util import call_all _LOGGER = logging.getLogger(__name__) @@ -60,8 +67,8 @@ class VoiceSatelliteProtocol(APIServer): self.state.entities.append(self.media_player_entity) # State machine - self._state: SatelliteState = SatelliteState.STARTING # <-- MODIFIED - self._is_streaming_audio: bool = False # Is HA requesting audio? + self._state: SatelliteState = SatelliteState.STARTING + self._is_streaming_audio: bool = False # Announce/TTS state self._tts_url: Optional[str] = None @@ -71,19 +78,19 @@ class VoiceSatelliteProtocol(APIServer): # Flags set by events, processed in _determine_final_state self._run_end_received: bool = False self._tts_end_received: bool = False + + self._external_wake_words: Dict[str, VoiceAssistantExternalWakeWord] = {} # <-- ADDED def _set_state(self, new_state: SatelliteState): - """Manages state transitions and publishes events.""" if self._state == new_state: return _LOGGER.debug(f"State transition: {self._state} -> {new_state}") self._state = new_state - # Publish events for LEDs, etc. if new_state == SatelliteState.IDLE: self.unduck() - self.state.stop_word.is_active = False + self.state.active_wake_words.discard(self.state.stop_word.id) self.state.event_bus.publish("voice_idle") elif new_state == SatelliteState.LISTENING: self.duck() @@ -91,7 +98,7 @@ class VoiceSatelliteProtocol(APIServer): elif new_state == SatelliteState.THINKING: self.state.event_bus.publish("voice_thinking") elif new_state == SatelliteState.RESPONDING: - self.state.stop_word.is_active = True + self.state.active_wake_words.add(self.state.stop_word.id) self.state.event_bus.publish("voice_responding") elif new_state == SatelliteState.ERROR: self.state.event_bus.publish("voice_error") @@ -106,10 +113,8 @@ class VoiceSatelliteProtocol(APIServer): self._tts_end_received = False self._tts_url = data.get("url") self._continue_conversation = False - # DO NOT transition to THINKING here. Stay in LISTENING. elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_STT_START: - # This confirms we are in the LISTENING state self._set_state(SatelliteState.LISTENING) elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_STT_VAD_START: @@ -117,11 +122,9 @@ class VoiceSatelliteProtocol(APIServer): elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_STT_END: self._is_streaming_audio = False - # THIS is when we move to THINKING self._set_state(SatelliteState.THINKING) elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_STT_VAD_END: - # We wait for STT_END before moving to THINKING pass elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_INTENT_END: @@ -134,17 +137,15 @@ class VoiceSatelliteProtocol(APIServer): elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_TTS_END: self._tts_url = data.get("url") self._tts_end_received = True - self.play_tts() # Play the final TTS URL + self.play_tts() elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_RUN_END: self._run_end_received = True - # If TTS is already done, or we're not speaking, we can finalize. if self._state != SatelliteState.RESPONDING: self._determine_final_state() elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_ERROR: self._set_state(SatelliteState.ERROR) - # Use a timer to reset to idle after error self.state.loop.call_later(5.0, self._set_state, SatelliteState.IDLE) @@ -156,7 +157,7 @@ class VoiceSatelliteProtocol(APIServer): _LOGGER.debug("Timer event: type=%s", event_type.name) if event_type == VoiceAssistantTimerEventType.VOICE_ASSISTANT_TIMER_FINISHED: if not self._timer_finished: - self.state.stop_word.is_active = True + self.state.active_wake_words.add(self.state.stop_word.id) self._timer_finished = True self.duck() self._play_timer_finished() @@ -173,10 +174,10 @@ class VoiceSatelliteProtocol(APIServer): if msg.preannounce_media_id: urls.append(msg.preannounce_media_id) urls.append(msg.media_id) - self.state.stop_word.is_active = True + self.state.active_wake_words.add(self.state.stop_word.id) self._continue_conversation = msg.start_conversation self.duck() - self._set_state(SatelliteState.RESPONDING) # Announce is a response + self._set_state(SatelliteState.RESPONDING) yield from self.media_player_entity.play( urls, announcement=True, done_callback=self._tts_finished ) @@ -209,22 +210,44 @@ class VoiceSatelliteProtocol(APIServer): if isinstance(msg, ListEntitiesRequest): yield ListEntitiesDoneResponse() elif isinstance(msg, VoiceAssistantConfigurationRequest): - yield VoiceAssistantConfigurationResponse( - available_wake_words=[ + # --- MODIFIED --- + available_wake_words = [ + VoiceAssistantWakeWord( + id=ww.id, + wake_word=ww.wake_word, + trained_languages=ww.trained_languages, + ) + for ww in self.state.available_wake_words.values() + ] + + # Add external wake words + self._external_wake_words.clear() + for eww in msg.external_wake_words: + if eww.model_type != "micro": + _LOGGER.warning("Skipping external wake word %s (type=%s)", eww.id, eww.model_type) + continue + + available_wake_words.append( VoiceAssistantWakeWord( - id=ww.id, - wake_word=ww.wake_word, - trained_languages=ww.trained_languages, + id=eww.id, + wake_word=eww.wake_word, + trained_languages=eww.trained_languages, ) - for ww in self.state.available_wake_words.values() - ], + ) + self._external_wake_words[eww.id] = eww + + yield VoiceAssistantConfigurationResponse( + available_wake_words=available_wake_words, active_wake_words=[ - ww.id for ww in self.state.wake_words.values() if ww.is_active + ww.id for ww in self.state.wake_words.values() + if ww.id in self.state.active_wake_words ], max_active_wake_words=2, ) + # --- END MODIFIED --- + _LOGGER.info("Connected to Home Assistant") - self._set_state(SatelliteState.IDLE) # Connected, move to idle + self._set_state(SatelliteState.IDLE) self.state.event_bus.publish("ha_connected") elif isinstance(msg, VoiceAssistantSetConfiguration): active_wake_words: Set[str] = set() @@ -232,19 +255,34 @@ class VoiceSatelliteProtocol(APIServer): if wake_word_id in self.state.wake_words: active_wake_words.add(wake_word_id) continue + model_info = self.state.available_wake_words.get(wake_word_id) + + # --- ADDED --- if not model_info: - continue - _LOGGER.debug("Loading wake word: %s", model_info.config_path) - self.state.wake_words[wake_word_id] = model_info.load( - self.state.libtensorflowlite_c_path - ) + # Check external wake words (may require download) + external_wake_word = self._external_wake_words.get(wake_word_id) + if not external_wake_word: + _LOGGER.warning("Unknown wake word: %s", wake_word_id) + continue + + model_info = self._download_external_wake_word(external_wake_word) + if not model_info: + continue # download failed + + self.state.available_wake_words[wake_word_id] = model_info + # --- END ADDED --- + + _LOGGER.debug("Loading wake word: %s", model_info.wake_word_path) + self.state.wake_words[wake_word_id] = model_info.load() + _LOGGER.info("Wake word set: %s", wake_word_id) active_wake_words.add(wake_word_id) break - for wake_word in self.state.wake_words.values(): - wake_word.is_active = wake_word.id in active_wake_words + + self.state.active_wake_words = active_wake_words _LOGGER.debug("Active wake words: %s", active_wake_words) + self.state.preferences.active_wake_words = list(active_wake_words) self.state.save_preferences() self.state.wake_words_changed = True @@ -256,7 +294,7 @@ class VoiceSatelliteProtocol(APIServer): def wakeup(self, wake_word: Union[MicroWakeWord, OpenWakeWord]) -> None: if self._state not in (SatelliteState.IDLE, SatelliteState.STARTING): - return # Already awake + return if self._timer_finished: self._timer_finished = False @@ -264,7 +302,7 @@ class VoiceSatelliteProtocol(APIServer): _LOGGER.debug("Stopping timer finished sound") return - wake_word_phrase = wake_word.wake_word + wake_word_phrase = getattr(wake_word, "wake_word", "wake word") _LOGGER.debug("Detected wake word: %s", wake_word_phrase) self.send_messages( [VoiceAssistantRequest(start=True, wake_word_phrase=wake_word_phrase)] @@ -274,22 +312,23 @@ class VoiceSatelliteProtocol(APIServer): self.state.tts_player.play(self.state.wakeup_sound) def stop(self) -> None: + self.state.active_wake_words.discard(self.state.stop_word.id) self.state.tts_player.stop() if self._timer_finished: self._timer_finished = False _LOGGER.debug("Stopping timer finished sound") else: _LOGGER.debug("TTS response stopped manually") - self._tts_finished() # Manually trigger TTS finished logic + self._tts_finished() def play_tts(self) -> None: if not self._tts_url: return _LOGGER.debug("Playing TTS response: %s", self._tts_url) - self._set_state(SatelliteState.RESPONDING) + self.state.active_wake_words.add(self.state.stop_word.id) self.state.tts_player.play(self._tts_url, done_callback=self._tts_finished) - self._tts_url = None # Clear after starting playback + self._tts_url = None def duck(self) -> None: _LOGGER.debug("Ducking music") @@ -300,7 +339,6 @@ class VoiceSatelliteProtocol(APIServer): self.state.music_player.unduck() def _determine_final_state(self) -> None: - """Called when both TTS and the pipeline run are finished.""" if self._continue_conversation: self.send_messages([VoiceAssistantRequest(start=True)]) self._is_streaming_audio = True @@ -312,18 +350,13 @@ class VoiceSatelliteProtocol(APIServer): _LOGGER.debug("Final state determined") def _tts_finished(self) -> None: - """Callback when TTS audio finishes playing.""" + self.state.active_wake_words.discard(self.state.stop_word.id) self.send_messages([VoiceAssistantAnnounceFinished()]) _LOGGER.debug("TTS audio playback finished") - # If the run already ended, we can move to the final state. - # Otherwise, we wait for the RUN_END event. if self._run_end_received: self._determine_final_state() else: - # We've finished speaking but the pipeline is still running - # (e.g., waiting for 'continue_conversation') - # Move to THINKING state if self._state == SatelliteState.RESPONDING: self._set_state(SatelliteState.THINKING) @@ -339,6 +372,75 @@ class VoiceSatelliteProtocol(APIServer): ), ) + # --- ADDED --- + def _download_external_wake_word( + self, external_wake_word: VoiceAssistantExternalWakeWord + ) -> Optional[AvailableWakeWord]: + eww_dir = self.state.download_dir / "external_wake_words" + eww_dir.mkdir(parents=True, exist_ok=True) + + config_path = eww_dir / f"{external_wake_word.id}.json" + should_download_config = not config_path.exists() + + # Check if we need to download the model file + model_path = eww_dir / f"{external_wake_word.id}.tflite" + should_download_model = True + if model_path.exists(): + model_size = model_path.stat().st_size + if model_size == external_wake_word.model_size: + with open(model_path, "rb") as model_file: + model_hash = hashlib.sha256(model_file.read()).hexdigest() + + if model_hash == external_wake_word.model_hash: + should_download_model = False + _LOGGER.debug( + "Model size and hash match for %s. Skipping download.", + external_wake_word.id, + ) + + if should_download_config or should_download_model: + # Download config + _LOGGER.debug("Downloading %s to %s", external_wake_word.url, config_path) + with urlopen(external_wake_word.url) as request: + if request.status != 200: + _LOGGER.warning( + "Failed to download: %s, status=%s", + external_wake_word.url, + request.status, + ) + return None + + with open(config_path, "wb") as model_file: + shutil.copyfileobj(request, model_file) + + if should_download_model: + # Download model file + parsed_url = urlparse(external_wake_word.url) + parsed_url = parsed_url._replace( + path=posixpath.join(posixpath.dirname(parsed_url.path), model_path.name) + ) + model_url = urlunparse(parsed_url) + + _LOGGER.debug("Downloading %s to %s", model_url, model_path) + with urlopen(model_url) as request: + if request.status != 200: + _LOGGER.warning( + "Failed to download: %s, status=%s", model_url, request.status + ) + return None + + with open(model_path, "wb") as model_file: + shutil.copyfileobj(request, model_file) + + return AvailableWakeWord( + id=external_wake_word.id, + type=WakeWordType.MICRO_WAKE_WORD, + wake_word=external_wake_word.wake_word, + trained_languages=external_wake_word.trained_languages, + wake_word_path=config_path, # micro-wakeword loads from config + ) + # --- END ADDED --- + def connection_lost(self, exc): super().connection_lost(exc) self._set_state(SatelliteState.ERROR) diff --git a/linux_voice_assistant/util.py b/linux_voice_assistant/util.py index 92726ef..31e184b 100644 --- a/linux_voice_assistant/util.py +++ b/linux_voice_assistant/util.py @@ -1,22 +1,17 @@ """Utility methods.""" -import platform import uuid from collections.abc import Callable from typing import Optional def get_mac() -> str: - mac = uuid.getnode() - mac_str = ":".join(f"{(mac >> i) & 0xff:02x}" for i in range(40, -1, -8)) - return mac_str + """Get MAC address as a string.""" + return ":".join( + f"{((uuid.getnode() >> i) & 0xFF):02x}" for i in range(0, 8 * 6, 8) + )[::-1] def call_all(*callables: Optional[Callable[[], None]]) -> None: for item in filter(None, callables): item() - - -def is_arm() -> bool: - machine = platform.machine() - return ("arm" in machine) or ("aarch" in machine) diff --git a/linux_voice_assistant/wakeword.py b/linux_voice_assistant/wakeword.py deleted file mode 100644 index ae9f40e..0000000 --- a/linux_voice_assistant/wakeword.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Base class for wake words.""" - -import ctypes as C -import os -from pathlib import Path -from typing import Union - -TfLiteStatus = C.c_int # kTfLiteOk == 0 - - -class TfLiteQuantizationParams(C.Structure): - _fields_ = [("scale", C.c_float), ("zero_point", C.c_int32)] - - -c_void_p = C.c_void_p -c_int32 = C.c_int32 -c_size_t = C.c_size_t - - -def _not_null(result, func, args): - if not result: - raise RuntimeError(f"{func.__name__} returned NULL") - return result - - -class TfLiteWakeWord: - def __init__(self, libtensorflowlite_c_path: Union[str, Path]): - self.libtensorflowlite_c_path = Path(libtensorflowlite_c_path) - # Load ONCE with RTLD_GLOBAL - self.lib = C.CDLL( - str(self.libtensorflowlite_c_path.resolve()), mode=os.RTLD_GLOBAL - ) - - lib = self.lib - - # Model / Interpreter - lib.TfLiteModelCreateFromFile.argtypes = [C.c_char_p] - lib.TfLiteModelCreateFromFile.restype = c_void_p - - lib.TfLiteInterpreterCreate.argtypes = [c_void_p, c_void_p] - lib.TfLiteInterpreterCreate.restype = c_void_p - - lib.TfLiteInterpreterAllocateTensors.argtypes = [c_void_p] - lib.TfLiteInterpreterAllocateTensors.restype = TfLiteStatus - - lib.TfLiteInterpreterInvoke.argtypes = [c_void_p] - lib.TfLiteInterpreterInvoke.restype = TfLiteStatus - - # Tensors: get - lib.TfLiteInterpreterGetInputTensor.argtypes = [c_void_p, c_int32] - lib.TfLiteInterpreterGetInputTensor.restype = c_void_p - lib.TfLiteInterpreterGetInputTensor.errcheck = _not_null - - lib.TfLiteInterpreterGetOutputTensor.argtypes = [c_void_p, c_int32] - lib.TfLiteInterpreterGetOutputTensor.restype = c_void_p - lib.TfLiteInterpreterGetOutputTensor.errcheck = _not_null - - # Sizes / dims - lib.TfLiteTensorByteSize.argtypes = [c_void_p] - lib.TfLiteTensorByteSize.restype = c_size_t - - lib.TfLiteTensorNumDims.argtypes = [c_void_p] - lib.TfLiteTensorNumDims.restype = c_int32 - - lib.TfLiteTensorDim.argtypes = [c_void_p, c_int32] - lib.TfLiteTensorDim.restype = c_int32 - - # Type & quant params - lib.TfLiteTensorType.argtypes = [c_void_p] - lib.TfLiteTensorType.restype = c_int32 - - lib.TfLiteTensorQuantizationParams.argtypes = [c_void_p] - lib.TfLiteTensorQuantizationParams.restype = TfLiteQuantizationParams - - # Resize - lib.TfLiteInterpreterResizeInputTensor.argtypes = [ - c_void_p, - c_int32, - C.POINTER(c_int32), - c_int32, - ] - lib.TfLiteInterpreterResizeInputTensor.restype = TfLiteStatus - - # Copy buffers - lib.TfLiteTensorCopyFromBuffer.argtypes = [c_void_p, c_void_p, c_size_t] - lib.TfLiteTensorCopyFromBuffer.restype = TfLiteStatus - - lib.TfLiteTensorCopyToBuffer.argtypes = [c_void_p, c_void_p, c_size_t] - lib.TfLiteTensorCopyToBuffer.restype = TfLiteStatus diff --git a/mypy.ini b/mypy.ini index 7e5ee40..a6e2d1d 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,23 +1,52 @@ - [mypy] +python_version = 3.9 +warn_return_any = True +warn_unused_configs = True -[mypy-setuptools.*] +[mypy-aioesphomeapi._frame_helper.packets.*] ignore_missing_imports = True [mypy-aioesphomeapi.api_pb2.*] ignore_missing_imports = True -[mypy-pymicro_features.*] +[mypy-pymicro_wakeword.*] ignore_missing_imports = True -[mypy-sounddevice.*] +[mypy-pyopen_wakeword.*] +ignore_missing_imports = True + +[mypy-soundcard.*] ignore_missing_imports = True [mypy-mpv.*] ignore_missing_imports = True -[mypy-google.protobuf.*] +[mypy-zeroconf.*] ignore_missing_imports = True -[mypy-aioesphomeapi.*] +[mypy-paho.mqtt.client.*] +ignore_missing_imports = True + +[mypy-RPi.*] +ignore_missing_imports = True + +[mypy-spidev.*] +ignore_missing_imports = True + +[mypy-board.*] +ignore_missing_imports = True + +[mypy-busio.*] +ignore_missing_imports = True + +[mypy-gpiozero.*] +ignore_missing_imports = True + +[mypy-adafruit_dotstar.*] +ignore_missing_imports = True + +[mypy-neopixel.*] +ignore_missing_imports = True + +[mypy-neopixel_spi.*] ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml index 661134e..63e76f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,11 +26,12 @@ classifiers = [ requires-python = ">=3.9.0" dependencies = [ "aioesphomeapi==42.7.0", - "sounddevice<1", + "soundcard<1", "numpy>=2,<3", - "pymicro-features==1.0.0", + "pymicro-wakeword>=2,<3", + "pyopen-wakeword>=1,<2", "python-mpv>=1,<2", - "zeroconf", + "zeroconf<1", "paho-mqtt", "adafruit-circuitpython-dotstar", "adafruit-circuitpython-neopixel", @@ -42,15 +43,15 @@ dependencies = [ [project.optional-dependencies] dev = [ - "black==24.8.0", - "flake8==7.2.0", - "mypy==1.14.0", - "pylint==3.2.7", - "pytest==8.3.5", + "black", + "flake8", + "mypy", + "pylint", + "pytest", ] [project.urls] -"Source Code" = "http://github.com/OHF-Voice/linux-voice-assistant" +"Source Code" = "https://github.com/imonlinux/linux-voice-assistant" [tool.setuptools] platforms = ["any"] diff --git a/wakewords/openWakeWord/embedding_model.tflite b/wakewords/openWakeWord/embedding_model.tflite deleted file mode 100644 index d3c3bed..0000000 Binary files a/wakewords/openWakeWord/embedding_model.tflite and /dev/null differ diff --git a/wakewords/openWakeWord/melspectrogram.tflite b/wakewords/openWakeWord/melspectrogram.tflite deleted file mode 100644 index e290399..0000000 Binary files a/wakewords/openWakeWord/melspectrogram.tflite and /dev/null differ