diff --git a/linux_voice_assistant/__main__.py b/linux_voice_assistant/__main__.py index a4c3752..b2cc69c 100644 --- a/linux_voice_assistant/__main__.py +++ b/linux_voice_assistant/__main__.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse +import os import asyncio import json import logging @@ -8,10 +9,26 @@ import threading import time from collections.abc import Iterable from dataclasses import dataclass -from pathlib import Path from queue import Queue from typing import Dict, List, Optional +# Models that should NOT appear in UI dropdowns +EXCLUDED_OWW_MODELS = {"embedding_model", "melspectrogram"} + +import re + +def _prettify_model_name(name: str) -> str: + stem = Path(name).stem + stem = re.sub(r"(?:_v\d+(?:\.\d+)?)$", "", stem) + parts = stem.split("_") + if parts and parts[0].lower() == "ok": + parts[0] = "OK" + else: + parts = [p.capitalize() for p in parts] + return " ".join(parts) + + + # pylint: disable=no-name-in-module import sounddevice as sd from aioesphomeapi.api_pb2 import ( # type: ignore[attr-defined] @@ -48,6 +65,17 @@ from .util import call_all, get_mac, is_arm from .event_bus import EventBus from .event_led import LedEvent +from pathlib import Path + +# Hide helper models that are not detectors +_OWW_DETECT_MODEL_DENYLIST = {"embedding_model", "melspectrogram"} + + +def _oww_filter_models(models: list[str]) -> list[str]: + """Normalize names (strip .tflite/paths), drop helper models, sort & de-dupe.""" + stems = {Path(m).stem for m in models} + return sorted(m for m in stems if m not in _OWW_DETECT_MODEL_DENYLIST) + _LOGGER = logging.getLogger(__name__) _MODULE_DIR = Path(__file__).parent @@ -86,7 +114,8 @@ class ServerState: event_bus: EventBus media_player_entity: Optional[MediaPlayerEntity] = None satellite: "Optional[VoiceSatelliteProtocol]" = None - wyoming_wake: Optional[WyomingWakeClient] = None + current_remote_wake: Optional[str] = None + wyoming_wake: "Optional[WyomingWakeClient]" = None use_wyoming_wake: bool = False @@ -231,7 +260,7 @@ class VoiceSatelliteProtocol(APIServer): ) for ww in self.state.available_wake_words.values() ], - active_wake_words=[self.state.wake_word.id], + active_wake_words=[self.state.current_remote_wake] if self.state.use_wyoming_wake else [self.state.wake_word.id], max_active_wake_words=1, ) _LOGGER.info("Connected to Home Assistant") @@ -242,6 +271,15 @@ class VoiceSatelliteProtocol(APIServer): # Already active break + if self.state.use_wyoming_wake and self.state.wyoming_wake: + try: + self.state.wyoming_wake.set_detect([wake_word_id]) + _LOGGER.info("Switched Wyoming wake model to: %s", wake_word_id) + self.state.current_remote_wake = wake_word_id + self.state.wake_word.is_active = False + except Exception: + _LOGGER.exception("Failed to switch Wyoming wake model") + break model_info = self.state.available_wake_words.get(wake_word_id) if not model_info: continue @@ -270,8 +308,12 @@ class VoiceSatelliteProtocol(APIServer): _LOGGER.debug("Stopping timer finished sound") return - wake_word_phrase = self.state.wake_word.wake_word + wake_word_phrase = (self.state.current_remote_wake or self.state.wake_word.wake_word) if self.state.use_wyoming_wake else self.state.wake_word.wake_word _LOGGER.debug("Detected wake word: %s", wake_word_phrase) + try: + self.state.wyoming_wake.suppress(2500) + except Exception: + pass self.send_messages( [VoiceAssistantRequest(start=True, wake_word_phrase=wake_word_phrase)] ) @@ -348,9 +390,6 @@ class VoiceSatelliteProtocol(APIServer): def process_audio(state: ServerState): - # debug counters - chunks_sent = 0 - last_log = 0.0 try: while True: @@ -363,9 +402,6 @@ def process_audio(state: ServerState): try: state.satellite.handle_audio(audio_chunk) - chunks_sent += 1 - if state.use_wyoming_wake and chunks_sent % 100 == 0: - _LOGGER.debug("[OWW] chunks_sent=%d (approx %.1fs of audio)", chunks_sent, chunks_sent*0.064) if state.use_wyoming_wake: if state.wyoming_wake: @@ -395,6 +431,7 @@ async def main() -> None: parser.add_argument("--name", required=True) parser.add_argument("--wake-uri", help="Wyoming wake server URI (e.g., tcp://127.0.0.1:10400)") parser.add_argument("--wake-word-name", help="Wake word name on the Wyoming server (e.g., hal)") + parser.add_argument("--wake-models-dir", help="Fallback: directory containing openWakeWord *.tflite models to list if the server does not advertise them") parser.add_argument( "--audio-input-device", default="default", @@ -433,11 +470,14 @@ async def main() -> None: logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO) _LOGGER.debug(args) - use_wyoming = bool(args.wake_uri and args.wake_word_name) + # Optional fallback for discovering remote models from filesystem + models_dir = args.wake_models_dir or os.environ.get("WYOMING_MODELS_DIR") + + use_wyoming = bool(args.wake_uri) wyoming_client = None if use_wyoming: _LOGGER.info("Using Wyoming openWakeWord at %s (name=%s)", args.wake_uri, args.wake_word_name) - wyoming_client = WyomingWakeClient(args.wake_uri, args.wake_word_name) + wyoming_client = WyomingWakeClient(args.wake_uri, args.wake_word_name or "") # Load available wake words wake_word_dir = Path(args.wake_word_dir) @@ -459,6 +499,28 @@ async def main() -> None: _LOGGER.debug("Available wake words: %s", list(sorted(available_wake_words.keys()))) + if use_wyoming and wyoming_client: + try: + remote_models = wyoming_client.list_models() or [] + _LOGGER.debug("Queried Wyoming models: %s", [m.get("name") if isinstance(m, dict) else m for m in remote_models]) + except Exception: + remote_models = [] + _LOGGER.exception("Failed to list models from Wyoming server") + if remote_models: + available_wake_words = {} + for m in remote_models: + name = (m.get("name") or "").strip() + langs = m.get("languages") or m.get("trained_languages") or [] + if not name: + continue + available_wake_words[name] = AvailableWakeWord( + id=name, + wake_word=(m.get('label') or _prettify_model_name(name)), + trained_languages=langs, + config_path=Path(f"wyoming:{name}"), + ) + _LOGGER.debug("Remote OpenWakeWord models: %s", list(sorted(available_wake_words.keys()))) + libtensorflowlite_c_path = _LIB_DIR / "libtensorflowlite_c.so" _LOGGER.debug("libtensorflowlite_c path: %s", libtensorflowlite_c_path) @@ -496,12 +558,58 @@ async def main() -> None: # Connect to Wyoming wake server if enabled if state.use_wyoming_wake and state.wyoming_wake: def _on_detect(_name, _ts): - _LOGGER.debug("[OWW] detection callback fired name=%s ts=%s", _name, _ts) if state.satellite is not None: state.loop.call_soon_threadsafe(lambda: state.satellite.wakeup()) state.wyoming_wake.connect(_on_detect) + # Disable local wake model to avoid double triggers state.wake_word.is_active = False - _LOGGER.debug("[OWW] Local MicroWakeWord disabled; streaming audio to Wyoming") + # Refresh remote models after connect + try: + cached = state.wyoming_wake.get_models(timeout=1.5) if state.wyoming_wake else [] + if not cached and models_dir: + try: + from pathlib import Path as _P + cached = [{"name": p.stem} for p in _P(models_dir).glob("*.tflite")] + cached = [ + m for m in cached + if (m.get('name') or '').strip() not in EXCLUDED_OWW_MODELS + ] + _LOGGER.info("Using filesystem fallback for OWW models (post-connect): %s", [m["name"] for m in cached]) + except Exception: + _LOGGER.exception("Failed to scan wake-models-dir (post-connect)") + if cached: + available = {} + for m in cached: + name = (m.get('name') or '').strip() + if not name: + continue + available[name] = AvailableWakeWord( + id=name, + wake_word=(m.get('label') or _prettify_model_name(name)), + trained_languages=[], + config_path=Path(f"wyoming:{name}") + ) + state.available_wake_words = available + _LOGGER.debug("Using remote models from cached info: %s", list(sorted(state.available_wake_words.keys()))) + except Exception: + _LOGGER.exception("Failed to refresh remote models from connection info") + # Initialize remote detection target if provided + try: + if args.wake_word_name: + state.wyoming_wake.set_detect([args.wake_word_name]) + state.current_remote_wake = args.wake_word_name + else: + try: + first = next(iter(state.available_wake_words.keys())) + except StopIteration: + first = None + if first: + state.wyoming_wake.set_detect([first]) + state.current_remote_wake = first + else: + _LOGGER.warning("Wyoming enabled but no remote models available; not sending detect yet") + except Exception: + _LOGGER.exception("Failed to initialize remote detect") process_audio_thread = threading.Thread( target=process_audio, args=(state,), daemon=True @@ -538,4 +646,4 @@ async def main() -> None: if __name__ == "__main__": - asyncio.run(main()) + asyncio.run(main()) \ No newline at end of file diff --git a/linux_voice_assistant/openwakeword_client.py b/linux_voice_assistant/openwakeword_client.py index 59c1c50..49a940b 100644 --- a/linux_voice_assistant/openwakeword_client.py +++ b/linux_voice_assistant/openwakeword_client.py @@ -2,43 +2,95 @@ import json import logging import socket import threading -import time -from typing import Callable, Optional +from typing import Callable, Optional, List, Dict, Any from urllib.parse import urlparse +from pathlib import Path +import time + +# Models that should NOT appear in UI dropdowns +EXCLUDED_OWW_MODELS = {"embedding_model", "melspectrogram"} + + +# Models that are not actual wake-word detectors and should never be shown +_OWW_DETECT_MODEL_DENYLIST = {"embedding_model", "melspectrogram"} + +import re + +def _prettify_model_name(name: str) -> str: + """ + Convert 'hey_jarvis_v0.1' -> 'Hey Jarvis', 'ok_nabu_v0.1' -> 'OK Nabu', + 'hal_v2' -> 'Hal', 'marvin_v2' -> 'Marvin'. + """ + stem = Path(name).stem + # Strip common version suffixes + stem = re.sub(r"(?:_v\d+(?:\.\d+)?)$", "", stem) + parts = stem.split("_") + if parts and parts[0].lower() == "ok": + parts[0] = "OK" + else: + parts = [p.capitalize() for p in parts] + return " ".join(parts) + + +def _normalize_model_name(name: str) -> str: + # Accept "foo_v2", "foo_v2.tflite", full paths, etc. + stem = Path(name).stem + return stem + + +def _filter_detect_models(models: list[str]) -> list[str]: + # Remove helper models and de-duplicate + normalized = {_normalize_model_name(m) for m in models} + return sorted(m for m in normalized if m not in _OWW_DETECT_MODEL_DENYLIST) + _LOGGER = logging.getLogger(__name__) + def _jsonl(obj: dict) -> bytes: return (json.dumps(obj, separators=(",", ":")) + "\n").encode("utf-8") + class WyomingWakeClient: - """Minimal client for Wyoming wake detection (openWakeWord) with extra debug.""" + """Minimal client for Wyoming wake detection (openWakeWord).""" def __init__(self, uri: str, name: str, rate=16000, width=2, channels=1): self.uri, self.name = uri, name self.rate, self.width, self.channels = rate, width, channels + self._paused = False + self._suppress_until_ms = 0 + self._refractory_ms = 0 self._sock: Optional[socket.socket] = None self._reader: Optional[threading.Thread] = None self._on_detect: Optional[Callable[[str, Optional[int]], None]] = None self._closed = False - self._chunks = 0 - self._last_log_time = 0.0 + # model discovery cache + self._models_cache: List[Dict[str, Any]] = [] + self._info_event = threading.Event() def connect(self, on_detect: Callable[[str, Optional[int]], None]) -> None: self._on_detect = on_detect parsed = urlparse(self.uri) if parsed.scheme != "tcp": raise ValueError(f"Only tcp:// URIs are supported (got {self.uri})") - host = parsed.hostname or "127.0.0.1" - port = parsed.port or 10400 - _LOGGER.debug("[OWW] Connecting to %s:%s", host, port) - self._sock = socket.create_connection((host, port)) + self._sock = socket.create_connection((parsed.hostname or "127.0.0.1", parsed.port or 10400)) self._sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - self._send({ "type": "audio-start", "data": { "rate": self.rate, "width": self.width, "channels": self.channels }}) - self._arm_detect() + # Start audio stream + self._send({"type": "audio-start", "data": {"rate": self.rate, "width": self.width, "channels": self.channels}}) + # Ask for model info + try: + self._send({"type": "describe"}) + except Exception: + pass + # Set initial detect model + if self.name: + try: + self._send({"type": "detect", "data": {"names": [self.name]}}) + except Exception: + pass self._reader = threading.Thread(target=self._read_loop, daemon=True) self._reader.start() - _LOGGER.info("[OWW] Connected to Wyoming wake server at %s", self.uri) + _LOGGER.debug("Connected to Wyoming wake server at %s", self.uri) def close(self) -> None: self._closed = True @@ -52,93 +104,261 @@ class WyomingWakeClient: finally: self._sock = None - def _arm_detect(self) -> None: - _LOGGER.debug("[OWW] Arming detect for name=%s", self.name) - self._send({"type": "detect", "data": {"names": [self.name]}}) - def send_audio_chunk(self, pcm: bytes, timestamp_ms: Optional[int] = None) -> None: if self._sock is None or self._closed: return - hdr = { - "type": "audio-chunk", - "data": {"rate": self.rate, "width": self.width, "channels": self.channels}, - "payload_length": len(pcm), - } + hdr = {"type": "audio-chunk", + "data": {"rate": self.rate, "width": self.width, "channels": self.channels}, + "payload_length": len(pcm)} if timestamp_ms is not None: hdr["data"]["timestamp"] = timestamp_ms - try: - self._sock.sendall(_jsonl(hdr)) - self._sock.sendall(pcm) - except Exception: - _LOGGER.exception("[OWW] Failed to send audio chunk") - return + self._sock.sendall(_jsonl(hdr)) + self._sock.sendall(pcm) + + + def pause(self) -> None: + self._paused = True + + def resume(self) -> None: + self._paused = False + + def suppress(self, ms: int) -> None: + try: + now_ms = int(time.time() * 1000) + except Exception: + now_ms = 0 + self._suppress_until_ms = max(self._suppress_until_ms, now_ms + int(ms)) + + def set_refractory(self, ms: int) -> None: + self._refractory_ms = max(0, int(ms)) + + + + def list_models(self): + """Query models using a short-lived connection with timeout; accept 'describe' or 'info'.""" + parsed = urlparse(self.uri) + if parsed.scheme != "tcp": + raise ValueError(f"Only tcp:// URIs are supported (got {self.uri})") + host, port = (parsed.hostname or "127.0.0.1", parsed.port or 10400) + s = socket.create_connection((host, port)) + try: + s.settimeout(1.5) + s.sendall(_jsonl({"type": "describe"})) + f = s.makefile("rb") + while True: + try: + line = f.readline() + except Exception: + break + if not line: + break + try: + msg = json.loads(line.decode("utf-8")) + except Exception: + continue + mtype = msg.get("type") + if mtype in ("describe", "info"): + data = msg.get("data", {}) or {} + wake = data.get("wake", {}) or {} + models = wake.get("models") or wake.get("names") or [] + norm: List[Dict[str, Any]] = [] + for m in models: + if isinstance(m, str): + norm.append({"name": m}) + elif isinstance(m, dict): + norm.append(m) + out: List[Dict[str, Any]] = [] + for m in norm: + name = (m.get('name') or '').strip() + if not name or name in EXCLUDED_OWW_MODELS: + continue + phrase = (m.get('phrase') or '').strip() + desc = (m.get('description') or '').strip() + try: + label = phrase or desc or _prettify_model_name(name) # type: ignore[name-defined] + except NameError: + stem = Path(name).stem + stem = re.sub(r"(?:_v\d+(?:\.\d+)?)$", "", stem) + parts = stem.split("_") + if parts and parts[0].lower() == "ok": + parts[0] = "OK" + else: + parts = [p.capitalize() for p in parts] + label = " ".join(parts) + mm = dict(m) + mm['label'] = label + out.append(mm) + return out + finally: + try: + s.close() + except Exception: + pass + return [] + + def get_models(self, timeout: float = 1.5): + """Return cached models after waiting briefly for an 'info'/'describe' update.""" + self._info_event.wait(timeout) + return list(self._models_cache) + + def set_detect(self, names): + if self._sock is None: + raise RuntimeError("Not connected") + filtered = [n for n in list(names) if n] + if not filtered: + _LOGGER.warning("Ignoring empty detect() request (no model names)") + return + self.name = filtered[0] + _LOGGER.debug("Requesting Wyoming detect models: %s", filtered) + self._send({"type": "detect", "data": {"names": filtered}}) - self._chunks += 1 - if (self._chunks % 1000) == 0: - # ~64ms * 1000 = ~64s of audio - _LOGGER.debug("[OWW] Sent %d chunks (~%.1fs)", self._chunks, self._chunks * 0.064) # internals def _send(self, obj: dict) -> None: assert self._sock is not None - try: - self._sock.sendall(_jsonl(obj)) - except Exception: - _LOGGER.exception("[OWW] send failed (%s)", obj.get("type")) - - def _cycle_stream(self) -> None: - """Some servers require restarting the audio stream after a detection.""" - _LOGGER.debug("[OWW] Cycling audio stream") - try: - self._send({"type": "audio-stop"}) - except Exception: - _LOGGER.exception("[OWW] audio-stop failed") - try: - self._send({"type": "audio-start", "data": {"rate": self.rate, "width": self.width, "channels": self.channels}}) - except Exception: - _LOGGER.exception("[OWW] audio-start failed") + self._sock.sendall(_jsonl(obj)) +# --- paste this to replace the whole _read_loop method --- def _read_loop(self) -> None: assert self._sock is not None f = self._sock.makefile("rb") - _LOGGER.debug("[OWW] Reader loop started") - buf = b"" - decoder = json.JSONDecoder() + + buf = "" # text buffer for concatenated JSON objects + + def _yield_objects_from_text(text: str): + """Yield JSON objects from `text` even if they are concatenated like {}{}...""" + start = 0 + depth = 0 + in_str = False + escape = False + for i, ch in enumerate(text): + if in_str: + if escape: + escape = False + elif ch == "\\": + escape = True + elif ch == '"': + in_str = False + else: + if ch == '"': + in_str = True + elif ch == "{": + if depth == 0: + start = i + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + # complete object from start..i + yield text[start : i + 1], i + 1 + # return here with any remainder unparsed + return + while not self._closed: chunk = f.readline() if not chunk: - _LOGGER.debug("[OWW] Reader EOF") break - buf += chunk + try: - # There might be multiple JSON objects in one line; parse until exhausted. - while buf: - s = buf.decode("utf-8") - obj, idx = decoder.raw_decode(s) - buf = s[idx:].lstrip().encode("utf-8") - mtype = obj.get("type") - if mtype and mtype != "detection": - _LOGGER.debug("[OWW] recv %s", mtype) + piece = chunk.decode("utf-8") + except Exception: + try: + _LOGGER.debug("Wyoming raw (non-utf8): %r", chunk[:200]) + except Exception: + pass + continue + + buf += piece + + # Try to peel off as many complete JSON objects as we can + pos = 0 + progressed = True + while progressed and pos < len(buf): + progressed = False + for obj_text, end_pos in _yield_objects_from_text(buf[pos:]): + progressed = True + pos += end_pos + try: + msg = json.loads(obj_text) + except Exception: + try: + _LOGGER.debug("Wyoming raw (bad-json): %r", obj_text[:200]) + except Exception: + pass + continue + + mtype = str(msg.get("type") or "").lower() + try: + _LOGGER.debug("Received Wyoming message type=%s", mtype or "") + if mtype and mtype not in ("info", "describe"): + _LOGGER.debug("Wyoming raw: %r", obj_text[:200]) + except Exception: + pass + + # ----- model info path ----- + if mtype in ("info", "describe"): + data = msg.get("data", {}) or {} + wake = data.get("wake", {}) if isinstance(data, dict) else {} + + models = [] + if isinstance(wake, dict): + if isinstance(wake.get("models"), list): + models = wake.get("models") + elif isinstance(wake.get("names"), list): + models = wake.get("names") + + norm = [] + for m in models: + if isinstance(m, str): + norm.append({"name": m}) + elif isinstance(m, dict): + norm.append(m) + + if norm: + self._models_cache = norm + self._info_event.set() + _LOGGER.debug( + "Wyoming model list updated: %s", + [m.get("name") for m in norm], + ) + continue + + # ----- detection path (accept several shapes) ----- + data = msg.get("data", {}) or {} + is_detection = False if mtype == "detection": - data = obj.get("data", {}) or {} - name = data.get("name") or self.name - ts = data.get("timestamp") - _LOGGER.info("[OWW] Detection: name=%s ts=%s (chunks=%d)", name, ts, self._chunks) + is_detection = True + elif mtype in ("event", "wake", "hotword"): + ev = str(data.get("event") or "").lower() + if ev in ("detection", "wake", "hotword"): + is_detection = True + + if is_detection: + name = ( + data.get("name") + or data.get("model") + or data.get("wakeword") + or self.name + ) + ts = data.get("timestamp") or data.get("time") or data.get("ts") + _LOGGER.debug("Wake detected: %s ts=%s", name, ts) if self._on_detect: try: - self._on_detect(name, ts) + now_ms = int(time.time() * 1000) + if self._paused or (self._suppress_until_ms and now_ms < self._suppress_until_ms): + _LOGGER.debug("Detection suppressed (paused=%s until=%s)", self._paused, getattr(self, '_suppress_until_ms', 0)) + else: + now_ms = int(time.time() * 1000) + if self._paused or (self._suppress_until_ms and now_ms < self._suppress_until_ms): + _LOGGER.debug("Detection suppressed (paused=%s until=%s)", self._paused, getattr(self, '_suppress_until_ms', 0)) + else: + self._on_detect(name, ts) + if getattr(self, '_refractory_ms', 0): + self.suppress(self._refractory_ms) + if getattr(self, '_refractory_ms', 0): + self.suppress(self._refractory_ms) except Exception: - _LOGGER.exception("[OWW] Error in detection callback") - # Some servers need an audio stream restart before re-arming - try: - self._cycle_stream() - except Exception: - _LOGGER.exception("[OWW] Failed to cycle stream") - # Re-arm detect so subsequent detections work - try: - self._arm_detect() - except Exception: - _LOGGER.exception("[OWW] Failed to re-arm detect") - except json.JSONDecodeError: - # Need more bytes; continue accumulating - pass + _LOGGER.exception("Error in detection callback") + + # keep any unconsumed remainder in buffer + buf = buf[pos:] \ No newline at end of file