sirmrak/linux-voice-assistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add files via upload

Browse Source
This commit is contained in:
imonlinux
2025-09-11 20:45:18 -07:00
committed by GitHub
parent e72635978f
commit 07eb4ed3bf
2 changed files with 422 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
linux_voice_assistant/__main__.py
+122 -14
View File
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
import argparse
import os
import asyncio
import json
import logging
@@ -8,10 +9,26 @@ import threading
import time
from collections.abc import Iterable
from dataclasses import dataclass
from pathlib import Path
from queue import Queue
from typing import Dict, List, Optional
# Models that should NOT appear in UI dropdowns
EXCLUDED_OWW_MODELS = {"embedding_model", "melspectrogram"}
import re
def _prettify_model_name(name: str) -> str:
stem = Path(name).stem
stem = re.sub(r"(?:_v\d+(?:\.\d+)?)$", "", stem)
parts = stem.split("_")
if parts and parts[0].lower() == "ok":
parts[0] = "OK"
else:
parts = [p.capitalize() for p in parts]
return " ".join(parts)
# pylint: disable=no-name-in-module
import sounddevice as sd
from aioesphomeapi.api_pb2 import ( # type: ignore[attr-defined]
@@ -48,6 +65,17 @@ from .util import call_all, get_mac, is_arm
from .event_bus import EventBus
from .event_led import LedEvent
from pathlib import Path
# Hide helper models that are not detectors
_OWW_DETECT_MODEL_DENYLIST = {"embedding_model", "melspectrogram"}
def _oww_filter_models(models: list[str]) -> list[str]:
"""Normalize names (strip .tflite/paths), drop helper models, sort & de-dupe."""
stems = {Path(m).stem for m in models}
return sorted(m for m in stems if m not in _OWW_DETECT_MODEL_DENYLIST)
_LOGGER = logging.getLogger(__name__)
_MODULE_DIR = Path(__file__).parent
@@ -86,7 +114,8 @@ class ServerState:
event_bus: EventBus
media_player_entity: Optional[MediaPlayerEntity] = None
satellite: "Optional[VoiceSatelliteProtocol]" = None
wyoming_wake: Optional[WyomingWakeClient] = None
current_remote_wake: Optional[str] = None
wyoming_wake: "Optional[WyomingWakeClient]" = None
use_wyoming_wake: bool = False
@@ -231,7 +260,7 @@ class VoiceSatelliteProtocol(APIServer):
)
for ww in self.state.available_wake_words.values()
],
active_wake_words=[self.state.wake_word.id],
active_wake_words=[self.state.current_remote_wake] if self.state.use_wyoming_wake else [self.state.wake_word.id],
max_active_wake_words=1,
)
_LOGGER.info("Connected to Home Assistant")
@@ -242,6 +271,15 @@ class VoiceSatelliteProtocol(APIServer):
# Already active
break
if self.state.use_wyoming_wake and self.state.wyoming_wake:
try:
self.state.wyoming_wake.set_detect([wake_word_id])
_LOGGER.info("Switched Wyoming wake model to: %s", wake_word_id)
self.state.current_remote_wake = wake_word_id
self.state.wake_word.is_active = False
except Exception:
_LOGGER.exception("Failed to switch Wyoming wake model")
break
model_info = self.state.available_wake_words.get(wake_word_id)
if not model_info:
continue
@@ -270,8 +308,12 @@ class VoiceSatelliteProtocol(APIServer):
_LOGGER.debug("Stopping timer finished sound")
return
wake_word_phrase = self.state.wake_word.wake_word
wake_word_phrase = (self.state.current_remote_wake or self.state.wake_word.wake_word) if self.state.use_wyoming_wake else self.state.wake_word.wake_word
_LOGGER.debug("Detected wake word: %s", wake_word_phrase)
try:
self.state.wyoming_wake.suppress(2500)
except Exception:
pass
self.send_messages(
[VoiceAssistantRequest(start=True, wake_word_phrase=wake_word_phrase)]
)
@@ -348,9 +390,6 @@ class VoiceSatelliteProtocol(APIServer):
def process_audio(state: ServerState):
# debug counters
chunks_sent = 0
last_log = 0.0
try:
while True:
@@ -363,9 +402,6 @@ def process_audio(state: ServerState):
try:
state.satellite.handle_audio(audio_chunk)
chunks_sent += 1
if state.use_wyoming_wake and chunks_sent % 100 == 0:
_LOGGER.debug("[OWW] chunks_sent=%d (approx %.1fs of audio)", chunks_sent, chunks_sent*0.064)
if state.use_wyoming_wake:
if state.wyoming_wake:
@@ -395,6 +431,7 @@ async def main() -> None:
parser.add_argument("--name", required=True)
parser.add_argument("--wake-uri", help="Wyoming wake server URI (e.g., tcp://127.0.0.1:10400)")
parser.add_argument("--wake-word-name", help="Wake word name on the Wyoming server (e.g., hal)")
parser.add_argument("--wake-models-dir", help="Fallback: directory containing openWakeWord *.tflite models to list if the server does not advertise them")
parser.add_argument(
"--audio-input-device",
default="default",
@@ -433,11 +470,14 @@ async def main() -> None:
logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO)
_LOGGER.debug(args)
use_wyoming = bool(args.wake_uri and args.wake_word_name)
# Optional fallback for discovering remote models from filesystem
models_dir = args.wake_models_dir or os.environ.get("WYOMING_MODELS_DIR")
use_wyoming = bool(args.wake_uri)
wyoming_client = None
if use_wyoming:
_LOGGER.info("Using Wyoming openWakeWord at %s (name=%s)", args.wake_uri, args.wake_word_name)
wyoming_client = WyomingWakeClient(args.wake_uri, args.wake_word_name)
wyoming_client = WyomingWakeClient(args.wake_uri, args.wake_word_name or "")
# Load available wake words
wake_word_dir = Path(args.wake_word_dir)
@@ -459,6 +499,28 @@ async def main() -> None:
_LOGGER.debug("Available wake words: %s", list(sorted(available_wake_words.keys())))
if use_wyoming and wyoming_client:
try:
remote_models = wyoming_client.list_models() or []
_LOGGER.debug("Queried Wyoming models: %s", [m.get("name") if isinstance(m, dict) else m for m in remote_models])
except Exception:
remote_models = []
_LOGGER.exception("Failed to list models from Wyoming server")
if remote_models:
available_wake_words = {}
for m in remote_models:
name = (m.get("name") or "").strip()
langs = m.get("languages") or m.get("trained_languages") or []
if not name:
continue
available_wake_words[name] = AvailableWakeWord(
id=name,
wake_word=(m.get('label') or _prettify_model_name(name)),
trained_languages=langs,
config_path=Path(f"wyoming:{name}"),
)
_LOGGER.debug("Remote OpenWakeWord models: %s", list(sorted(available_wake_words.keys())))
libtensorflowlite_c_path = _LIB_DIR / "libtensorflowlite_c.so"
_LOGGER.debug("libtensorflowlite_c path: %s", libtensorflowlite_c_path)
@@ -496,12 +558,58 @@ async def main() -> None:
# Connect to Wyoming wake server if enabled
if state.use_wyoming_wake and state.wyoming_wake:
def _on_detect(_name, _ts):
_LOGGER.debug("[OWW] detection callback fired name=%s ts=%s", _name, _ts)
if state.satellite is not None:
state.loop.call_soon_threadsafe(lambda: state.satellite.wakeup())
state.wyoming_wake.connect(_on_detect)
# Disable local wake model to avoid double triggers
state.wake_word.is_active = False
_LOGGER.debug("[OWW] Local MicroWakeWord disabled; streaming audio to Wyoming")
# Refresh remote models after connect
try:
cached = state.wyoming_wake.get_models(timeout=1.5) if state.wyoming_wake else []
if not cached and models_dir:
try:
from pathlib import Path as _P
cached = [{"name": p.stem} for p in _P(models_dir).glob("*.tflite")]
cached = [
m for m in cached
if (m.get('name') or '').strip() not in EXCLUDED_OWW_MODELS
]
_LOGGER.info("Using filesystem fallback for OWW models (post-connect): %s", [m["name"] for m in cached])
except Exception:
_LOGGER.exception("Failed to scan wake-models-dir (post-connect)")
if cached:
available = {}
for m in cached:
name = (m.get('name') or '').strip()
if not name:
continue
available[name] = AvailableWakeWord(
id=name,
wake_word=(m.get('label') or _prettify_model_name(name)),
trained_languages=[],
config_path=Path(f"wyoming:{name}")
)
state.available_wake_words = available
_LOGGER.debug("Using remote models from cached info: %s", list(sorted(state.available_wake_words.keys())))
except Exception:
_LOGGER.exception("Failed to refresh remote models from connection info")
# Initialize remote detection target if provided
try:
if args.wake_word_name:
state.wyoming_wake.set_detect([args.wake_word_name])
state.current_remote_wake = args.wake_word_name
else:
try:
first = next(iter(state.available_wake_words.keys()))
except StopIteration:
first = None
if first:
state.wyoming_wake.set_detect([first])
state.current_remote_wake = first
else:
_LOGGER.warning("Wyoming enabled but no remote models available; not sending detect yet")
except Exception:
_LOGGER.exception("Failed to initialize remote detect")
process_audio_thread = threading.Thread(
target=process_audio, args=(state,), daemon=True
linux_voice_assistant/openwakeword_client.py
+299 -79
View File
@@ -2,43 +2,95 @@ import json
import logging
import socket
import threading
import time
from typing import Callable, Optional
from typing import Callable, Optional, List, Dict, Any
from urllib.parse import urlparse
from pathlib import Path
import time
# Models that should NOT appear in UI dropdowns
EXCLUDED_OWW_MODELS = {"embedding_model", "melspectrogram"}
# Models that are not actual wake-word detectors and should never be shown
_OWW_DETECT_MODEL_DENYLIST = {"embedding_model", "melspectrogram"}
import re
def _prettify_model_name(name: str) -> str:
"""
Convert 'hey_jarvis_v0.1' -> 'Hey Jarvis', 'ok_nabu_v0.1' -> 'OK Nabu',
'hal_v2' -> 'Hal', 'marvin_v2' -> 'Marvin'.
"""
stem = Path(name).stem
# Strip common version suffixes
stem = re.sub(r"(?:_v\d+(?:\.\d+)?)$", "", stem)
parts = stem.split("_")
if parts and parts[0].lower() == "ok":
parts[0] = "OK"
else:
parts = [p.capitalize() for p in parts]
return " ".join(parts)
def _normalize_model_name(name: str) -> str:
# Accept "foo_v2", "foo_v2.tflite", full paths, etc.
stem = Path(name).stem
return stem
def _filter_detect_models(models: list[str]) -> list[str]:
# Remove helper models and de-duplicate
normalized = {_normalize_model_name(m) for m in models}
return sorted(m for m in normalized if m not in _OWW_DETECT_MODEL_DENYLIST)
_LOGGER = logging.getLogger(__name__)
def _jsonl(obj: dict) -> bytes:
return (json.dumps(obj, separators=(",", ":")) + "\n").encode("utf-8")
class WyomingWakeClient:
"""Minimal client for Wyoming wake detection (openWakeWord) with extra debug."""
"""Minimal client for Wyoming wake detection (openWakeWord)."""
def __init__(self, uri: str, name: str, rate=16000, width=2, channels=1):
self.uri, self.name = uri, name
self.rate, self.width, self.channels = rate, width, channels
self._paused = False
self._suppress_until_ms = 0
self._refractory_ms = 0
self._sock: Optional[socket.socket] = None
self._reader: Optional[threading.Thread] = None
self._on_detect: Optional[Callable[[str, Optional[int]], None]] = None
self._closed = False
self._chunks = 0
self._last_log_time = 0.0
# model discovery cache
self._models_cache: List[Dict[str, Any]] = []
self._info_event = threading.Event()
def connect(self, on_detect: Callable[[str, Optional[int]], None]) -> None:
self._on_detect = on_detect
parsed = urlparse(self.uri)
if parsed.scheme != "tcp":
raise ValueError(f"Only tcp:// URIs are supported (got {self.uri})")
host = parsed.hostname or "127.0.0.1"
port = parsed.port or 10400
_LOGGER.debug("[OWW] Connecting to %s:%s", host, port)
self._sock = socket.create_connection((host, port))
self._sock = socket.create_connection((parsed.hostname or "127.0.0.1", parsed.port or 10400))
self._sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self._send({ "type": "audio-start", "data": { "rate": self.rate, "width": self.width, "channels": self.channels }})
self._arm_detect()
# Start audio stream
self._send({"type": "audio-start", "data": {"rate": self.rate, "width": self.width, "channels": self.channels}})
# Ask for model info
try:
self._send({"type": "describe"})
except Exception:
pass
# Set initial detect model
if self.name:
try:
self._send({"type": "detect", "data": {"names": [self.name]}})
except Exception:
pass
self._reader = threading.Thread(target=self._read_loop, daemon=True)
self._reader.start()
_LOGGER.info("[OWW] Connected to Wyoming wake server at %s", self.uri)
_LOGGER.debug("Connected to Wyoming wake server at %s", self.uri)
def close(self) -> None:
self._closed = True
@@ -52,93 +104,261 @@ class WyomingWakeClient:
finally:
self._sock = None
def _arm_detect(self) -> None:
_LOGGER.debug("[OWW] Arming detect for name=%s", self.name)
self._send({"type": "detect", "data": {"names": [self.name]}})
def send_audio_chunk(self, pcm: bytes, timestamp_ms: Optional[int] = None) -> None:
if self._sock is None or self._closed:
return
hdr = {
"type": "audio-chunk",
"data": {"rate": self.rate, "width": self.width, "channels": self.channels},
"payload_length": len(pcm),
}
hdr = {"type": "audio-chunk",
"data": {"rate": self.rate, "width": self.width, "channels": self.channels},
"payload_length": len(pcm)}
if timestamp_ms is not None:
hdr["data"]["timestamp"] = timestamp_ms
try:
self._sock.sendall(_jsonl(hdr))
self._sock.sendall(pcm)
except Exception:
_LOGGER.exception("[OWW] Failed to send audio chunk")
return
self._sock.sendall(_jsonl(hdr))
self._sock.sendall(pcm)
def pause(self) -> None:
self._paused = True
def resume(self) -> None:
self._paused = False
def suppress(self, ms: int) -> None:
try:
now_ms = int(time.time() * 1000)
except Exception:
now_ms = 0
self._suppress_until_ms = max(self._suppress_until_ms, now_ms + int(ms))
def set_refractory(self, ms: int) -> None:
self._refractory_ms = max(0, int(ms))
def list_models(self):
"""Query models using a short-lived connection with timeout; accept 'describe' or 'info'."""
parsed = urlparse(self.uri)
if parsed.scheme != "tcp":
raise ValueError(f"Only tcp:// URIs are supported (got {self.uri})")
host, port = (parsed.hostname or "127.0.0.1", parsed.port or 10400)
s = socket.create_connection((host, port))
try:
s.settimeout(1.5)
s.sendall(_jsonl({"type": "describe"}))
f = s.makefile("rb")
while True:
try:
line = f.readline()
except Exception:
break
if not line:
break
try:
msg = json.loads(line.decode("utf-8"))
except Exception:
continue
mtype = msg.get("type")
if mtype in ("describe", "info"):
data = msg.get("data", {}) or {}
wake = data.get("wake", {}) or {}
models = wake.get("models") or wake.get("names") or []
norm: List[Dict[str, Any]] = []
for m in models:
if isinstance(m, str):
norm.append({"name": m})
elif isinstance(m, dict):
norm.append(m)
out: List[Dict[str, Any]] = []
for m in norm:
name = (m.get('name') or '').strip()
if not name or name in EXCLUDED_OWW_MODELS:
continue
phrase = (m.get('phrase') or '').strip()
desc = (m.get('description') or '').strip()
try:
label = phrase or desc or _prettify_model_name(name) # type: ignore[name-defined]
except NameError:
stem = Path(name).stem
stem = re.sub(r"(?:_v\d+(?:\.\d+)?)$", "", stem)
parts = stem.split("_")
if parts and parts[0].lower() == "ok":
parts[0] = "OK"
else:
parts = [p.capitalize() for p in parts]
label = " ".join(parts)
mm = dict(m)
mm['label'] = label
out.append(mm)
return out
finally:
try:
s.close()
except Exception:
pass
return []
def get_models(self, timeout: float = 1.5):
"""Return cached models after waiting briefly for an 'info'/'describe' update."""
self._info_event.wait(timeout)
return list(self._models_cache)
def set_detect(self, names):
if self._sock is None:
raise RuntimeError("Not connected")
filtered = [n for n in list(names) if n]
if not filtered:
_LOGGER.warning("Ignoring empty detect() request (no model names)")
return
self.name = filtered[0]
_LOGGER.debug("Requesting Wyoming detect models: %s", filtered)
self._send({"type": "detect", "data": {"names": filtered}})
self._chunks += 1
if (self._chunks % 1000) == 0:
# ~64ms * 1000 = ~64s of audio
_LOGGER.debug("[OWW] Sent %d chunks (~%.1fs)", self._chunks, self._chunks * 0.064)
# internals
def _send(self, obj: dict) -> None:
assert self._sock is not None
try:
self._sock.sendall(_jsonl(obj))
except Exception:
_LOGGER.exception("[OWW] send failed (%s)", obj.get("type"))
def _cycle_stream(self) -> None:
"""Some servers require restarting the audio stream after a detection."""
_LOGGER.debug("[OWW] Cycling audio stream")
try:
self._send({"type": "audio-stop"})
except Exception:
_LOGGER.exception("[OWW] audio-stop failed")
try:
self._send({"type": "audio-start", "data": {"rate": self.rate, "width": self.width, "channels": self.channels}})
except Exception:
_LOGGER.exception("[OWW] audio-start failed")
self._sock.sendall(_jsonl(obj))
# --- paste this to replace the whole _read_loop method ---
def _read_loop(self) -> None:
assert self._sock is not None
f = self._sock.makefile("rb")
_LOGGER.debug("[OWW] Reader loop started")
buf = b""
decoder = json.JSONDecoder()
buf = "" # text buffer for concatenated JSON objects
def _yield_objects_from_text(text: str):
"""Yield JSON objects from `text` even if they are concatenated like {}{}..."""
start = 0
depth = 0
in_str = False
escape = False
for i, ch in enumerate(text):
if in_str:
if escape:
escape = False
elif ch == "\\":
escape = True
elif ch == '"':
in_str = False
else:
if ch == '"':
in_str = True
elif ch == "{":
if depth == 0:
start = i
depth += 1
elif ch == "}":
depth -= 1
if depth == 0:
# complete object from start..i
yield text[start : i + 1], i + 1
# return here with any remainder unparsed
return
while not self._closed:
chunk = f.readline()
if not chunk:
_LOGGER.debug("[OWW] Reader EOF")
break
buf += chunk
try:
# There might be multiple JSON objects in one line; parse until exhausted.
while buf:
s = buf.decode("utf-8")
obj, idx = decoder.raw_decode(s)
buf = s[idx:].lstrip().encode("utf-8")
mtype = obj.get("type")
if mtype and mtype != "detection":
_LOGGER.debug("[OWW] recv %s", mtype)
piece = chunk.decode("utf-8")
except Exception:
try:
_LOGGER.debug("Wyoming raw (non-utf8): %r", chunk[:200])
except Exception:
pass
continue
buf += piece
# Try to peel off as many complete JSON objects as we can
pos = 0
progressed = True
while progressed and pos < len(buf):
progressed = False
for obj_text, end_pos in _yield_objects_from_text(buf[pos:]):
progressed = True
pos += end_pos
try:
msg = json.loads(obj_text)
except Exception:
try:
_LOGGER.debug("Wyoming raw (bad-json): %r", obj_text[:200])
except Exception:
pass
continue
mtype = str(msg.get("type") or "").lower()
try:
_LOGGER.debug("Received Wyoming message type=%s", mtype or "<?>")
if mtype and mtype not in ("info", "describe"):
_LOGGER.debug("Wyoming raw: %r", obj_text[:200])
except Exception:
pass
# ----- model info path -----
if mtype in ("info", "describe"):
data = msg.get("data", {}) or {}
wake = data.get("wake", {}) if isinstance(data, dict) else {}
models = []
if isinstance(wake, dict):
if isinstance(wake.get("models"), list):
models = wake.get("models")
elif isinstance(wake.get("names"), list):
models = wake.get("names")
norm = []
for m in models:
if isinstance(m, str):
norm.append({"name": m})
elif isinstance(m, dict):
norm.append(m)
if norm:
self._models_cache = norm
self._info_event.set()
_LOGGER.debug(
"Wyoming model list updated: %s",
[m.get("name") for m in norm],
)
continue
# ----- detection path (accept several shapes) -----
data = msg.get("data", {}) or {}
is_detection = False
if mtype == "detection":
data = obj.get("data", {}) or {}
name = data.get("name") or self.name
ts = data.get("timestamp")
_LOGGER.info("[OWW] Detection: name=%s ts=%s (chunks=%d)", name, ts, self._chunks)
is_detection = True
elif mtype in ("event", "wake", "hotword"):
ev = str(data.get("event") or "").lower()
if ev in ("detection", "wake", "hotword"):
is_detection = True
if is_detection:
name = (
data.get("name")
or data.get("model")
or data.get("wakeword")
or self.name
)
ts = data.get("timestamp") or data.get("time") or data.get("ts")
_LOGGER.debug("Wake detected: %s ts=%s", name, ts)
if self._on_detect:
try:
self._on_detect(name, ts)
now_ms = int(time.time() * 1000)
if self._paused or (self._suppress_until_ms and now_ms < self._suppress_until_ms):
_LOGGER.debug("Detection suppressed (paused=%s until=%s)", self._paused, getattr(self, '_suppress_until_ms', 0))
else:
now_ms = int(time.time() * 1000)
if self._paused or (self._suppress_until_ms and now_ms < self._suppress_until_ms):
_LOGGER.debug("Detection suppressed (paused=%s until=%s)", self._paused, getattr(self, '_suppress_until_ms', 0))
else:
self._on_detect(name, ts)
if getattr(self, '_refractory_ms', 0):
self.suppress(self._refractory_ms)
if getattr(self, '_refractory_ms', 0):
self.suppress(self._refractory_ms)
except Exception:
_LOGGER.exception("[OWW] Error in detection callback")
# Some servers need an audio stream restart before re-arming
try:
self._cycle_stream()
except Exception:
_LOGGER.exception("[OWW] Failed to cycle stream")
# Re-arm detect so subsequent detections work
try:
self._arm_detect()
except Exception:
_LOGGER.exception("[OWW] Failed to re-arm detect")
except json.JSONDecodeError:
# Need more bytes; continue accumulating
pass
_LOGGER.exception("Error in detection callback")
# keep any unconsumed remainder in buffer
buf = buf[pos:]