sirmrak/linux-voice-assistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #35 from imonlinux/refactored-upstream-changes

Browse Source 
Refactored upstream changes
This commit is contained in:
imonlinux
2025-11-17 20:26:43 -08:00
committed by GitHub
parent 90f37df4e8 4d8f5c97b7
commit fc6d750257
17 changed files with 824 additions and 863 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+71 -15
View File
@@ -1,19 +1,75 @@
.DS_Store
.idea
*.log
tmp/
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*.egg
build
htmlcov
*$py.class
.projectile
.venv/
venv/
.tox/
.mypy_cache/
*.egg-info/
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
preferences.json
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a script, but we don't want them in .git
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
# Foreign VSCode settings
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
# Ruff cache
.ruff_cache/
# Pyright cache
.pyright_cache/
# mypy
.mypy_cache/
# venv
venv/
.venv/
# App-specific
/dist/
/preferences.json
/local/
README.md
+1
View File
@@ -9,6 +9,7 @@ See [the tutorial](docs/linux-voice-assistant-2mic-install.md) to build a satell
### What's working:
- This fork is from https://github.com/OHF-Voice/linux-voice-assistant Release v1.0.0 which introduces the ablity to use both MicroWakeWord and OpenWakeWord detections models.
- Refactor: Sync with Upstream Architectural Changes (Nov 2025) incudes the 7 commits since the release of v1.0.0.
- Updated to support LED Events including GPIO based LED controls. Defaults to the ReSpeaker 2Mic Hat SPI leds, but you can use the Grove port GPIO12/13 by updating the config.json file.
- Updated to support running either APA102 or WS2812B LEDs from the SPI interface using a Micro Connectors 40-pin GPIO 1 to 2 Expansion Board. See the tutorial for instructions.
- Now supports **ALSA, PulseAudio and PipeWire** playback backends using the updated `linux_voice_assistant/mpv_player.py`.
lib/SOURCE
-1
View File
@@ -1 +0,0 @@
https://github.com/tphakala/tflite_c/
lib/linux_amd64/libtensorflowlite_c.so
BIN
View File
Binary file not shown.
lib/linux_arm64/libtensorflowlite_c.so
BIN
View File
Binary file not shown.
linux_voice_assistant/__main__.py
+179 -97
View File
@@ -3,36 +3,31 @@ import argparse
import asyncio
import json
import logging
import sys
import threading
import time
from pathlib import Path
from queue import Queue
from typing import Dict, List, Optional, Union
from typing import Dict, List, Optional, Set, Union
import numpy as np
import sounddevice as sd
import soundcard as sc
from pymicro_wakeword import MicroWakeWord, MicroWakeWordFeatures
from pyopen_wakeword import OpenWakeWord, OpenWakeWordFeatures
from .config import Config, load_config_from_json # <-- NEW
from .config import Config, load_config_from_json
from .event_bus import EventBus, EventHandler, subscribe
from .led_controller import LedController
from .mqtt_controller import MqttController
from .microwakeword import MicroWakeWord, MicroWakeWordFeatures
from .models import AvailableWakeWord, Preferences, ServerState, WakeWordType
from .mpv_player import MpvMediaPlayer
from .openwakeword import OpenWakeWord, OpenWakeWordFeatures
from .satellite import VoiceSatelliteProtocol
from .util import get_mac, is_arm
from .util import get_mac
from .zeroconf import HomeAssistantZeroconf
_LOGGER = logging.getLogger(__name__)
_MODULE_DIR = Path(__file__).parent
_REPO_DIR = _MODULE_DIR.parent
if is_arm():
_LIB_DIR = _REPO_DIR / "lib" / "linux_arm64"
else:
_LIB_DIR = _REPO_DIR / "lib" / "linux_amd64"
# -----------------------------------------------------------------------------
class MicMuteHandler(EventHandler):
@@ -54,12 +49,6 @@ class MicMuteHandler(EventHandler):
if is_muted:
self.state.event_bus.publish("mic_muted")
else:
_LOGGER.debug("Clearing stale audio queue...")
while not self.state.audio_queue.empty():
try:
self.state.audio_queue.get_nowait()
except Queue.Empty:
break
self.state.event_bus.publish("mic_unmuted")
# -----------------------------------------------------------------------------
@@ -71,21 +60,69 @@ async def main() -> None:
"-c",
"--config",
type=Path,
required=True,
help="Path to configuration.json file",
required=False,
default=_MODULE_DIR / "config.json",
help="Path to configuration.json file (default: linux_voice_assistant/config.json)",
)
parser.add_argument(
"--debug",
action="store_true",
help="Enable debug logging (overrides config file)",
)
parser.add_argument(
"--list-input-devices",
action="store_true",
help="List audio input devices and exit",
)
parser.add_argument(
"--list-output-devices",
action="store_true",
help="List audio output devices and exit",
)
args = parser.parse_args()
# --- Load Configuration ---
config = load_config_from_json(args.config)
# --- Handle List Devices ---
if args.list_input_devices:
print("Input devices")
print("=" * 13)
for idx, mic in enumerate(sc.all_microphones(include_loopback=False)):
print(f"[{idx}]", mic.name)
return
if args.list_output_devices:
player = MpvMediaPlayer(loop=None)
print("Output devices")
print("=" * 14)
try:
for speaker in player.player.audio_device_list:
print(speaker["name"] + ":", speaker["description"])
except Exception as e:
_LOGGER.error("Failed to list output devices: %s", e)
return
# --- Load Configuration ---
config_path = args.config
if not config_path.is_absolute():
config_path = _REPO_DIR / config_path
config = load_config_from_json(config_path)
if args.debug:
config.app.debug = True
# --- Setup Logging ---
logging.basicConfig(level=logging.DEBUG if config.app.debug else logging.INFO)
_LOGGER.info("Loading configuration from: %s", config_path)
_LOGGER.debug("Configuration loaded: %s", config)
loop = asyncio.get_running_loop()
event_bus = EventBus()
# --- Create Download Dir ---
download_dir = _REPO_DIR / config.wake_word.download_dir
download_dir.mkdir(parents=True, exist_ok=True)
_LOGGER.debug("Download directory: %s", download_dir)
# --- Load Preferences ---
preferences_path = _REPO_DIR / config.app.preferences_file
if preferences_path.exists():
@@ -95,51 +132,63 @@ async def main() -> None:
else:
preferences = Preferences()
# Set default num_leds from config, allowing preferences to override
preferences.num_leds = getattr(preferences, 'num_leds', config.led.num_leds)
# --- Load Wake Words ---
available_wake_words: Dict[str, AvailableWakeWord] = {}
# Add default directories if none are specified
if not config.wake_word.directories:
config.wake_word.directories = ["wakewords", "wakewords/openWakeWord"]
for ww_dir_str in config.wake_word.directories:
wake_word_dir = _REPO_DIR / ww_dir_str
# Add download directory to search path
wake_word_dirs = [_REPO_DIR / d for d in config.wake_word.directories]
wake_word_dirs.append(download_dir / "external_wake_words") # <-- ADDED
for wake_word_dir in wake_word_dirs:
if not wake_word_dir.exists():
continue
for model_config_path in wake_word_dir.glob("*.json"):
model_id = model_config_path.stem
if model_id == config.wake_word.stop_model:
continue
with open(model_config_path, "r", encoding="utf-8") as model_config_file:
model_config = json.load(model_config_file)
model_type = model_config["type"]
model_type = WakeWordType(model_config["type"])
if model_type == WakeWordType.OPEN_WAKE_WORD:
wake_word_path = model_config_path.parent / model_config["model"]
else:
wake_word_path = model_config_path
available_wake_words[model_id] = AvailableWakeWord(
id=model_id,
type=WakeWordType(model_type),
type=model_type,
wake_word=model_config["wake_word"],
trained_languages=model_config.get("trained_languages", []),
config_path=model_config_path,
wake_word_path=wake_word_path,
)
_LOGGER.debug("Available wake words: %s", list(sorted(available_wake_words.keys())))
libtensorflowlite_c_path = _LIB_DIR / "libtensorflowlite_c.so"
# --- Load Active Wake Word Models ---
active_wake_words: Set[str] = set()
wake_models: Dict[str, Union[MicroWakeWord, OpenWakeWord]] = {}
if preferences.active_wake_words:
for wake_word_id in preferences.active_wake_words:
wake_word = available_wake_words.get(wake_word_id)
if wake_word is None: continue
wake_models[wake_word_id] = wake_word.load(libtensorflowlite_c_path)
_LOGGER.debug("Loading wake model: %s", wake_word_id)
wake_models[wake_word_id] = wake_word.load()
active_wake_words.add(wake_word_id)
# Load default model if no preferences are set
if not wake_models:
wake_word_id = config.wake_word.model
wake_word = available_wake_words.get(wake_word_id)
if wake_word:
wake_models[wake_word_id] = wake_word.load(libtensorflowlite_c_path)
_LOGGER.debug("Loading wake model: %s", wake_word_id)
wake_models[wake_word_id] = wake_word.load()
active_wake_words.add(wake_word_id)
# --- Load Stop Model ---
stop_model: Optional[MicroWakeWord] = None
@@ -147,10 +196,27 @@ async def main() -> None:
wake_word_dir = _REPO_DIR / ww_dir_str
stop_config_path = wake_word_dir / f"{config.wake_word.stop_model}.json"
if not stop_config_path.exists(): continue
stop_model = MicroWakeWord.from_config(stop_config_path, libtensorflowlite_c_path)
_LOGGER.debug("Loading stop model: %s", stop_config_path)
stop_model = MicroWakeWord.from_config(stop_config_path)
break
assert stop_model is not None
# --- Resolve Microphone ---
if config.audio.input_device is not None:
try:
input_device_idx = int(config.audio.input_device)
mic = sc.all_microphones(include_loopback=False)[input_device_idx]
except (ValueError, IndexError):
mic = sc.get_microphone(config.audio.input_device)
else:
mic = sc.default_microphone()
if mic is None:
_LOGGER.critical("No microphone found.")
sys.exit(1)
_LOGGER.info("Using audio input device: %s", mic.name)
# --- Initialize Media Players ---
music_player = MpvMediaPlayer(
loop=loop,
@@ -167,10 +233,10 @@ async def main() -> None:
state = ServerState(
name=config.app.name,
mac_address=get_mac(),
audio_queue=Queue(),
entities=[],
available_wake_words=available_wake_words,
wake_words=wake_models,
active_wake_words=active_wake_words,
stop_word=stop_model,
music_player=music_player,
tts_player=tts_player,
@@ -178,11 +244,9 @@ async def main() -> None:
timer_finished_sound=str(_REPO_DIR / config.app.timer_finished_sound),
preferences=preferences,
preferences_path=preferences_path,
libtensorflowlite_c_path=libtensorflowlite_c_path,
event_bus=event_bus,
loop=loop,
oww_melspectrogram_path=(_REPO_DIR / config.wake_word.oww_melspectrogram_model),
oww_embedding_path=(_REPO_DIR / config.wake_word.oww_embedding_model),
download_dir=download_dir, # <-- ADDED
refractory_seconds=config.wake_word.refractory_seconds,
)
@@ -210,11 +274,12 @@ async def main() -> None:
mic_mute_handler = MicMuteHandler(state, mqtt_controller)
# --- Start Audio Processing Thread ---
process_audio_thread = threading.Thread(target=process_audio, args=(state,), daemon=True)
process_audio_thread = threading.Thread(
target=process_audio,
args=(state, mic, config.audio.input_block_size),
daemon=True
)
process_audio_thread.start()
def sd_callback(indata, _frames, _time, _status):
state.audio_queue.put_nowait(bytes(indata))
# --- Start ESPHome Server ---
server = await loop.create_server(
@@ -227,31 +292,22 @@ async def main() -> None:
# --- Run Forever ---
try:
input_device = config.audio.input_device if config.audio.input_device else "default"
with sd.RawInputStream(
samplerate=16000,
blocksize=config.audio.input_block_size,
device=input_device,
dtype="int16",
channels=1,
callback=sd_callback
):
async with server:
_LOGGER.info("Server started (host=%s, port=%s)", config.esphome.host, config.esphome.port)
await server.serve_forever()
async with server:
_LOGGER.info("Server started (host=%s, port=%s)", config.esphome.host, config.esphome.port)
await server.serve_forever()
except KeyboardInterrupt:
pass
except sd.PortAudioError as e:
_LOGGER.critical("Failed to open audio input device '%s': %s", input_device, e)
finally:
if mqtt_controller:
mqtt_controller.stop()
state.audio_queue.put_nowait(None)
state.mic_muted = True
process_audio_thread.join()
_LOGGER.debug("Server stopped")
def process_audio(state: ServerState):
def process_audio(state: ServerState, mic, block_size: int):
"""Process audio chunks from the microphone in a separate thread."""
wake_words: List[Union[MicroWakeWord, OpenWakeWord]] = []
micro_features: Optional[MicroWakeWordFeatures] = None
micro_inputs: List[np.ndarray] = []
@@ -260,43 +316,69 @@ def process_audio(state: ServerState):
has_oww = False
last_active: Optional[float] = None
try:
while True:
if state.mic_muted:
time.sleep(0.1)
continue
audio_chunk = state.audio_queue.get()
if audio_chunk is None: break
if state.satellite is None: continue
if (not wake_words) or (state.wake_words_changed and state.wake_words):
state.wake_words_changed = False
wake_words = [ww for ww in state.wake_words.values() if ww.is_active]
has_oww = any(isinstance(ww, OpenWakeWord) for ww in wake_words)
if micro_features is None:
micro_features = MicroWakeWordFeatures(libtensorflowlite_c_path=state.libtensorflowlite_c_path)
if has_oww and (oww_features is None):
oww_features = OpenWakeWordFeatures(melspectrogram_model=state.oww_melspectrogram_path, embedding_model=state.oww_embedding_path, libtensorflowlite_c_path=state.libtensorflowlite_c_path)
try:
state.satellite.handle_audio(audio_chunk)
assert micro_features is not None
micro_inputs.clear(); micro_inputs.extend(micro_features.process_streaming(audio_chunk))
if has_oww:
assert oww_features is not None
oww_inputs.clear(); oww_inputs.extend(oww_features.process_streaming(audio_chunk))
for wake_word in wake_words:
activated = False
if isinstance(wake_word, MicroWakeWord):
if any(wake_word.process_streaming(mi) for mi in micro_inputs): activated = True
elif isinstance(wake_word, OpenWakeWord):
if any(p > 0.5 for oi in oww_inputs for p in wake_word.process_streaming(oi)): activated = True
if activated:
now = time.monotonic()
if (last_active is None) or ((now - last_active) > state.refractory_seconds):
state.satellite.wakeup(wake_word)
last_active = now
if any(state.stop_word.process_streaming(mi) for mi in micro_inputs) and state.stop_word.is_active:
state.satellite.stop()
except Exception: _LOGGER.exception("Unexpected error handling audio")
except Exception: _LOGGER.exception("Unexpected error processing audio")
_LOGGER.debug("Opening audio input device: %s", mic.name)
with mic.recorder(samplerate=16000, channels=1, blocksize=block_size) as mic_in:
while True:
if state.mic_muted:
time.sleep(0.1)
continue
audio_chunk_array = mic_in.record(block_size).reshape(-1)
audio_chunk = (
(np.clip(audio_chunk_array, -1.0, 1.0) * 32767.0)
.astype("<i2")
.tobytes()
)
if state.satellite is None:
continue
if (not wake_words) or (state.wake_words_changed and state.wake_words):
state.wake_words_changed = False
wake_words = [
ww for ww in state.wake_words.values()
if ww.id in state.active_wake_words
]
has_oww = any(isinstance(ww, OpenWakeWord) for ww in wake_words)
if micro_features is None:
micro_features = MicroWakeWordFeatures()
if has_oww and (oww_features is None):
oww_features = OpenWakeWordFeatures.from_builtin()
try:
state.satellite.handle_audio(audio_chunk)
assert micro_features is not None
micro_inputs.clear(); micro_inputs.extend(micro_features.process_streaming(audio_chunk))
if has_oww:
assert oww_features is not None
oww_inputs.clear(); oww_inputs.extend(oww_features.process_streaming(audio_chunk))
for wake_word in wake_words:
activated = False
if isinstance(wake_word, MicroWakeWord):
if any(wake_word.process_streaming(mi) for mi in micro_inputs): activated = True
elif isinstance(wake_word, OpenWakeWord):
if any(p > 0.5 for oi in oww_inputs for p in wake_word.process_streaming(oi)): activated = True
if activated:
now = time.monotonic()
if (last_active is None) or ((now - last_active) > state.refractory_seconds):
state.satellite.wakeup(wake_word)
last_active = now
stopped = False
for micro_input in micro_inputs:
if state.stop_word.process_streaming(micro_input):
stopped = True
if stopped and (state.stop_word.id in state.active_wake_words):
state.satellite.stop()
except Exception: _LOGGER.exception("Unexpected error handling audio")
except Exception as e:
_LOGGER.critical("Error in audio processing thread: %s", e)
state.loop.call_soon_threadsafe(state.loop.stop)
if __name__ == "__main__":
print("--- __main__ block executing ---")
linux_voice_assistant/config.py
+1 -8
View File
@@ -23,8 +23,7 @@ class WakeWordConfig:
model: str = "okay_nabu"
stop_model: str = "stop"
refractory_seconds: float = 2.0
oww_melspectrogram_model: str = "wakewords/openWakeWord/melspectrogram.tflite"
oww_embedding_model: str = "wakewords/openWakeWord/embedding_model.tflite"
download_dir: str = "local"
@dataclass
class ESPHomeConfig:
@@ -88,10 +87,6 @@ def load_config_from_json(config_path: Path) -> Config:
raise
# --- Step 2: Create config objects from raw data ---
# This nested loading allows users to omit entire sections (like 'audio')
# and still get the defaults.
# 'app' is the only required section
if "app" not in raw_data:
raise ValueError("Configuration file must contain an 'app' section with a 'name'.")
@@ -103,7 +98,6 @@ def load_config_from_json(config_path: Path) -> Config:
mqtt_config = MqttConfig(**raw_data.get("mqtt", {}))
# --- Step 3: Set MQTT 'enabled' flag ---
# If host is provided, automatically enable MQTT
if mqtt_config.host:
mqtt_config.enabled = True
@@ -117,6 +111,5 @@ def load_config_from_json(config_path: Path) -> Config:
mqtt=mqtt_config,
)
# Logger is needed for the load function
import logging
_LOGGER = logging.getLogger(__name__)
linux_voice_assistant/microwakeword.py
-189
View File
@@ -1,189 +0,0 @@
import ctypes
import json
import statistics
from collections import deque
from collections.abc import Iterable
from pathlib import Path
from typing import Deque, List, Union
import numpy as np
from pymicro_features import MicroFrontend
from .wakeword import TfLiteWakeWord
SAMPLES_PER_SECOND = 16000
SAMPLES_PER_CHUNK = 160 # 10ms
BYTES_PER_SAMPLE = 2 # 16-bit
BYTES_PER_CHUNK = SAMPLES_PER_CHUNK * BYTES_PER_SAMPLE
SECONDS_PER_CHUNK = SAMPLES_PER_CHUNK / SAMPLES_PER_SECOND
STRIDE = 3
class MicroWakeWord(TfLiteWakeWord):
def __init__(
self,
id: str, # pylint: disable=redefined-builtin
wake_word: str,
tflite_model: Union[str, Path],
probability_cutoff: float,
sliding_window_size: int,
trained_languages: List[str],
libtensorflowlite_c_path: Union[str, Path],
) -> None:
TfLiteWakeWord.__init__(self, libtensorflowlite_c_path)
self.id = id
self.wake_word = wake_word
self.tflite_model = tflite_model
self.probability_cutoff = probability_cutoff
self.sliding_window_size = sliding_window_size
self.trained_languages = trained_languages
self.is_active = True
# Load the model and create interpreter
self.model_path = str(Path(tflite_model).resolve()).encode("utf-8")
self._load_model()
self._features: List[np.ndarray] = []
self._probabilities: Deque[float] = deque(maxlen=self.sliding_window_size)
self._audio_buffer = bytes()
def _load_model(self) -> None:
self.model = self.lib.TfLiteModelCreateFromFile(self.model_path)
self.interpreter = self.lib.TfLiteInterpreterCreate(self.model, None)
self.lib.TfLiteInterpreterAllocateTensors(self.interpreter)
# Access input and output tensor
self.input_tensor = self.lib.TfLiteInterpreterGetInputTensor(
self.interpreter, 0
)
self.output_tensor = self.lib.TfLiteInterpreterGetOutputTensor(
self.interpreter, 0
)
# Get quantization parameters
input_q = self.lib.TfLiteTensorQuantizationParams(self.input_tensor)
output_q = self.lib.TfLiteTensorQuantizationParams(self.output_tensor)
self.input_scale, self.input_zero_point = input_q.scale, input_q.zero_point
self.output_scale, self.output_zero_point = output_q.scale, output_q.zero_point
@staticmethod
def from_config(
config_path: Union[str, Path],
libtensorflowlite_c_path: Union[str, Path],
) -> "MicroWakeWord":
"""Load a microWakeWord model from a JSON config file.
Parameters
----------
config_path: str or Path
Path to JSON configuration file
"""
config_path = Path(config_path)
with open(config_path, "r", encoding="utf-8") as config_file:
config = json.load(config_file)
micro_config = config["micro"]
return MicroWakeWord(
id=Path(config["model"]).stem,
wake_word=config["wake_word"],
tflite_model=config_path.parent / config["model"],
probability_cutoff=micro_config["probability_cutoff"],
sliding_window_size=micro_config["sliding_window_size"],
trained_languages=micro_config.get("trained_languages", []),
libtensorflowlite_c_path=libtensorflowlite_c_path,
)
def process_streaming(self, features: np.ndarray) -> bool:
self._features.append(features)
if len(self._features) < STRIDE:
# Not enough windows
return False
# Allocate and quantize input data
quant_features = np.round(
np.concatenate(self._features, axis=1) / self.input_scale
+ self.input_zero_point
).astype(np.uint8)
# Stride instead of rolling
self._features.clear()
# Set tensor
quant_ptr = quant_features.ctypes.data_as(ctypes.c_void_p)
self.lib.TfLiteTensorCopyFromBuffer(
self.input_tensor, quant_ptr, quant_features.nbytes
)
# Run inference
self.lib.TfLiteInterpreterInvoke(self.interpreter)
# Read output
output_bytes = self.lib.TfLiteTensorByteSize(self.output_tensor)
output_data = np.empty(output_bytes, dtype=np.uint8)
self.lib.TfLiteTensorCopyToBuffer(
self.output_tensor,
output_data.ctypes.data_as(ctypes.c_void_p),
output_bytes,
)
# Dequantize output
result = (
output_data.astype(np.float32) - self.output_zero_point
) * self.output_scale
self._probabilities.append(result.item())
if len(self._probabilities) < self.sliding_window_size:
# Not enough probabilities
return False
if statistics.mean(self._probabilities) > self.probability_cutoff:
return True
return False
# -----------------------------------------------------------------------------
class MicroWakeWordFeatures(TfLiteWakeWord):
def __init__(
self,
libtensorflowlite_c_path: Union[str, Path],
) -> None:
TfLiteWakeWord.__init__(self, libtensorflowlite_c_path)
self._audio_buffer = bytes()
self._frontend = MicroFrontend()
def process_streaming(self, audio_bytes: bytes) -> Iterable[np.ndarray]:
self._audio_buffer += audio_bytes
if len(self._audio_buffer) < BYTES_PER_CHUNK:
# Not enough audio to get features
return
audio_buffer_idx = 0
while (audio_buffer_idx + BYTES_PER_CHUNK) <= len(self._audio_buffer):
# Process chunk
chunk_bytes = self._audio_buffer[
audio_buffer_idx : audio_buffer_idx + BYTES_PER_CHUNK
]
frontend_result = self._frontend.ProcessSamples(chunk_bytes)
audio_buffer_idx += frontend_result.samples_read * BYTES_PER_SAMPLE
if not frontend_result.features:
# Not enough audio for a full window
continue
yield np.array(frontend_result.features).reshape(
(1, 1, len(frontend_result.features))
)
# Remove processed audio
self._audio_buffer = self._audio_buffer[audio_buffer_idx:]
linux_voice_assistant/models.py
+373 -99
View File
@@ -1,111 +1,385 @@
"""Shared models."""
#!/usr/bin/env python3
import argparse
import asyncio
import json
import logging
from dataclasses import asdict, dataclass, field
from enum import Enum
import sys
import threading
import time
from pathlib import Path
from queue import Queue
from typing import TYPE_CHECKING, Dict, List, Optional, Union
from typing import Dict, List, Optional, Set, Union
from .event_bus import EventBus
import numpy as np
import soundcard as sc
from pymicro_wakeword import MicroWakeWord, MicroWakeWordFeatures
from pyopen_wakeword import OpenWakeWord, OpenWakeWordFeatures
if TYPE_CHECKING:
from .entity import ESPHomeEntity, MediaPlayerEntity
from .microwakeword import MicroWakeWord
from .mpv_player import MpvMediaPlayer
from .openwakeword import OpenWakeWord
from .satellite import VoiceSatelliteProtocol
from .config import Config, load_config_from_json
from .event_bus import EventBus, EventHandler, subscribe
from .led_controller import LedController
from .mqtt_controller import MqttController
from .models import AvailableWakeWord, Preferences, ServerState, WakeWordType
from .mpv_player import MpvMediaPlayer
from .satellite import VoiceSatelliteProtocol
from .util import get_mac
from .zeroconf import HomeAssistantZeroconf
_LOGGER = logging.getLogger(__name__)
_MODULE_DIR = Path(__file__).parent
_REPO_DIR = _MODULE_DIR.parent
# -----------------------------------------------------------------------------
class MicMuteHandler(EventHandler):
"""Event handler for mic mute switch."""
def __init__(self, state, mqtt_controller: Optional[MqttController]):
super().__init__(state)
self.mqtt_controller = mqtt_controller
@subscribe
def set_mic_mute(self, data: dict):
is_muted = data.get("state", False)
if self.state.mic_muted != is_muted:
self.state.mic_muted = is_muted
_LOGGER.debug("Mic muted = %s", is_muted)
if self.mqtt_controller:
self.mqtt_controller.publish_mute_state(is_muted)
if is_muted:
self.state.event_bus.publish("mic_muted")
else:
self.state.event_bus.publish("mic_unmuted")
# -----------------------------------------------------------------------------
class SatelliteState(str, Enum):
"""Voice satellite state."""
STARTING = "starting" # <-- ADDED
IDLE = "idle"
LISTENING = "listening" # Wake word detected
THINKING = "thinking" # VAD, waiting for STT/intent
RESPONDING = "responding" # Playing TTS
ERROR = "error"
async def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument(
"-c",
"--config",
type=Path,
required=False,
default=_MODULE_DIR / "config.json",
help="Path to configuration.json file (default: linux_voice_assistant/config.json)",
)
parser.add_argument(
"--debug",
action="store_true",
help="Enable debug logging (overrides config file)",
)
parser.add_argument(
"--list-input-devices",
action="store_true",
help="List audio input devices and exit",
)
parser.add_argument(
"--list-output-devices",
action="store_true",
help="List audio output devices and exit",
)
args = parser.parse_args()
# --- Handle List Devices ---
if args.list_input_devices:
print("Input devices")
print("=" * 13)
for idx, mic in enumerate(sc.all_microphones(include_loopback=False)):
print(f"[{idx}]", mic.name)
return
if args.list_output_devices:
player = MpvMediaPlayer(loop=None)
print("Output devices")
print("=" * 14)
try:
for speaker in player.player.audio_device_list:
print(speaker["name"] + ":", speaker["description"])
except Exception as e:
_LOGGER.error("Failed to list output devices: %s", e)
return
# --- Load Configuration ---
config_path = args.config
if not config_path.is_absolute():
config_path = _REPO_DIR / config_path
config = load_config_from_json(config_path)
if args.debug:
config.app.debug = True
logging.basicConfig(level=logging.DEBUG if config.app.debug else logging.INFO)
_LOGGER.info("Loading configuration from: %s", config_path)
_LOGGER.debug("Configuration loaded: %s", config)
loop = asyncio.get_running_loop()
event_bus = EventBus()
# --- Create Download Dir ---
download_dir = _REPO_DIR / config.wake_word.download_dir
download_dir.mkdir(parents=True, exist_ok=True)
_LOGGER.debug("Download directory: %s", download_dir)
# --- Load Preferences ---
preferences_path = _REPO_DIR / config.app.preferences_file
if preferences_path.exists():
with open(preferences_path, "r", encoding="utf-8") as preferences_file:
preferences_dict = json.load(preferences_file)
preferences = Preferences(**preferences_dict)
else:
preferences = Preferences()
preferences.num_leds = getattr(preferences, 'num_leds', config.led.num_leds)
# --- Load Wake Words ---
available_wake_words: Dict[str, AvailableWakeWord] = {}
if not config.wake_word.directories:
config.wake_word.directories = ["wakewords", "wakewords/openWakeWord"]
# Add download directory to search path
wake_word_dirs = [_REPO_DIR / d for d in config.wake_word.directories]
wake_word_dirs.append(download_dir / "external_wake_words") # <-- ADDED
for wake_word_dir in wake_word_dirs:
if not wake_word_dir.exists():
continue
for model_config_path in wake_word_dir.glob("*.json"):
model_id = model_config_path.stem
if model_id == config.wake_word.stop_model:
continue
with open(model_config_path, "r", encoding="utf-8") as model_config_file:
model_config = json.load(model_config_file)
model_type = WakeWordType(model_config["type"])
if model_type == WakeWordType.OPEN_WAKE_WORD:
wake_word_path = model_config_path.parent / model_config["model"]
else:
wake_word_path = model_config_path
available_wake_words[model_id] = AvailableWakeWord(
id=model_id,
type=model_type,
wake_word=model_config["wake_word"],
trained_languages=model_config.get("trained_languages", []),
wake_word_path=wake_word_path,
)
_LOGGER.debug("Available wake words: %s", list(sorted(available_wake_words.keys())))
# --- Load Active Wake Word Models ---
active_wake_words: Set[str] = set()
wake_models: Dict[str, Union[MicroWakeWord, OpenWakeWord]] = {}
if preferences.active_wake_words:
for wake_word_id in preferences.active_wake_words:
wake_word = available_wake_words.get(wake_word_id)
if wake_word is None: continue
_LOGGER.debug("Loading wake model: %s", wake_word_id)
wake_models[wake_word_id] = wake_word.load()
active_wake_words.add(wake_word_id)
if not wake_models:
wake_word_id = config.wake_word.model
wake_word = available_wake_words.get(wake_word_id)
if wake_word:
_LOGGER.debug("Loading wake model: %s", wake_word_id)
wake_models[wake_word_id] = wake_word.load()
active_wake_words.add(wake_word_id)
# --- Load Stop Model ---
stop_model: Optional[MicroWakeWord] = None
for ww_dir_str in config.wake_word.directories:
wake_word_dir = _REPO_DIR / ww_dir_str
stop_config_path = wake_word_dir / f"{config.wake_word.stop_model}.json"
if not stop_config_path.exists(): continue
_LOGGER.debug("Loading stop model: %s", stop_config_path)
stop_model = MicroWakeWord.from_config(stop_config_path)
break
assert stop_model is not None
# --- Resolve Microphone ---
if config.audio.input_device is not None:
try:
input_device_idx = int(config.audio.input_device)
mic = sc.all_microphones(include_loopback=False)[input_device_idx]
except (ValueError, IndexError):
mic = sc.get_microphone(config.audio.input_device)
else:
mic = sc.default_microphone()
if mic is None:
_LOGGER.critical("No microphone found.")
sys.exit(1)
_LOGGER.info("Using audio input device: %s", mic.name)
# --- Initialize Media Players ---
music_player = MpvMediaPlayer(
loop=loop,
device=config.audio.output_device,
initial_volume=preferences.volume_level
)
tts_player = MpvMediaPlayer(
loop=loop,
device=config.audio.output_device,
initial_volume=preferences.volume_level
)
# --- Create Global Server State ---
state = ServerState(
name=config.app.name,
mac_address=get_mac(),
entities=[],
available_wake_words=available_wake_words,
wake_words=wake_models,
active_wake_words=active_wake_words,
stop_word=stop_model,
music_player=music_player,
tts_player=tts_player,
wakeup_sound=str(_REPO_DIR / config.app.wakeup_sound),
timer_finished_sound=str(_REPO_DIR / config.app.timer_finished_sound),
preferences=preferences,
preferences_path=preferences_path,
event_bus=event_bus,
loop=loop,
download_dir=download_dir, # <-- ADDED
refractory_seconds=config.wake_word.refractory_seconds,
)
# --- Initialize Controllers ---
led_controller = LedController(
state,
led_type=config.led.led_type,
interface=config.led.interface,
clock_pin=config.led.clock_pin,
data_pin=config.led.data_pin,
num_leds=state.preferences.num_leds,
)
mqtt_controller: Optional[MqttController] = None
if config.mqtt.enabled:
mqtt_controller = MqttController(
state=state,
host=config.mqtt.host,
port=config.mqtt.port,
username=config.mqtt.username,
password=config.mqtt.password
)
mqtt_controller.start()
mic_mute_handler = MicMuteHandler(state, mqtt_controller)
# --- Start Audio Processing Thread ---
process_audio_thread = threading.Thread(
target=process_audio,
args=(state, mic, config.audio.input_block_size),
daemon=True
)
process_audio_thread.start()
# --- Start ESPHome Server ---
server = await loop.create_server(
lambda: VoiceSatelliteProtocol(state),
host=config.esphome.host,
port=config.esphome.port
)
discovery = HomeAssistantZeroconf(port=config.esphome.port, name=config.app.name)
await discovery.register_server()
# --- Run Forever ---
try:
async with server:
_LOGGER.info("Server started (host=%s, port=%s)", config.esphome.host, config.esphome.port)
await server.serve_forever()
except KeyboardInterrupt:
pass
finally:
if mqtt_controller:
mqtt_controller.stop()
state.mic_muted = True
process_audio_thread.join()
_LOGGER.debug("Server stopped")
def process_audio(state: ServerState, mic, block_size: int):
"""Process audio chunks from the microphone in a separate thread."""
wake_words: List[Union[MicroWakeWord, OpenWakeWord]] = []
micro_features: Optional[MicroWakeWordFeatures] = None
micro_inputs: List[np.ndarray] = []
oww_features: Optional[OpenWakeWordFeatures] = None
oww_inputs: List[np.ndarray] = []
has_oww = False
last_active: Optional[float] = None
try:
_LOGGER.debug("Opening audio input device: %s", mic.name)
with mic.recorder(samplerate=16000, channels=1, blocksize=block_size) as mic_in:
while True:
if state.mic_muted:
time.sleep(0.1)
continue
audio_chunk_array = mic_in.record(block_size).reshape(-1)
audio_chunk = (
(np.clip(audio_chunk_array, -1.0, 1.0) * 32767.0)
.astype("<i2")
.tobytes()
)
if state.satellite is None:
continue
if (not wake_words) or (state.wake_words_changed and state.wake_words):
state.wake_words_changed = False
wake_words = [
ww for ww in state.wake_words.values()
if ww.id in state.active_wake_words
]
has_oww = any(isinstance(ww, OpenWakeWord) for ww in wake_words)
if micro_features is None:
micro_features = MicroWakeWordFeatures()
if has_oww and (oww_features is None):
oww_features = OpenWakeWordFeatures.from_builtin()
try:
state.satellite.handle_audio(audio_chunk)
assert micro_features is not None
micro_inputs.clear(); micro_inputs.extend(micro_features.process_streaming(audio_chunk))
if has_oww:
assert oww_features is not None
oww_inputs.clear(); oww_inputs.extend(oww_features.process_streaming(audio_chunk))
for wake_word in wake_words:
activated = False
if isinstance(wake_word, MicroWakeWord):
if any(wake_word.process_streaming(mi) for mi in micro_inputs): activated = True
elif isinstance(wake_word, OpenWakeWord):
if any(p > 0.5 for oi in oww_inputs for p in wake_word.process_streaming(oi)): activated = True
if activated:
now = time.monotonic()
if (last_active is None) or ((now - last_active) > state.refractory_seconds):
state.satellite.wakeup(wake_word)
last_active = now
stopped = False
for micro_input in micro_inputs:
if state.stop_word.process_streaming(micro_input):
stopped = True
if stopped and (state.stop_word.id in state.active_wake_words):
state.satellite.stop()
except Exception: _LOGGER.exception("Unexpected error handling audio")
except Exception as e:
_LOGGER.critical("Error in audio processing thread: %s", e)
state.loop.call_soon_threadsafe(state.loop.stop)
class WakeWordType(str, Enum):
MICRO_WAKE_WORD = "micro"
OPEN_WAKE_WORD = "openWakeWord"
@dataclass
class AvailableWakeWord:
id: str
type: WakeWordType
wake_word: str
trained_languages: List[str]
config_path: Path
def load(
self, libtensorflowlite_c_path: Path
) -> "Union[MicroWakeWord, OpenWakeWord]":
if self.type == WakeWordType.MICRO_WAKE_WORD:
from .microwakeword import MicroWakeWord
return MicroWakeWord.from_config(
config_path=self.config_path,
libtensorflowlite_c_path=libtensorflowlite_c_path,
)
if self.type == WakeWordType.OPEN_WAKE_WORD:
from .openwakeword import OpenWakeWord
return OpenWakeWord.from_config(
config_path=self.config_path,
libtensorflowlite_c_path=libtensorflowlite_c_path,
)
raise ValueError(f"Unexpected wake word type: {self.type}")
@dataclass
class Preferences:
active_wake_words: List[str] = field(default_factory=list)
volume_level: float = 1.0
num_leds: int = 3
@dataclass
class ServerState:
# --- Fields WITHOUT default values ---
name: str
mac_address: str
audio_queue: "Queue[Optional[bytes]]"
entities: "List[ESPHomeEntity]"
available_wake_words: "Dict[str, AvailableWakeWord]"
wake_words: "Dict[str, Union[MicroWakeWord, OpenWakeWord]]"
stop_word: "MicroWakeWord"
music_player: "MpvMediaPlayer"
tts_player: "MpvMediaPlayer"
wakeup_sound: str
timer_finished_sound: str
preferences: Preferences
preferences_path: Path
libtensorflowlite_c_path: Path
event_bus: EventBus
loop: asyncio.AbstractEventLoop
oww_melspectrogram_path: Path
oww_embedding_path: Path
# --- Fields WITH default values ---
satellite: "Optional[VoiceSatelliteProtocol]" = None
wake_words_changed: bool = False
refractory_seconds: float = 2.0
mic_muted: bool = False
def save_preferences(self) -> None:
"""Save preferences as JSON."""
_LOGGER.debug("Saving preferences: %s", self.preferences_path)
self.preferences_path.parent.mkdir(parents=True, exist_ok=True)
with open(self.preferences_path, "w", encoding="utf-8") as preferences_file:
json.dump(
asdict(self.preferences), preferences_file, ensure_ascii=False, indent=4
)
if __name__ == "__main__":
print("--- __main__ block executing ---")
asyncio.run(main())
linux_voice_assistant/openwakeword.py
-293
View File
@@ -1,293 +0,0 @@
"""openWakeWord implementation."""
import ctypes as C
import json
from collections.abc import Iterable
from pathlib import Path
from typing import Final, Union
import numpy as np
from .wakeword import TfLiteWakeWord
c_void_p = C.c_void_p
c_int32 = C.c_int32
c_size_t = C.c_size_t
BATCH_SIZE: Final = 1
AUTOFILL_SECONDS: Final = 8
MAX_SECONDS: Final = 10
SAMPLE_RATE: Final = 16000 # 16Khz
_MAX_SAMPLES: Final = MAX_SECONDS * SAMPLE_RATE
SAMPLES_PER_CHUNK: Final = 1280 # 80 ms @ 16Khz
MS_PER_CHUNK: Final = SAMPLES_PER_CHUNK // SAMPLE_RATE
# window = 400, hop length = 160
MELS_PER_SECOND: Final = 97
MAX_MELS: Final = MAX_SECONDS * MELS_PER_SECOND
MEL_SAMPLES: Final = 1760
NUM_MELS: Final = 32
EMB_FEATURES: Final = 76 # 775 ms
EMB_STEP: Final = 8
MAX_EMB: Final = MAX_SECONDS * EMB_STEP
WW_FEATURES: Final = 96
MEL_SHAPE: Final = (BATCH_SIZE, MEL_SAMPLES)
EMB_SHAPE: Final = (BATCH_SIZE, EMB_FEATURES, NUM_MELS, 1)
# melspec = [batch x samples (min: 1280)] => [batch x 1 x window x mels (32)]
# stft window size: 25ms (400)
# stft window step: 10ms (160)
# mel band limits: 60Hz - 3800Hz
# mel frequency bins: 32
#
# embedding = [batch x window x mels (32) x 1] => [batch x 1 x 1 x features (96)]
# ww = [batch x window x features (96)] => [batch x probability]
class OpenWakeWord(TfLiteWakeWord):
def __init__(
self,
id: str, # pylint: disable=redefined-builtin
wake_word: str,
tflite_model: Union[str, Path],
libtensorflowlite_c_path: Union[str, Path],
):
TfLiteWakeWord.__init__(self, libtensorflowlite_c_path)
self.id = id
self.wake_word = wake_word
self.tflite_model = tflite_model
self.is_active = True
# Load the model and create interpreter
self.model_path = str(Path(tflite_model).resolve()).encode("utf-8")
self.model = self.lib.TfLiteModelCreateFromFile(self.model_path)
self.interpreter = self.lib.TfLiteInterpreterCreate(self.model, None)
self.lib.TfLiteInterpreterAllocateTensors(self.interpreter)
self.input_tensor = self.lib.TfLiteInterpreterGetInputTensor(
self.interpreter, c_int32(0)
)
self.output_tensor = self.lib.TfLiteInterpreterGetOutputTensor(
self.interpreter, c_int32(0)
)
num_input_dims = self.lib.TfLiteTensorNumDims(self.input_tensor)
input_shape = [
self.lib.TfLiteTensorDim(self.input_tensor, i)
for i in range(num_input_dims)
]
self.input_windows = input_shape[1]
self.new_embeddings: int = 0
self.embeddings: np.ndarray = np.zeros(
shape=(MAX_EMB, WW_FEATURES), dtype=np.float32
)
def process_streaming(self, embeddings: np.ndarray) -> Iterable[float]:
"""Generate probabilities from embeddings."""
num_embedding_windows = embeddings.shape[2]
# Shift
self.embeddings[:-num_embedding_windows] = self.embeddings[
num_embedding_windows:
]
# Overwrite
self.embeddings[-num_embedding_windows:] = embeddings[0, 0, :, :]
self.new_embeddings = min(
len(self.embeddings),
self.new_embeddings + num_embedding_windows,
)
while self.new_embeddings >= self.input_windows:
emb_tensor = np.zeros(
shape=(1, self.input_windows, WW_FEATURES),
dtype=np.float32,
)
emb_tensor[0, :] = self.embeddings[
-self.new_embeddings : len(self.embeddings)
- self.new_embeddings
+ self.input_windows
]
self.new_embeddings = max(0, self.new_embeddings - 1)
# Run inference
emb_ptr = emb_tensor.ctypes.data_as(c_void_p)
self.lib.TfLiteTensorCopyFromBuffer(
self.input_tensor, emb_ptr, c_size_t(emb_tensor.nbytes)
)
self.lib.TfLiteInterpreterInvoke(self.interpreter)
output_bytes = self.lib.TfLiteTensorByteSize(self.output_tensor)
probs = np.empty(
output_bytes // np.dtype(np.float32).itemsize, dtype=np.float32
)
self.lib.TfLiteTensorCopyToBuffer(
self.output_tensor,
probs.ctypes.data_as(c_void_p),
c_size_t(output_bytes),
)
yield probs.item()
@staticmethod
def from_config(
config_path: Union[str, Path],
libtensorflowlite_c_path: Union[str, Path],
) -> "OpenWakeWord":
config_path = Path(config_path)
with open(config_path, "r", encoding="utf-8") as config_file:
config = json.load(config_file)
return OpenWakeWord(
id=Path(config["model"]).stem,
wake_word=config["wake_word"],
tflite_model=config_path.parent / config["model"],
libtensorflowlite_c_path=libtensorflowlite_c_path,
)
# -----------------------------------------------------------------------------
class OpenWakeWordFeatures(TfLiteWakeWord):
def __init__(
self,
melspectrogram_model: Union[str, Path],
embedding_model: Union[str, Path],
libtensorflowlite_c_path: Union[str, Path],
) -> None:
TfLiteWakeWord.__init__(self, libtensorflowlite_c_path)
# Melspectrogram
mel_path = str(Path(melspectrogram_model).resolve()).encode("utf-8")
self.mel_model = self.lib.TfLiteModelCreateFromFile(mel_path)
self.mel_interpreter = self.lib.TfLiteInterpreterCreate(self.mel_model, None)
mels_dims = (c_int32 * len(MEL_SHAPE))(*MEL_SHAPE)
self.lib.TfLiteInterpreterResizeInputTensor(
self.mel_interpreter,
c_int32(0),
mels_dims,
c_int32(len(MEL_SHAPE)),
)
self.lib.TfLiteInterpreterAllocateTensors(self.mel_interpreter)
self.mel_input_tensor = self.lib.TfLiteInterpreterGetInputTensor(
self.mel_interpreter, c_int32(0)
)
self.mel_output_tensor = self.lib.TfLiteInterpreterGetOutputTensor(
self.mel_interpreter, c_int32(0)
)
# Embedding
emb_path = str(Path(embedding_model).resolve()).encode("utf-8")
self.emb_model = self.lib.TfLiteModelCreateFromFile(emb_path)
self.emb_interpreter = self.lib.TfLiteInterpreterCreate(self.emb_model, None)
emb_dims = (c_int32 * len(EMB_SHAPE))(*EMB_SHAPE)
self.lib.TfLiteInterpreterResizeInputTensor(
self.emb_interpreter,
c_int32(0),
emb_dims,
c_int32(len(EMB_SHAPE)),
)
self.lib.TfLiteInterpreterAllocateTensors(self.emb_interpreter)
self.emb_input_tensor = self.lib.TfLiteInterpreterGetInputTensor(
self.emb_interpreter, c_int32(0)
)
self.emb_output_tensor = self.lib.TfLiteInterpreterGetOutputTensor(
self.emb_interpreter, c_int32(0)
)
# State
self.new_audio_samples: int = AUTOFILL_SECONDS * SAMPLE_RATE
self.audio: np.ndarray = np.zeros(shape=(_MAX_SAMPLES,), dtype=np.float32)
self.new_mels: int = 0
self.mels: np.ndarray = np.zeros(shape=(MAX_MELS, NUM_MELS), dtype=np.float32)
def process_streaming(self, audio_chunk: bytes) -> Iterable[np.ndarray]:
"""Generate embeddings from audio."""
chunk_array = np.frombuffer(audio_chunk, dtype=np.int16).astype(np.float32)
# Shift samples left
self.audio[: -len(chunk_array)] = self.audio[len(chunk_array) :]
# Add new samples to end
self.audio[-len(chunk_array) :] = chunk_array
self.new_audio_samples = min(
len(self.audio),
self.new_audio_samples + len(chunk_array),
)
while self.new_audio_samples >= MEL_SAMPLES:
audio_tensor = np.zeros(shape=(BATCH_SIZE, MEL_SAMPLES), dtype=np.float32)
audio_tensor[0, :] = self.audio[
-self.new_audio_samples : len(self.audio)
- self.new_audio_samples
+ MEL_SAMPLES
]
audio_tensor = np.ascontiguousarray(audio_tensor)
self.new_audio_samples = max(0, self.new_audio_samples - SAMPLES_PER_CHUNK)
# Run inference (mels)
audio_ptr = audio_tensor.ctypes.data_as(c_void_p)
self.lib.TfLiteTensorCopyFromBuffer(
self.mel_input_tensor, audio_ptr, c_size_t(audio_tensor.nbytes)
)
self.lib.TfLiteInterpreterInvoke(self.mel_interpreter)
mels_output_bytes = self.lib.TfLiteTensorByteSize(self.mel_output_tensor)
mels = np.empty(
mels_output_bytes // np.dtype(np.float32).itemsize, dtype=np.float32
)
self.lib.TfLiteTensorCopyToBuffer(
self.mel_output_tensor,
mels.ctypes.data_as(c_void_p),
c_size_t(mels_output_bytes),
)
mels = (mels / 10) + 2 # transform to fit embedding
mels = mels.reshape((1, 1, -1, NUM_MELS))
# Shift left
num_mel_windows = mels.shape[2]
self.mels[:-num_mel_windows] = self.mels[num_mel_windows:]
# Overwrite
self.mels[-num_mel_windows:] = mels[0, 0, :, :]
self.new_mels = min(len(self.mels), self.new_mels + num_mel_windows)
while self.new_mels >= EMB_FEATURES:
mels_tensor = np.ascontiguousarray(
np.zeros(shape=EMB_SHAPE, dtype=np.float32)
)
mels_tensor[0, :, :, 0] = self.mels[
-self.new_mels : len(self.mels) - self.new_mels + EMB_FEATURES, :
]
self.new_mels = max(0, self.new_mels - EMB_STEP)
# Run inference (embedding)
mels_ptr = mels_tensor.ctypes.data_as(c_void_p)
self.lib.TfLiteTensorCopyFromBuffer(
self.emb_input_tensor, mels_ptr, c_size_t(mels_tensor.nbytes)
)
self.lib.TfLiteInterpreterInvoke(self.emb_interpreter)
emb_output_bytes = self.lib.TfLiteTensorByteSize(self.emb_output_tensor)
emb = np.empty(
emb_output_bytes // np.dtype(np.float32).itemsize, dtype=np.float32
)
self.lib.TfLiteTensorCopyToBuffer(
self.emb_output_tensor,
emb.ctypes.data_as(c_void_p),
c_size_t(emb_output_bytes),
)
emb = emb.reshape((1, 1, -1, WW_FEATURES))
yield emb
linux_voice_assistant/satellite.py
+150 -48
View File
@@ -1,12 +1,18 @@
"""Voice satellite protocol."""
import hashlib
import logging
import posixpath
import shutil
import time
from collections.abc import Iterable
from pathlib import Path
from typing import Dict, Optional, Set, Union
from urllib.parse import urlparse, urlunparse
from urllib.request import urlopen
# pylint: disable=no-name-in-module
from aioesphomeapi.api_pb2 import ( # type: ignore[attr-defined]
from aioesphomeapi.api_pb2 import (
DeviceInfoRequest,
DeviceInfoResponse,
ListEntitiesDoneResponse,
@@ -19,6 +25,7 @@ from aioesphomeapi.api_pb2 import ( # type: ignore[attr-defined]
VoiceAssistantConfigurationRequest,
VoiceAssistantConfigurationResponse,
VoiceAssistantEventResponse,
VoiceAssistantExternalWakeWord, # <-- ADDED
VoiceAssistantRequest,
VoiceAssistantSetConfiguration,
VoiceAssistantTimerEventResponse,
@@ -30,12 +37,12 @@ from aioesphomeapi.model import (
VoiceAssistantTimerEventType,
)
from google.protobuf import message
from pymicro_wakeword import MicroWakeWord
from pyopen_wakeword import OpenWakeWord
from .api_server import APIServer
from .entity import MediaPlayerEntity
from .microwakeword import MicroWakeWord
from .models import ServerState, SatelliteState
from .openwakeword import OpenWakeWord
from .models import AvailableWakeWord, ServerState, SatelliteState, WakeWordType # <-- ADDED
from .util import call_all
_LOGGER = logging.getLogger(__name__)
@@ -60,8 +67,8 @@ class VoiceSatelliteProtocol(APIServer):
self.state.entities.append(self.media_player_entity)
# State machine
self._state: SatelliteState = SatelliteState.STARTING # <-- MODIFIED
self._is_streaming_audio: bool = False # Is HA requesting audio?
self._state: SatelliteState = SatelliteState.STARTING
self._is_streaming_audio: bool = False
# Announce/TTS state
self._tts_url: Optional[str] = None
@@ -71,19 +78,19 @@ class VoiceSatelliteProtocol(APIServer):
# Flags set by events, processed in _determine_final_state
self._run_end_received: bool = False
self._tts_end_received: bool = False
self._external_wake_words: Dict[str, VoiceAssistantExternalWakeWord] = {} # <-- ADDED
def _set_state(self, new_state: SatelliteState):
"""Manages state transitions and publishes events."""
if self._state == new_state:
return
_LOGGER.debug(f"State transition: {self._state} -> {new_state}")
self._state = new_state
# Publish events for LEDs, etc.
if new_state == SatelliteState.IDLE:
self.unduck()
self.state.stop_word.is_active = False
self.state.active_wake_words.discard(self.state.stop_word.id)
self.state.event_bus.publish("voice_idle")
elif new_state == SatelliteState.LISTENING:
self.duck()
@@ -91,7 +98,7 @@ class VoiceSatelliteProtocol(APIServer):
elif new_state == SatelliteState.THINKING:
self.state.event_bus.publish("voice_thinking")
elif new_state == SatelliteState.RESPONDING:
self.state.stop_word.is_active = True
self.state.active_wake_words.add(self.state.stop_word.id)
self.state.event_bus.publish("voice_responding")
elif new_state == SatelliteState.ERROR:
self.state.event_bus.publish("voice_error")
@@ -106,10 +113,8 @@ class VoiceSatelliteProtocol(APIServer):
self._tts_end_received = False
self._tts_url = data.get("url")
self._continue_conversation = False
# DO NOT transition to THINKING here. Stay in LISTENING.
elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_STT_START:
# This confirms we are in the LISTENING state
self._set_state(SatelliteState.LISTENING)
elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_STT_VAD_START:
@@ -117,11 +122,9 @@ class VoiceSatelliteProtocol(APIServer):
elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_STT_END:
self._is_streaming_audio = False
# THIS is when we move to THINKING
self._set_state(SatelliteState.THINKING)
elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_STT_VAD_END:
# We wait for STT_END before moving to THINKING
pass
elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_INTENT_END:
@@ -134,17 +137,15 @@ class VoiceSatelliteProtocol(APIServer):
elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_TTS_END:
self._tts_url = data.get("url")
self._tts_end_received = True
self.play_tts() # Play the final TTS URL
self.play_tts()
elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_RUN_END:
self._run_end_received = True
# If TTS is already done, or we're not speaking, we can finalize.
if self._state != SatelliteState.RESPONDING:
self._determine_final_state()
elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_ERROR:
self._set_state(SatelliteState.ERROR)
# Use a timer to reset to idle after error
self.state.loop.call_later(5.0, self._set_state, SatelliteState.IDLE)
@@ -156,7 +157,7 @@ class VoiceSatelliteProtocol(APIServer):
_LOGGER.debug("Timer event: type=%s", event_type.name)
if event_type == VoiceAssistantTimerEventType.VOICE_ASSISTANT_TIMER_FINISHED:
if not self._timer_finished:
self.state.stop_word.is_active = True
self.state.active_wake_words.add(self.state.stop_word.id)
self._timer_finished = True
self.duck()
self._play_timer_finished()
@@ -173,10 +174,10 @@ class VoiceSatelliteProtocol(APIServer):
if msg.preannounce_media_id:
urls.append(msg.preannounce_media_id)
urls.append(msg.media_id)
self.state.stop_word.is_active = True
self.state.active_wake_words.add(self.state.stop_word.id)
self._continue_conversation = msg.start_conversation
self.duck()
self._set_state(SatelliteState.RESPONDING) # Announce is a response
self._set_state(SatelliteState.RESPONDING)
yield from self.media_player_entity.play(
urls, announcement=True, done_callback=self._tts_finished
)
@@ -209,22 +210,44 @@ class VoiceSatelliteProtocol(APIServer):
if isinstance(msg, ListEntitiesRequest):
yield ListEntitiesDoneResponse()
elif isinstance(msg, VoiceAssistantConfigurationRequest):
yield VoiceAssistantConfigurationResponse(
available_wake_words=[
# --- MODIFIED ---
available_wake_words = [
VoiceAssistantWakeWord(
id=ww.id,
wake_word=ww.wake_word,
trained_languages=ww.trained_languages,
)
for ww in self.state.available_wake_words.values()
]
# Add external wake words
self._external_wake_words.clear()
for eww in msg.external_wake_words:
if eww.model_type != "micro":
_LOGGER.warning("Skipping external wake word %s (type=%s)", eww.id, eww.model_type)
continue
available_wake_words.append(
VoiceAssistantWakeWord(
id=ww.id,
wake_word=ww.wake_word,
trained_languages=ww.trained_languages,
id=eww.id,
wake_word=eww.wake_word,
trained_languages=eww.trained_languages,
)
for ww in self.state.available_wake_words.values()
],
)
self._external_wake_words[eww.id] = eww
yield VoiceAssistantConfigurationResponse(
available_wake_words=available_wake_words,
active_wake_words=[
ww.id for ww in self.state.wake_words.values() if ww.is_active
ww.id for ww in self.state.wake_words.values()
if ww.id in self.state.active_wake_words
],
max_active_wake_words=2,
)
# --- END MODIFIED ---
_LOGGER.info("Connected to Home Assistant")
self._set_state(SatelliteState.IDLE) # Connected, move to idle
self._set_state(SatelliteState.IDLE)
self.state.event_bus.publish("ha_connected")
elif isinstance(msg, VoiceAssistantSetConfiguration):
active_wake_words: Set[str] = set()
@@ -232,19 +255,34 @@ class VoiceSatelliteProtocol(APIServer):
if wake_word_id in self.state.wake_words:
active_wake_words.add(wake_word_id)
continue
model_info = self.state.available_wake_words.get(wake_word_id)
# --- ADDED ---
if not model_info:
continue
_LOGGER.debug("Loading wake word: %s", model_info.config_path)
self.state.wake_words[wake_word_id] = model_info.load(
self.state.libtensorflowlite_c_path
)
# Check external wake words (may require download)
external_wake_word = self._external_wake_words.get(wake_word_id)
if not external_wake_word:
_LOGGER.warning("Unknown wake word: %s", wake_word_id)
continue
model_info = self._download_external_wake_word(external_wake_word)
if not model_info:
continue # download failed
self.state.available_wake_words[wake_word_id] = model_info
# --- END ADDED ---
_LOGGER.debug("Loading wake word: %s", model_info.wake_word_path)
self.state.wake_words[wake_word_id] = model_info.load()
_LOGGER.info("Wake word set: %s", wake_word_id)
active_wake_words.add(wake_word_id)
break
for wake_word in self.state.wake_words.values():
wake_word.is_active = wake_word.id in active_wake_words
self.state.active_wake_words = active_wake_words
_LOGGER.debug("Active wake words: %s", active_wake_words)
self.state.preferences.active_wake_words = list(active_wake_words)
self.state.save_preferences()
self.state.wake_words_changed = True
@@ -256,7 +294,7 @@ class VoiceSatelliteProtocol(APIServer):
def wakeup(self, wake_word: Union[MicroWakeWord, OpenWakeWord]) -> None:
if self._state not in (SatelliteState.IDLE, SatelliteState.STARTING):
return # Already awake
return
if self._timer_finished:
self._timer_finished = False
@@ -264,7 +302,7 @@ class VoiceSatelliteProtocol(APIServer):
_LOGGER.debug("Stopping timer finished sound")
return
wake_word_phrase = wake_word.wake_word
wake_word_phrase = getattr(wake_word, "wake_word", "wake word")
_LOGGER.debug("Detected wake word: %s", wake_word_phrase)
self.send_messages(
[VoiceAssistantRequest(start=True, wake_word_phrase=wake_word_phrase)]
@@ -274,22 +312,23 @@ class VoiceSatelliteProtocol(APIServer):
self.state.tts_player.play(self.state.wakeup_sound)
def stop(self) -> None:
self.state.active_wake_words.discard(self.state.stop_word.id)
self.state.tts_player.stop()
if self._timer_finished:
self._timer_finished = False
_LOGGER.debug("Stopping timer finished sound")
else:
_LOGGER.debug("TTS response stopped manually")
self._tts_finished() # Manually trigger TTS finished logic
self._tts_finished()
def play_tts(self) -> None:
if not self._tts_url:
return
_LOGGER.debug("Playing TTS response: %s", self._tts_url)
self._set_state(SatelliteState.RESPONDING)
self.state.active_wake_words.add(self.state.stop_word.id)
self.state.tts_player.play(self._tts_url, done_callback=self._tts_finished)
self._tts_url = None # Clear after starting playback
self._tts_url = None
def duck(self) -> None:
_LOGGER.debug("Ducking music")
@@ -300,7 +339,6 @@ class VoiceSatelliteProtocol(APIServer):
self.state.music_player.unduck()
def _determine_final_state(self) -> None:
"""Called when both TTS and the pipeline run are finished."""
if self._continue_conversation:
self.send_messages([VoiceAssistantRequest(start=True)])
self._is_streaming_audio = True
@@ -312,18 +350,13 @@ class VoiceSatelliteProtocol(APIServer):
_LOGGER.debug("Final state determined")
def _tts_finished(self) -> None:
"""Callback when TTS audio finishes playing."""
self.state.active_wake_words.discard(self.state.stop_word.id)
self.send_messages([VoiceAssistantAnnounceFinished()])
_LOGGER.debug("TTS audio playback finished")
# If the run already ended, we can move to the final state.
# Otherwise, we wait for the RUN_END event.
if self._run_end_received:
self._determine_final_state()
else:
# We've finished speaking but the pipeline is still running
# (e.g., waiting for 'continue_conversation')
# Move to THINKING state
if self._state == SatelliteState.RESPONDING:
self._set_state(SatelliteState.THINKING)
@@ -339,6 +372,75 @@ class VoiceSatelliteProtocol(APIServer):
),
)
# --- ADDED ---
def _download_external_wake_word(
self, external_wake_word: VoiceAssistantExternalWakeWord
) -> Optional[AvailableWakeWord]:
eww_dir = self.state.download_dir / "external_wake_words"
eww_dir.mkdir(parents=True, exist_ok=True)
config_path = eww_dir / f"{external_wake_word.id}.json"
should_download_config = not config_path.exists()
# Check if we need to download the model file
model_path = eww_dir / f"{external_wake_word.id}.tflite"
should_download_model = True
if model_path.exists():
model_size = model_path.stat().st_size
if model_size == external_wake_word.model_size:
with open(model_path, "rb") as model_file:
model_hash = hashlib.sha256(model_file.read()).hexdigest()
if model_hash == external_wake_word.model_hash:
should_download_model = False
_LOGGER.debug(
"Model size and hash match for %s. Skipping download.",
external_wake_word.id,
)
if should_download_config or should_download_model:
# Download config
_LOGGER.debug("Downloading %s to %s", external_wake_word.url, config_path)
with urlopen(external_wake_word.url) as request:
if request.status != 200:
_LOGGER.warning(
"Failed to download: %s, status=%s",
external_wake_word.url,
request.status,
)
return None
with open(config_path, "wb") as model_file:
shutil.copyfileobj(request, model_file)
if should_download_model:
# Download model file
parsed_url = urlparse(external_wake_word.url)
parsed_url = parsed_url._replace(
path=posixpath.join(posixpath.dirname(parsed_url.path), model_path.name)
)
model_url = urlunparse(parsed_url)
_LOGGER.debug("Downloading %s to %s", model_url, model_path)
with urlopen(model_url) as request:
if request.status != 200:
_LOGGER.warning(
"Failed to download: %s, status=%s", model_url, request.status
)
return None
with open(model_path, "wb") as model_file:
shutil.copyfileobj(request, model_file)
return AvailableWakeWord(
id=external_wake_word.id,
type=WakeWordType.MICRO_WAKE_WORD,
wake_word=external_wake_word.wake_word,
trained_languages=external_wake_word.trained_languages,
wake_word_path=config_path, # micro-wakeword loads from config
)
# --- END ADDED ---
def connection_lost(self, exc):
super().connection_lost(exc)
self._set_state(SatelliteState.ERROR)
linux_voice_assistant/util.py
+4 -9
View File
@@ -1,22 +1,17 @@
"""Utility methods."""
import platform
import uuid
from collections.abc import Callable
from typing import Optional
def get_mac() -> str:
mac = uuid.getnode()
mac_str = ":".join(f"{(mac >> i) & 0xff:02x}" for i in range(40, -1, -8))
return mac_str
"""Get MAC address as a string."""
return ":".join(
f"{((uuid.getnode() >> i) & 0xFF):02x}" for i in range(0, 8 * 6, 8)
)[::-1]
def call_all(*callables: Optional[Callable[[], None]]) -> None:
for item in filter(None, callables):
item()
def is_arm() -> bool:
machine = platform.machine()
return ("arm" in machine) or ("aarch" in machine)
linux_voice_assistant/wakeword.py
-89
View File
@@ -1,89 +0,0 @@
"""Base class for wake words."""
import ctypes as C
import os
from pathlib import Path
from typing import Union
TfLiteStatus = C.c_int # kTfLiteOk == 0
class TfLiteQuantizationParams(C.Structure):
_fields_ = [("scale", C.c_float), ("zero_point", C.c_int32)]
c_void_p = C.c_void_p
c_int32 = C.c_int32
c_size_t = C.c_size_t
def _not_null(result, func, args):
if not result:
raise RuntimeError(f"{func.__name__} returned NULL")
return result
class TfLiteWakeWord:
def __init__(self, libtensorflowlite_c_path: Union[str, Path]):
self.libtensorflowlite_c_path = Path(libtensorflowlite_c_path)
# Load ONCE with RTLD_GLOBAL
self.lib = C.CDLL(
str(self.libtensorflowlite_c_path.resolve()), mode=os.RTLD_GLOBAL
)
lib = self.lib
# Model / Interpreter
lib.TfLiteModelCreateFromFile.argtypes = [C.c_char_p]
lib.TfLiteModelCreateFromFile.restype = c_void_p
lib.TfLiteInterpreterCreate.argtypes = [c_void_p, c_void_p]
lib.TfLiteInterpreterCreate.restype = c_void_p
lib.TfLiteInterpreterAllocateTensors.argtypes = [c_void_p]
lib.TfLiteInterpreterAllocateTensors.restype = TfLiteStatus
lib.TfLiteInterpreterInvoke.argtypes = [c_void_p]
lib.TfLiteInterpreterInvoke.restype = TfLiteStatus
# Tensors: get
lib.TfLiteInterpreterGetInputTensor.argtypes = [c_void_p, c_int32]
lib.TfLiteInterpreterGetInputTensor.restype = c_void_p
lib.TfLiteInterpreterGetInputTensor.errcheck = _not_null
lib.TfLiteInterpreterGetOutputTensor.argtypes = [c_void_p, c_int32]
lib.TfLiteInterpreterGetOutputTensor.restype = c_void_p
lib.TfLiteInterpreterGetOutputTensor.errcheck = _not_null
# Sizes / dims
lib.TfLiteTensorByteSize.argtypes = [c_void_p]
lib.TfLiteTensorByteSize.restype = c_size_t
lib.TfLiteTensorNumDims.argtypes = [c_void_p]
lib.TfLiteTensorNumDims.restype = c_int32
lib.TfLiteTensorDim.argtypes = [c_void_p, c_int32]
lib.TfLiteTensorDim.restype = c_int32
# Type & quant params
lib.TfLiteTensorType.argtypes = [c_void_p]
lib.TfLiteTensorType.restype = c_int32
lib.TfLiteTensorQuantizationParams.argtypes = [c_void_p]
lib.TfLiteTensorQuantizationParams.restype = TfLiteQuantizationParams
# Resize
lib.TfLiteInterpreterResizeInputTensor.argtypes = [
c_void_p,
c_int32,
C.POINTER(c_int32),
c_int32,
]
lib.TfLiteInterpreterResizeInputTensor.restype = TfLiteStatus
# Copy buffers
lib.TfLiteTensorCopyFromBuffer.argtypes = [c_void_p, c_void_p, c_size_t]
lib.TfLiteTensorCopyFromBuffer.restype = TfLiteStatus
lib.TfLiteTensorCopyToBuffer.argtypes = [c_void_p, c_void_p, c_size_t]
lib.TfLiteTensorCopyToBuffer.restype = TfLiteStatus
mypy.ini
+35 -6
View File
@@ -1,23 +1,52 @@
[mypy]
python_version = 3.9
warn_return_any = True
warn_unused_configs = True
[mypy-setuptools.*]
[mypy-aioesphomeapi._frame_helper.packets.*]
ignore_missing_imports = True
[mypy-aioesphomeapi.api_pb2.*]
ignore_missing_imports = True
[mypy-pymicro_features.*]
[mypy-pymicro_wakeword.*]
ignore_missing_imports = True
[mypy-sounddevice.*]
[mypy-pyopen_wakeword.*]
ignore_missing_imports = True
[mypy-soundcard.*]
ignore_missing_imports = True
[mypy-mpv.*]
ignore_missing_imports = True
[mypy-google.protobuf.*]
[mypy-zeroconf.*]
ignore_missing_imports = True
[mypy-aioesphomeapi.*]
[mypy-paho.mqtt.client.*]
ignore_missing_imports = True
[mypy-RPi.*]
ignore_missing_imports = True
[mypy-spidev.*]
ignore_missing_imports = True
[mypy-board.*]
ignore_missing_imports = True
[mypy-busio.*]
ignore_missing_imports = True
[mypy-gpiozero.*]
ignore_missing_imports = True
[mypy-adafruit_dotstar.*]
ignore_missing_imports = True
[mypy-neopixel.*]
ignore_missing_imports = True
[mypy-neopixel_spi.*]
ignore_missing_imports = True
pyproject.toml
+10 -9
View File
@@ -26,11 +26,12 @@ classifiers = [
requires-python = ">=3.9.0"
dependencies = [
"aioesphomeapi==42.7.0",
"sounddevice<1",
"soundcard<1",
"numpy>=2,<3",
"pymicro-features==1.0.0",
"pymicro-wakeword>=2,<3",
"pyopen-wakeword>=1,<2",
"python-mpv>=1,<2",
"zeroconf",
"zeroconf<1",
"paho-mqtt",
"adafruit-circuitpython-dotstar",
"adafruit-circuitpython-neopixel",
@@ -42,15 +43,15 @@ dependencies = [
[project.optional-dependencies]
dev = [
"black==24.8.0",
"flake8==7.2.0",
"mypy==1.14.0",
"pylint==3.2.7",
"pytest==8.3.5",
"black",
"flake8",
"mypy",
"pylint",
"pytest",
]
[project.urls]
"Source Code" = "http://github.com/OHF-Voice/linux-voice-assistant"
"Source Code" = "https://github.com/imonlinux/linux-voice-assistant"
[tool.setuptools]
platforms = ["any"]
wakewords/openWakeWord/embedding_model.tflite
BIN
View File
Binary file not shown.
wakewords/openWakeWord/melspectrogram.tflite
BIN
View File
Binary file not shown.