Rename to Linux Voice Assistant

This commit is contained in:
Michael Hansen
2025-08-26 09:30:58 -05:00
parent 551cb35735
commit 4356633ed6
14 changed files with 92 additions and 33 deletions
View File
+563
View File
@@ -0,0 +1,563 @@
#!/usr/bin/env python3
import argparse
import asyncio
import json
import logging
import threading
import time
import tempfile
import shutil
import posixpath
from collections.abc import Iterable
from dataclasses import dataclass, field
from pathlib import Path
from queue import Queue
from typing import Dict, List, Optional
from urllib.request import urlopen
from urllib.parse import urlparse, urlunparse
# pylint: disable=no-name-in-module
import sounddevice as sd
from aioesphomeapi.api_pb2 import ( # type: ignore[attr-defined]
DeviceInfoRequest,
DeviceInfoResponse,
ListEntitiesDoneResponse,
ListEntitiesRequest,
MediaPlayerCommandRequest,
SubscribeHomeAssistantStatesRequest,
VoiceAssistantAnnounceFinished,
VoiceAssistantAnnounceRequest,
VoiceAssistantAudio,
VoiceAssistantConfigurationRequest,
VoiceAssistantConfigurationResponse,
VoiceAssistantEventResponse,
VoiceAssistantRequest,
VoiceAssistantSetConfiguration,
VoiceAssistantTimerEventResponse,
VoiceAssistantWakeWord,
)
from aioesphomeapi.model import (
VoiceAssistantEventType,
VoiceAssistantFeature,
VoiceAssistantTimerEventType,
)
from google.protobuf import message
from .api_server import APIServer
from .entity import ESPHomeEntity, MediaPlayerEntity
from .microwakeword import MicroWakeWord
from .mpv_player import MpvMediaPlayer
from .util import call_all, get_mac, is_arm
_LOGGER = logging.getLogger(__name__)
_MODULE_DIR = Path(__file__).parent
_REPO_DIR = _MODULE_DIR.parent
_WAKEWORDS_DIR = _REPO_DIR / "wakewords"
_SOUNDS_DIR = _REPO_DIR / "sounds"
if is_arm():
_LIB_DIR = _REPO_DIR / "lib" / "linux_arm64"
else:
_LIB_DIR = _REPO_DIR / "lib" / "linux_amd64"
@dataclass
class AvailableWakeWord:
id: str
wake_word: str
trained_languages: List[str]
config_path: Path
url: Optional[str] = None
@dataclass
class ServerState:
name: str
mac_address: str
audio_queue: "Queue[Optional[bytes]]"
entities: List[ESPHomeEntity]
available_wake_words: Dict[str, AvailableWakeWord]
wake_word: MicroWakeWord
stop_word: MicroWakeWord
music_player: MpvMediaPlayer
tts_player: MpvMediaPlayer
wakeup_sound: str
timer_finished_sound: str
media_player_entity: Optional[MediaPlayerEntity] = None
satellite: "Optional[VoiceSatelliteProtocol]" = None
_external_wake_word_dir: Optional[tempfile.TemporaryDirectory] = None
@property
def external_wake_word_dir(self) -> Path:
if self._external_wake_word_dir is None:
self._external_wake_word_dir = tempfile.TemporaryDirectory()
return Path(self._external_wake_word_dir.name)
# -----------------------------------------------------------------------------
class VoiceSatelliteProtocol(APIServer):
def __init__(self, state: ServerState) -> None:
super().__init__(state.name)
self.state = state
self.state.satellite = self
if self.state.media_player_entity is None:
self.state.media_player_entity = MediaPlayerEntity(
server=self,
key=len(state.entities),
name="Media Player",
object_id="linux_voice_assistant_media_player",
music_player=state.music_player,
announce_player=state.tts_player,
)
self.state.entities.append(self.state.media_player_entity)
self._is_streaming_audio = False
self._tts_url: Optional[str] = None
self._tts_played = False
self._continue_conversation = False
self._timer_finished = False
def handle_voice_event(
self, event_type: VoiceAssistantEventType, data: Dict[str, str]
) -> None:
_LOGGER.debug("Voice event: type=%s, data=%s", event_type.name, data)
if event_type == VoiceAssistantEventType.VOICE_ASSISTANT_RUN_START:
self._tts_url = data.get("url")
self._tts_played = False
self._continue_conversation = False
elif event_type in (
VoiceAssistantEventType.VOICE_ASSISTANT_STT_VAD_END,
VoiceAssistantEventType.VOICE_ASSISTANT_STT_END,
):
self._is_streaming_audio = False
elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_INTENT_PROGRESS:
if data.get("tts_start_streaming") == "1":
# Start streaming early
self.play_tts()
elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_INTENT_END:
if data.get("continue_conversation") == "1":
self._continue_conversation = True
elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_TTS_END:
self._tts_url = data.get("url")
self.play_tts()
elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_RUN_END:
self._is_streaming_audio = False
if not self._tts_played:
self._tts_finished()
self._tts_played = False
# TODO: handle error
def handle_timer_event(
self,
event_type: VoiceAssistantTimerEventType,
msg: VoiceAssistantTimerEventResponse,
) -> None:
_LOGGER.debug("Timer event: type=%s", event_type.name)
if event_type == VoiceAssistantTimerEventType.VOICE_ASSISTANT_TIMER_FINISHED:
if not self._timer_finished:
self.state.stop_word.is_active = True
self._timer_finished = True
self.duck()
self._play_timer_finished()
def handle_message(self, msg: message.Message) -> Iterable[message.Message]:
if isinstance(msg, VoiceAssistantEventResponse):
# Pipeline event
data: Dict[str, str] = {}
for arg in msg.data:
data[arg.name] = arg.value
self.handle_voice_event(VoiceAssistantEventType(msg.event_type), data)
elif isinstance(msg, VoiceAssistantAnnounceRequest):
_LOGGER.debug("Announcing: %s", msg.text)
assert self.state.media_player_entity is not None
urls = []
if msg.preannounce_media_id:
urls.append(msg.preannounce_media_id)
urls.append(msg.media_id)
self.state.stop_word.is_active = True
self._continue_conversation = msg.start_conversation
self.duck()
yield from self.state.media_player_entity.play(
urls, announcement=True, done_callback=self._tts_finished
)
elif isinstance(msg, VoiceAssistantTimerEventResponse):
self.handle_timer_event(VoiceAssistantTimerEventType(msg.event_type), msg)
elif isinstance(msg, DeviceInfoRequest):
yield DeviceInfoResponse(
uses_password=False,
name=self.state.name,
mac_address=self.state.mac_address,
voice_assistant_feature_flags=(
VoiceAssistantFeature.VOICE_ASSISTANT
| VoiceAssistantFeature.API_AUDIO
| VoiceAssistantFeature.ANNOUNCE
| VoiceAssistantFeature.START_CONVERSATION
| VoiceAssistantFeature.TIMERS
),
)
elif isinstance(
msg,
(
ListEntitiesRequest,
SubscribeHomeAssistantStatesRequest,
MediaPlayerCommandRequest,
),
):
for entity in self.state.entities:
yield from entity.handle_message(msg)
if isinstance(msg, ListEntitiesRequest):
yield ListEntitiesDoneResponse()
elif isinstance(msg, VoiceAssistantConfigurationRequest):
external_wake_words = {
external_ww.wake_word.id: external_ww
for external_ww in msg.external_wake_words
if external_ww.model_type == "micro_wake_word"
}
for wake_word_id, external_ww in external_wake_words.items():
if wake_word_id in self.state.available_wake_words:
# Skip if already added
continue
self.state.available_wake_words[wake_word_id] = AvailableWakeWord(
id=external_ww.wake_word.id,
wake_word=external_ww.wake_word.wake_word,
trained_languages=external_ww.wake_word.trained_languages,
config_path=self.state.external_wake_word_dir
/ f"{external_ww.wake_word.id}.json",
url=external_ww.url,
)
yield VoiceAssistantConfigurationResponse(
available_wake_words=[
VoiceAssistantWakeWord(
id=ww.id,
wake_word=ww.wake_word,
trained_languages=ww.trained_languages,
)
for ww in self.state.available_wake_words.values()
],
active_wake_words=[self.state.wake_word.id],
max_active_wake_words=1,
)
_LOGGER.info("Connected to Home Assistant")
elif isinstance(msg, VoiceAssistantSetConfiguration):
# TODO: support multiple wake words
for wake_word_id in msg.active_wake_words:
if wake_word_id == self.state.wake_word.id:
# Already active
break
model_info = self.state.available_wake_words.get(wake_word_id)
if not model_info:
continue
if (not model_info.config_path.exists()) and model_info.url:
_LOGGER.debug("Downloading wake word config: %s", model_info.url)
with (
open(model_info.config_path, "wb") as config_file,
urlopen(model_info.url) as response,
):
if response.status != 200:
_LOGGER.error(
"Download failed with status: %s", response.status
)
continue
shutil.copyfileobj(response, config_file)
with open(
model_info.config_path, "r", encoding="utf-8"
) as config_file:
config_dict = json.load(config_file)
parsed_config_url = urlparse(model_info.url)
model_url = urlunparse(
parsed_config_url._replace(
path=posixpath.join(
posixpath.dirname(parsed_config_url.path),
config_dict["model"],
)
)
)
_LOGGER.debug("Downloading wake word model: %s", model_url)
model_path = model_info.config_path.parent / config_dict["model"]
with (
open(model_path, "wb") as model_file,
urlopen(model_url) as response,
):
if response.status != 200:
_LOGGER.error(
"Download failed with status: %s", response.status
)
continue
shutil.copyfileobj(response, model_file)
_LOGGER.debug("Loading wake word: %s", model_info.config_path)
self.state.wake_word = MicroWakeWord.from_config(
model_info.config_path,
self.state.wake_word.libtensorflowlite_c_path,
)
_LOGGER.info("Wake word set: %s", self.state.wake_word.wake_word)
break
def handle_audio(self, audio_chunk: bytes) -> None:
if not self._is_streaming_audio:
return
self.send_messages([VoiceAssistantAudio(data=audio_chunk)])
def wakeup(self) -> None:
if self._timer_finished:
# Stop timer instead
self._timer_finished = False
self.state.tts_player.stop()
_LOGGER.debug("Stopping timer finished sound")
return
wake_word_phrase = self.state.wake_word.wake_word
_LOGGER.debug("Detected wake word: %s", wake_word_phrase)
self.send_messages(
[VoiceAssistantRequest(start=True, wake_word_phrase=wake_word_phrase)]
)
self.duck()
self._is_streaming_audio = True
self.state.tts_player.play(self.state.wakeup_sound)
def stop(self) -> None:
self.state.stop_word.is_active = False
self.state.tts_player.stop()
if self._timer_finished:
self._timer_finished = False
_LOGGER.debug("Stopping timer finished sound")
else:
_LOGGER.debug("TTS response stopped manually")
self._tts_finished()
def play_tts(self) -> None:
if (not self._tts_url) or self._tts_played:
return
self._tts_played = True
_LOGGER.debug("Playing TTS response: %s", self._tts_url)
self.state.stop_word.is_active = True
self.state.tts_player.play(self._tts_url, done_callback=self._tts_finished)
def duck(self) -> None:
_LOGGER.debug("Ducking music")
self.state.music_player.duck()
def unduck(self) -> None:
_LOGGER.debug("Unducking music")
self.state.music_player.unduck()
def _tts_finished(self) -> None:
self.state.stop_word.is_active = False
self.send_messages([VoiceAssistantAnnounceFinished()])
if self._continue_conversation:
self.send_messages([VoiceAssistantRequest(start=True)])
self._is_streaming_audio = True
_LOGGER.debug("Continuing conversation")
else:
self.unduck()
_LOGGER.debug("TTS response finished")
def _play_timer_finished(self) -> None:
if not self._timer_finished:
self.unduck()
return
self.state.tts_player.play(
self.state.timer_finished_sound,
done_callback=lambda: call_all(
lambda: time.sleep(1.0), self._play_timer_finished
),
)
def connection_lost(self, exc):
super().connection_lost(exc)
_LOGGER.info("Disconnected from Home Assistant")
def process_audio(state: ServerState):
try:
while True:
audio_chunk = state.audio_queue.get()
if audio_chunk is None:
break
if state.satellite is None:
continue
try:
state.satellite.handle_audio(audio_chunk)
if state.wake_word.is_active and state.wake_word.process_streaming(
audio_chunk
):
state.satellite.wakeup()
if state.stop_word.is_active and state.stop_word.process_streaming(
audio_chunk
):
state.satellite.stop()
except Exception:
_LOGGER.exception("Unexpected error handling audio")
except Exception:
_LOGGER.exception("Unexpected error processing audio")
# -----------------------------------------------------------------------------
async def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--name", required=True)
parser.add_argument(
"--audio-input-device",
default="default",
help="sounddevice name for input device",
)
parser.add_argument("--audio-input-block-size", type=int, default=1024)
parser.add_argument("--audio-output-device", help="mpv name for output device")
parser.add_argument(
"--wake-word-dir",
default=_WAKEWORDS_DIR,
help="Directory with wake word models (.tflite) and configs (.json)",
)
parser.add_argument(
"--wake-model", default="okay_nabu", help="Id of active wake model"
)
parser.add_argument("--stop-model", default="stop", help="Id of stop model")
parser.add_argument(
"--wakeup-sound", default=str(_SOUNDS_DIR / "wake_word_triggered.flac")
)
parser.add_argument(
"--timer-finished-sound", default=str(_SOUNDS_DIR / "timer_finished.flac")
)
parser.add_argument(
"--host",
default="0.0.0.0",
help="Address for ESPHome server (default: 0.0.0.0)",
)
parser.add_argument(
"--port", type=int, default=6053, help="Port for ESPHome server (default: 6053)"
)
parser.add_argument(
"--debug", action="store_true", help="Print DEBUG messages to console"
)
args = parser.parse_args()
logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO)
_LOGGER.debug(args)
# Load available wake words
wake_word_dir = Path(args.wake_word_dir)
available_wake_words: Dict[str, AvailableWakeWord] = {}
for model_config_path in wake_word_dir.glob("*.json"):
model_id = model_config_path.stem
if model_id == args.stop_model:
# Don't show stop model as an available wake word
continue
with open(model_config_path, "r", encoding="utf-8") as model_config_file:
model_config = json.load(model_config_file)
available_wake_words[model_id] = AvailableWakeWord(
id=model_id,
wake_word=model_config["wake_word"],
trained_languages=model_config.get("trained_languages", []),
config_path=model_config_path,
)
_LOGGER.debug("Available wake words: %s", list(sorted(available_wake_words.keys())))
libtensorflowlite_c_path = _LIB_DIR / "libtensorflowlite_c.so"
_LOGGER.debug("libtensorflowlite_c path: %s", libtensorflowlite_c_path)
# Load wake/stop models
wake_config_path = wake_word_dir / f"{args.wake_model}.json"
_LOGGER.debug("Loading wake model: %s", wake_config_path)
wake_model = MicroWakeWord.from_config(wake_config_path, libtensorflowlite_c_path)
stop_config_path = wake_word_dir / f"{args.stop_model}.json"
_LOGGER.debug("Loading stop model: %s", stop_config_path)
stop_model = MicroWakeWord.from_config(stop_config_path, libtensorflowlite_c_path)
state = ServerState(
name=args.name,
mac_address=get_mac(),
audio_queue=Queue(),
entities=[],
available_wake_words=available_wake_words,
wake_word=wake_model,
stop_word=stop_model,
music_player=MpvMediaPlayer(device=args.audio_output_device),
tts_player=MpvMediaPlayer(device=args.audio_output_device),
wakeup_sound=args.wakeup_sound,
timer_finished_sound=args.timer_finished_sound,
)
process_audio_thread = threading.Thread(
target=process_audio, args=(state,), daemon=True
)
process_audio_thread.start()
def sd_callback(indata, _frames, _time, _status):
state.audio_queue.put_nowait(bytes(indata))
loop = asyncio.get_running_loop()
server = await loop.create_server(
lambda: VoiceSatelliteProtocol(state), host=args.host, port=args.port
)
try:
_LOGGER.debug("Opening audio input device: %s", args.audio_input_device)
with sd.RawInputStream(
samplerate=16000,
blocksize=args.audio_input_block_size,
device=args.audio_input_device,
dtype="int16",
channels=1,
callback=sd_callback,
):
async with server:
_LOGGER.info("Server started (host=%s, port=%s)", args.host, args.port)
await server.serve_forever()
except KeyboardInterrupt:
pass
finally:
state.audio_queue.put_nowait(None)
process_audio_thread.join()
_LOGGER.debug("Server stopped")
if __name__ == "__main__":
asyncio.run(main())
+180
View File
@@ -0,0 +1,180 @@
"""Partial ESPHome server implementation."""
import asyncio
import logging
from abc import abstractmethod
from collections.abc import Iterable
from typing import TYPE_CHECKING, List, Optional
# pylint: disable=no-name-in-module
from aioesphomeapi._frame_helper.packets import make_plain_text_packets
from aioesphomeapi.api_pb2 import ( # type: ignore[attr-defined]
ConnectRequest,
ConnectResponse,
DisconnectRequest,
DisconnectResponse,
HelloRequest,
HelloResponse,
PingRequest,
PingResponse,
)
from aioesphomeapi.core import MESSAGE_TYPE_TO_PROTO
from google.protobuf import message
PROTO_TO_MESSAGE_TYPE = {v: k for k, v in MESSAGE_TYPE_TO_PROTO.items()}
_LOGGER = logging.getLogger(__name__)
class APIServer(asyncio.Protocol):
def __init__(self, name: str) -> None:
self.name = name
self._buffer: Optional[bytes] = None
self._buffer_len: int = 0
self._pos: int = 0
self._transport = None
self._writelines = None
@abstractmethod
def handle_message(self, msg: message.Message) -> Iterable[message.Message]:
pass
def process_packet(self, msg_type: int, packet_data: bytes) -> None:
msg_class = MESSAGE_TYPE_TO_PROTO[msg_type]
msg_inst = msg_class.FromString(packet_data)
if isinstance(msg_inst, HelloRequest):
self.send_messages(
[
HelloResponse(
api_version_major=1,
api_version_minor=10,
name=self.name,
)
]
)
return
if isinstance(msg_inst, ConnectRequest):
self.send_messages([ConnectResponse()])
elif isinstance(msg_inst, DisconnectRequest):
self.send_messages([DisconnectResponse()])
_LOGGER.debug("Disconnect requested")
if self._transport:
self._transport.close()
self._transport = None
self._writelines = None
elif isinstance(msg_inst, PingRequest):
self.send_messages([PingResponse()])
elif msgs := self.handle_message(msg_inst):
if isinstance(msgs, message.Message):
msgs = [msgs]
self.send_messages(msgs)
def send_messages(self, msgs: List[message.Message]):
if self._writelines is None:
return
packets = [
(PROTO_TO_MESSAGE_TYPE[msg.__class__], msg.SerializeToString())
for msg in msgs
]
packet_bytes = make_plain_text_packets(packets)
self._writelines(packet_bytes)
def connection_made(self, transport) -> None:
self._transport = transport
self._writelines = transport.writelines
def data_received(self, data: bytes):
if self._buffer is None:
self._buffer = data
self._buffer_len = len(data)
else:
self._buffer += data
self._buffer_len += len(data)
while self._buffer_len >= 3:
self._pos = 0
# Read preamble, which should always 0x00
if (preamble := self._read_varuint()) != 0x00:
_LOGGER.error("Incorrect preamble: %s", preamble)
return
if (length := self._read_varuint()) == -1:
_LOGGER.error("Incorrect length: %s", length)
return
if (msg_type := self._read_varuint()) == -1:
_LOGGER.error("Incorrect message type: %s", msg_type)
return
if length == 0:
# Empty message (allowed)
self._remove_from_buffer()
self.process_packet(msg_type, b"")
continue
if (packet_data := self._read(length)) is None:
return
self._remove_from_buffer()
self.process_packet(msg_type, packet_data)
def _read(self, length: int) -> bytes | None:
"""Read exactly length bytes from the buffer or None if all the bytes are not yet available."""
new_pos = self._pos + length
if self._buffer_len < new_pos:
return None
original_pos = self._pos
self._pos = new_pos
if TYPE_CHECKING:
assert self._buffer is not None, "Buffer should be set"
cstr = self._buffer
# Important: we must keep the bounds check (self._buffer_len < new_pos)
# above to verify we never try to read past the end of the buffer
return cstr[original_pos:new_pos]
def connection_lost(self, exc):
self._transport = None
self._writelines = None
def _read_varuint(self) -> int:
"""Read a varuint from the buffer or -1 if the buffer runs out of bytes."""
if not self._buffer:
return -1
result = 0
bitpos = 0
cstr = self._buffer
while self._buffer_len > self._pos:
val = cstr[self._pos]
self._pos += 1
result |= (val & 0x7F) << bitpos
if (val & 0x80) == 0:
return result
bitpos += 7
return -1
def _remove_from_buffer(self) -> None:
"""Remove data from the buffer."""
end_of_frame_pos = self._pos
self._buffer_len -= end_of_frame_pos
if self._buffer_len == 0:
# This is the best case scenario, we can just set the buffer to None
# and don't have to copy the data. This is the most common case as well.
self._buffer = None
return
if TYPE_CHECKING:
assert self._buffer is not None, "Buffer should be set"
# This is the worst case scenario, we have to copy the data
# and can't just use the buffer directly. This should only happen
# when we read multiple frames at once because the event loop
# is blocked and we cannot pull the data out of the buffer fast enough.
cstr = self._buffer
# Important: we must use the explicit length for the slice
# since Cython will stop at any '\0' character if we don't
self._buffer = cstr[end_of_frame_pos : self._buffer_len + end_of_frame_pos]
+133
View File
@@ -0,0 +1,133 @@
from abc import abstractmethod
from collections.abc import Iterable
from typing import Callable, List, Optional, Union
# pylint: disable=no-name-in-module
from aioesphomeapi.api_pb2 import ( # type: ignore[attr-defined]
ListEntitiesMediaPlayerResponse,
ListEntitiesRequest,
MediaPlayerCommandRequest,
MediaPlayerStateResponse,
SubscribeHomeAssistantStatesRequest,
)
from aioesphomeapi.model import MediaPlayerCommand, MediaPlayerState
from google.protobuf import message
from .api_server import APIServer
from .mpv_player import MpvMediaPlayer
from .util import call_all
class ESPHomeEntity:
def __init__(self, server: APIServer) -> None:
self.server = server
@abstractmethod
def handle_message(self, msg: message.Message) -> Iterable[message.Message]:
pass
# -----------------------------------------------------------------------------
class MediaPlayerEntity(ESPHomeEntity):
def __init__(
self,
server: APIServer,
key: int,
name: str,
object_id: str,
music_player: MpvMediaPlayer,
announce_player: MpvMediaPlayer,
) -> None:
ESPHomeEntity.__init__(self, server)
self.key = key
self.name = name
self.object_id = object_id
self.state = MediaPlayerState.IDLE
self.volume = 1.0
self.muted = False
self.music_player = music_player
self.announce_player = announce_player
def play(
self,
url: Union[str, List[str]],
announcement: bool = False,
done_callback: Optional[Callable[[], None]] = None,
) -> Iterable[message.Message]:
if announcement:
if self.music_player.is_playing:
# Announce, resume music
self.music_player.pause()
self.announce_player.play(
url,
done_callback=lambda: call_all(
self.music_player.resume, done_callback
),
)
else:
# Announce, idle
self.announce_player.play(
url,
done_callback=lambda: call_all(
self.server.send_messages(
[self._update_state(MediaPlayerState.IDLE)]
),
done_callback,
),
)
else:
# Music
self.music_player.play(
url,
done_callback=lambda: call_all(
self.server.send_messages(
[self._update_state(MediaPlayerState.IDLE)]
),
done_callback,
),
)
yield self._update_state(MediaPlayerState.PLAYING)
def handle_message(self, msg: message.Message) -> Iterable[message.Message]:
if isinstance(msg, MediaPlayerCommandRequest) and (msg.key == self.key):
if msg.has_media_url:
announcement = msg.has_announcement and msg.announcement
yield from self.play(msg.media_url, announcement=announcement)
elif msg.has_command:
if msg.command == MediaPlayerCommand.PAUSE:
self.music_player.pause()
yield self._update_state(MediaPlayerState.PAUSED)
elif msg.command == MediaPlayerCommand.PLAY:
self.music_player.resume()
yield self._update_state(MediaPlayerState.PLAYING)
elif msg.has_volume:
volume = int(msg.volume * 100)
self.music_player.set_volume(volume)
self.announce_player.set_volume(volume)
self.volume = msg.volume
yield self._update_state(self.state)
elif isinstance(msg, ListEntitiesRequest):
yield ListEntitiesMediaPlayerResponse(
object_id=self.object_id,
key=self.key,
name=self.name,
supports_pause=True,
)
elif isinstance(msg, SubscribeHomeAssistantStatesRequest):
yield self._get_state_message()
def _update_state(self, new_state: MediaPlayerState) -> MediaPlayerStateResponse:
self.state = new_state
return self._get_state_message()
def _get_state_message(self) -> MediaPlayerStateResponse:
return MediaPlayerStateResponse(
key=self.key,
state=self.state,
volume=self.volume,
muted=self.muted,
)
+272
View File
@@ -0,0 +1,272 @@
import ctypes
import json
import statistics
from collections import deque
from pathlib import Path
from typing import Deque, List, Union
import numpy as np
from pymicro_features import MicroFrontend
_SAMPLES_PER_SECOND = 16000
_SAMPLES_PER_CHUNK = 160 # 10ms
_BYTES_PER_SAMPLE = 2 # 16-bit
_BYTES_PER_CHUNK = _SAMPLES_PER_CHUNK * _BYTES_PER_SAMPLE
_SECONDS_PER_CHUNK = _SAMPLES_PER_CHUNK / _SAMPLES_PER_SECOND
_STRIDE = 3
_DEFAULT_REFRACTORY = 2 # seconds
class TfLiteQuantizationParams(ctypes.Structure):
_fields_ = [("scale", ctypes.c_float), ("zero_point", ctypes.c_int32)]
class MicroWakeWord:
def __init__(
self,
id: str, # pylint: disable=redefined-builtin
wake_word: str,
tflite_model: Union[str, Path],
probability_cutoff: float,
sliding_window_size: int,
refractory_seconds: float,
trained_languages: List[str],
libtensorflowlite_c_path: Union[str, Path],
):
self.id = id
self.wake_word = wake_word
self.tflite_model = tflite_model
self.probability_cutoff = probability_cutoff
self.sliding_window_size = sliding_window_size
self.refractory_seconds = refractory_seconds
self.trained_languages = trained_languages
self.is_active = True
# Load the shared library
self.libtensorflowlite_c_path = Path(libtensorflowlite_c_path)
self.lib = ctypes.cdll.LoadLibrary(str(self.libtensorflowlite_c_path.resolve()))
# Define required argument/return types for C API
self.lib.TfLiteModelCreateFromFile.argtypes = [ctypes.c_char_p]
self.lib.TfLiteModelCreateFromFile.restype = ctypes.c_void_p
self.lib.TfLiteInterpreterCreate.argtypes = [ctypes.c_void_p, ctypes.c_void_p]
self.lib.TfLiteInterpreterCreate.restype = ctypes.c_void_p
self.lib.TfLiteInterpreterAllocateTensors.argtypes = [ctypes.c_void_p]
self.lib.TfLiteInterpreterInvoke.argtypes = [ctypes.c_void_p]
self.lib.TfLiteInterpreterGetInputTensor.argtypes = [
ctypes.c_void_p,
ctypes.c_int,
]
self.lib.TfLiteInterpreterGetInputTensor.restype = ctypes.c_void_p
self.lib.TfLiteInterpreterGetOutputTensor.argtypes = [
ctypes.c_void_p,
ctypes.c_int,
]
self.lib.TfLiteInterpreterGetOutputTensor.restype = ctypes.c_void_p
self.lib.TfLiteTensorCopyFromBuffer.argtypes = [
ctypes.c_void_p,
ctypes.c_void_p,
ctypes.c_size_t,
]
self.lib.TfLiteTensorCopyToBuffer.argtypes = [
ctypes.c_void_p,
ctypes.c_void_p,
ctypes.c_size_t,
]
self.lib.TfLiteTensorType.restype = ctypes.c_int
self.lib.TfLiteTensorByteSize.argtypes = [ctypes.c_void_p]
self.lib.TfLiteTensorByteSize.restype = ctypes.c_size_t
self.lib.TfLiteTensorQuantizationParams.restype = ctypes.Structure
self.lib.TfLiteTensorQuantizationParams.argtypes = [ctypes.c_void_p]
self.lib.TfLiteTensorQuantizationParams.restype = TfLiteQuantizationParams
# Load the model and create interpreter
self.model_path = str(Path(tflite_model).resolve()).encode("utf-8")
self._load_model()
self._frontend = MicroFrontend()
self._features: List[np.ndarray] = []
self._probabilities: Deque[float] = deque(maxlen=self.sliding_window_size)
self._audio_buffer = bytes()
self._ignore_seconds: float = 0
def _load_model(self) -> None:
self.model = self.lib.TfLiteModelCreateFromFile(self.model_path)
self.interpreter = self.lib.TfLiteInterpreterCreate(self.model, None)
self.lib.TfLiteInterpreterAllocateTensors(self.interpreter)
# Access input and output tensor
self.input_tensor = self.lib.TfLiteInterpreterGetInputTensor(
self.interpreter, 0
)
self.output_tensor = self.lib.TfLiteInterpreterGetOutputTensor(
self.interpreter, 0
)
# Get quantization parameters
input_q = self.lib.TfLiteTensorQuantizationParams(self.input_tensor)
output_q = self.lib.TfLiteTensorQuantizationParams(self.output_tensor)
self.input_scale, self.input_zero_point = input_q.scale, input_q.zero_point
self.output_scale, self.output_zero_point = output_q.scale, output_q.zero_point
def reset(self) -> None:
"""Reload model and clear state.
This must be done between audio clips when not streaming.
"""
self._audio_buffer = bytes()
self._features.clear()
self._probabilities.clear()
self._ignore_seconds = 0
# Clear out residual features
self._frontend = MicroFrontend()
# Need to reload model to reset intermediary results
# reset_all_variables() doesn't work.
self._load_model()
@property
def samples_per_chunk(self) -> int:
"""Number of samples in a streaming audio chunk."""
return _SAMPLES_PER_CHUNK
@property
def bytes_per_chunk(self) -> int:
"""Number of bytes in a streaming audio chunk.
Assumes 16-bit mono samples at 16Khz.
"""
return _BYTES_PER_CHUNK
@staticmethod
def from_config(
config_path: Union[str, Path],
libtensorflowlite_c_path: Union[str, Path],
refractory_seconds: float = _DEFAULT_REFRACTORY,
) -> "MicroWakeWord":
"""Load a microWakeWord model from a JSON config file.
Parameters
----------
config_path: str or Path
Path to JSON configuration file
refractory_seconds: float
Number of seconds to ignore after detection
"""
config_path = Path(config_path)
with open(config_path, "r", encoding="utf-8") as config_file:
config = json.load(config_file)
micro_config = config["micro"]
return MicroWakeWord(
id=Path(config["model"]).stem,
wake_word=config["wake_word"],
tflite_model=config_path.parent / config["model"],
probability_cutoff=micro_config["probability_cutoff"],
sliding_window_size=micro_config["sliding_window_size"],
refractory_seconds=refractory_seconds,
trained_languages=micro_config.get("trained_languages", []),
libtensorflowlite_c_path=libtensorflowlite_c_path,
)
def process_streaming(self, audio_bytes: bytes) -> bool:
"""Process a chunk of audio in streaming mode.
Parameters
----------
audio_bytes: bytes
Raw 16-bit mono audio samples at 16Khz
Returns True if the wake word was detected.
"""
self._audio_buffer += audio_bytes
if len(self._audio_buffer) < _BYTES_PER_CHUNK:
# Not enough audio to get features
return False
detected = False
audio_buffer_idx = 0
while (audio_buffer_idx + _BYTES_PER_CHUNK) <= len(self._audio_buffer):
# Process chunk
chunk_bytes = self._audio_buffer[
audio_buffer_idx : audio_buffer_idx + _BYTES_PER_CHUNK
]
frontend_result = self._frontend.ProcessSamples(chunk_bytes)
audio_buffer_idx += frontend_result.samples_read * _BYTES_PER_SAMPLE
self._ignore_seconds = max(0, self._ignore_seconds - _SECONDS_PER_CHUNK)
if not frontend_result.features:
# Not enough audio for a full window
continue
self._features.append(
np.array(frontend_result.features).reshape(
(1, 1, len(frontend_result.features))
)
)
if len(self._features) < _STRIDE:
# Not enough windows
continue
# Allocate and quantize input data
quant_features = np.round(
np.concatenate(self._features, axis=1) / self.input_scale
+ self.input_zero_point
).astype(np.uint8)
# Stride instead of rolling
self._features.clear()
# Set tensor
quant_ptr = quant_features.ctypes.data_as(ctypes.c_void_p)
self.lib.TfLiteTensorCopyFromBuffer(
self.input_tensor, quant_ptr, quant_features.nbytes
)
# Run inference
self.lib.TfLiteInterpreterInvoke(self.interpreter)
# Read output
output_bytes = self.lib.TfLiteTensorByteSize(self.output_tensor)
output_data = np.empty(output_bytes, dtype=np.uint8)
self.lib.TfLiteTensorCopyToBuffer(
self.output_tensor,
output_data.ctypes.data_as(ctypes.c_void_p),
output_bytes,
)
# Dequantize output
result = (
output_data.astype(np.float32) - self.output_zero_point
) * self.output_scale
self._probabilities.append(result.item())
if len(self._probabilities) < self.sliding_window_size:
# Not enough probabilities
continue
if statistics.mean(self._probabilities) > self.probability_cutoff:
if self._ignore_seconds <= 0:
detected = True
self._ignore_seconds = self.refractory_seconds
# Remove processed audio
self._audio_buffer = self._audio_buffer[audio_buffer_idx:]
return detected
+94
View File
@@ -0,0 +1,94 @@
"""Media player using mpv in a subprocess."""
import logging
from collections.abc import Callable
from threading import Lock
from typing import List, Optional, Union
from mpv import MPV
_LOGGER = logging.getLogger(__name__)
class MpvMediaPlayer:
def __init__(self, device: Optional[str] = None) -> None:
self.player = MPV()
if device:
self.player["audio-device"] = device
self.is_playing = False
self._playlist: List[str] = []
self._done_callback: Optional[Callable[[], None]] = None
self._done_callback_lock = Lock()
self._duck_volume: int = 50
self._unduck_volume: int = 100
self.player.event_callback("end-file")(self._on_end_file)
def play(
self,
url: Union[str, List[str]],
done_callback: Optional[Callable[[], None]] = None,
stop_first: bool = True,
) -> None:
self.stop()
if isinstance(url, str):
self._playlist = [url]
else:
self._playlist = url
next_url = self._playlist.pop(0)
_LOGGER.debug("Playing %s", next_url)
self._done_callback = done_callback
self.is_playing = True
self.player.play(next_url)
def pause(self) -> None:
self.player.pause = True
self.is_playing = False
def resume(self) -> None:
self.player.pause = False
if self._playlist:
self.is_playing = True
def stop(self) -> None:
self.player.stop()
self._playlist.clear()
def duck(self) -> None:
self.player.volume = self._duck_volume
def unduck(self) -> None:
self.player.volume = self._unduck_volume
def set_volume(self, volume: int) -> None:
volume = max(0, min(100, volume))
self.player.volume = volume
self._unduck_volume = volume
self._duck_volume = volume // 2
def _on_end_file(self, event) -> None:
if self._playlist:
self.player.play(self._playlist.pop(0))
return
self.is_playing = False
todo_callback: Optional[Callable[[], None]] = None
with self._done_callback_lock:
if self._done_callback:
todo_callback = self._done_callback
self._done_callback = None
if todo_callback:
try:
todo_callback()
except Exception:
_LOGGER.exception("Unexpected error running done callback")
+22
View File
@@ -0,0 +1,22 @@
"""Utility methods."""
import platform
import uuid
from collections.abc import Callable
from typing import Optional
def get_mac() -> str:
mac = uuid.getnode()
mac_str = ":".join(f"{(mac >> i) & 0xff:02x}" for i in range(40, -1, -8))
return mac_str
def call_all(*callables: Optional[Callable[[], None]]) -> None:
for item in filter(None, callables):
item()
def is_arm() -> bool:
machine = platform.machine()
return ("arm" in machine) or ("aarch" in machine)