From 37c11e58c48d520ee480d3102a63e48989778495 Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Fri, 9 Aug 2024 14:05:20 -0500 Subject: [PATCH] Allow remotely triggered pipelines for wake word satellites --- examples/2mic_service.py | 2 +- examples/4mic_service.py | 2 +- examples/usbmic_service.py | 11 +- examples/websocket-service/server.py | 2 +- installer/__main__.py | 2 +- installer/const.py | 2 +- installer/microphone.py | 2 +- installer/packages.py | 2 +- installer/speakers.py | 2 +- installer/whiptail.py | 2 +- tests/test_satellite.py | 2 +- tests/test_wake_streaming.py | 2 +- wyoming_satellite/VERSION | 2 +- wyoming_satellite/__main__.py | 66 ++++- wyoming_satellite/event_handler.py | 2 +- wyoming_satellite/example_event_client.py | 2 +- wyoming_satellite/satellite.py | 336 ++++++++++++++++------ wyoming_satellite/settings.py | 13 +- wyoming_satellite/utils/audio.py | 2 +- wyoming_satellite/utils/misc.py | 2 +- wyoming_satellite/vad.py | 96 +++++++ 21 files changed, 415 insertions(+), 139 deletions(-) diff --git a/examples/2mic_service.py b/examples/2mic_service.py index 0b44e62..46a2078 100644 --- a/examples/2mic_service.py +++ b/examples/2mic_service.py @@ -23,7 +23,7 @@ from wyoming.server import AsyncEventHandler, AsyncServer from wyoming.vad import VoiceStarted from wyoming.wake import Detection -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger(__name__) NUM_LEDS = 3 LEDS_GPIO = 12 diff --git a/examples/4mic_service.py b/examples/4mic_service.py index 221528f..f1a8765 100644 --- a/examples/4mic_service.py +++ b/examples/4mic_service.py @@ -23,7 +23,7 @@ from wyoming.server import AsyncEventHandler, AsyncServer from wyoming.vad import VoiceStarted from wyoming.wake import Detection -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger(__name__) NUM_LEDS = 12 LEDS_GPIO = 5 diff --git a/examples/usbmic_service.py b/examples/usbmic_service.py index 2898913..e69ed78 100644 --- a/examples/usbmic_service.py +++ b/examples/usbmic_service.py @@ -6,6 +6,7 @@ import logging import time from functools import partial +from pixel_ring import pixel_ring from wyoming.event import Event from wyoming.satellite import ( SatelliteConnected, @@ -13,14 +14,13 @@ from wyoming.satellite import ( StreamingStarted, StreamingStopped, ) -from wyoming.snd import Played from wyoming.server import AsyncEventHandler, AsyncServer +from wyoming.snd import Played from wyoming.vad import VoiceStarted, VoiceStopped from wyoming.wake import Detection -from pixel_ring import pixel_ring +_LOGGER = logging.getLogger(__name__) -_LOGGER = logging.getLogger() async def main() -> None: """Main entry point.""" @@ -35,7 +35,7 @@ async def main() -> None: _LOGGER.info("Ready") # Turn on power to LEDs - pixel_ring.set_color_palette(0x0080ff, 0x007a37) + pixel_ring.set_color_palette(0x0080FF, 0x007A37) pixel_ring.think() await asyncio.sleep(3) pixel_ring.off() @@ -92,10 +92,11 @@ class LEDsEventHandler(AsyncEventHandler): pixel_ring.off() elif SatelliteDisconnected.is_type(event.type): _LOGGER.debug("SatelliteDisconnected") - pixel_ring.mono(0xff0000) + pixel_ring.mono(0xFF0000) return True + if __name__ == "__main__": try: asyncio.run(main()) diff --git a/examples/websocket-service/server.py b/examples/websocket-service/server.py index 6fbf84c..ddc6a5a 100644 --- a/examples/websocket-service/server.py +++ b/examples/websocket-service/server.py @@ -10,7 +10,7 @@ import websockets from wyoming.event import Event from wyoming.server import AsyncEventHandler, AsyncServer -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger(__name__) async def main() -> None: diff --git a/installer/__main__.py b/installer/__main__.py index d33fd85..1b4c635 100644 --- a/installer/__main__.py +++ b/installer/__main__.py @@ -17,7 +17,7 @@ from .speakers import configure_speakers from .wake_word import configure_wake_word from .whiptail import ItemType, error, menu, msgbox, passwordbox, run_with_gauge -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger(__name__) def main() -> None: diff --git a/installer/const.py b/installer/const.py index 61318dc..e3f61bc 100644 --- a/installer/const.py +++ b/installer/const.py @@ -9,7 +9,7 @@ from typing import List, Optional from .dataclasses_json import DataClassJsonMixin _DIR = Path(__file__).parent -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger(__name__) PROGRAM_DIR = _DIR.parent LOCAL_DIR = PROGRAM_DIR / "local" diff --git a/installer/microphone.py b/installer/microphone.py index a6b32e0..d6ef902 100644 --- a/installer/microphone.py +++ b/installer/microphone.py @@ -9,7 +9,7 @@ from typing import Dict, List, Optional from .const import RECORD_RMS_MIN, RECORD_SECONDS, Settings from .whiptail import gauge, inputbox, menu, msgbox, radiolist -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger(__name__) def configure_microphone(settings: Settings) -> None: diff --git a/installer/packages.py b/installer/packages.py index 1e3bacd..9afa9e9 100644 --- a/installer/packages.py +++ b/installer/packages.py @@ -5,7 +5,7 @@ import subprocess from .const import PROGRAM_DIR from .whiptail import run_with_gauge -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger(__name__) def packages_installed(*packages) -> bool: diff --git a/installer/speakers.py b/installer/speakers.py index bffbdbb..f3ad90c 100644 --- a/installer/speakers.py +++ b/installer/speakers.py @@ -6,7 +6,7 @@ from typing import List, Optional from .const import PROGRAM_DIR, Settings from .whiptail import checklist, inputbox, menu, msgbox, radiolist -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger(__name__) def configure_speakers(settings: Settings) -> None: diff --git a/installer/whiptail.py b/installer/whiptail.py index 4c4d911..24a5cae 100644 --- a/installer/whiptail.py +++ b/installer/whiptail.py @@ -11,7 +11,7 @@ from .const import HEIGHT, LIST_HEIGHT, TITLE, WIDTH ItemType = Union[str, Tuple[Any, str]] -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger(__name__) def whiptail(*args) -> Optional[str]: diff --git a/tests/test_satellite.py b/tests/test_satellite.py index a6686ab..cba6157 100644 --- a/tests/test_satellite.py +++ b/tests/test_satellite.py @@ -30,7 +30,7 @@ from .shared import ( MicClient, ) -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger(__name__) TIMEOUT: Final = 1 diff --git a/tests/test_wake_streaming.py b/tests/test_wake_streaming.py index 2550ee5..b5a3658 100644 --- a/tests/test_wake_streaming.py +++ b/tests/test_wake_streaming.py @@ -22,7 +22,7 @@ from wyoming_satellite import ( from .shared import MicClient -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger(__name__) class WakeClient(AsyncClient): diff --git a/wyoming_satellite/VERSION b/wyoming_satellite/VERSION index f0bb29e..88c5fb8 100644 --- a/wyoming_satellite/VERSION +++ b/wyoming_satellite/VERSION @@ -1 +1 @@ -1.3.0 +1.4.0 diff --git a/wyoming_satellite/__main__.py b/wyoming_satellite/__main__.py index 4cff57b..283bc01 100644 --- a/wyoming_satellite/__main__.py +++ b/wyoming_satellite/__main__.py @@ -3,11 +3,13 @@ import argparse import asyncio import logging +import shlex import sys from functools import partial from pathlib import Path -from wyoming.info import Attribution, Info, Satellite +from wyoming.audio import AudioFormat +from wyoming.info import Attribution, Info, MicProgram, Satellite, SndProgram from wyoming.server import AsyncServer, AsyncTcpServer from . import __version__ @@ -26,7 +28,6 @@ from .settings import ( TimerSettings, VadSettings, WakeSettings, - WakeWordAndPipeline, ) from .utils import ( get_mac_address, @@ -36,7 +37,7 @@ from .utils import ( split_command, ) -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger(__name__) _DIR = Path(__file__).parent @@ -122,9 +123,7 @@ async def main() -> None: "--wake-word-name", action="append", default=[], - nargs="+", - metavar=("name", "pipeline"), - help="Name of wake word to listen for and optional pipeline name to run (requires --wake-uri)", + help="Name of wake word to listen for (requires --wake-uri)", ) parser.add_argument("--wake-command", help="Program to run for wake word detection") parser.add_argument( @@ -154,9 +153,14 @@ async def main() -> None: # Voice activity detector parser.add_argument( - "--vad", action="store_true", help="Wait for speech before streaming audio" + "--vad", action="store_true", help="Enable voice activity detector" + ) + parser.add_argument( + "--vad-threshold", + type=float, + default=0.5, + help="Threshold for speech (0-1, default: 0.5)", ) - parser.add_argument("--vad-threshold", type=float, default=0.5) parser.add_argument("--vad-trigger-level", type=int, default=1) parser.add_argument("--vad-buffer-seconds", type=float, default=2) parser.add_argument( @@ -165,6 +169,7 @@ async def main() -> None: default=5.0, help="Seconds before going back to waiting for speech when wake word isn't detected", ) + parser.add_argument("--vad-finished-speaking-seconds", type=float, default=0.8) # External event handlers parser.add_argument( @@ -330,9 +335,6 @@ async def main() -> None: _LOGGER.fatal("%s does not exist", args.timer_finished_wav) sys.exit(1) - if args.vad and (args.wake_uri or args.wake_command): - _LOGGER.warning("VAD is not used with local wake word detection") - logging.basicConfig( level=logging.DEBUG if args.debug else logging.INFO, format=args.log_format ) @@ -350,9 +352,46 @@ async def main() -> None: attribution=Attribution(name="", url=""), installed=True, version=__version__, + has_vad=args.vad, + supports_trigger=True, + active_wake_words=args.wake_word_name, ) ) + if args.mic_command: + mic_program = shlex.split(args.mic_command)[0] + wyoming_info.mic = [ + MicProgram( + name=mic_program, + description=args.mic_command, + attribution=Attribution(name="", url=""), + installed=True, + version=None, + mic_format=AudioFormat( + rate=args.mic_command_rate, + width=args.mic_command_width, + channels=args.mic_command_channels, + ), + ) + ] + + if args.snd_command: + snd_program = shlex.split(args.snd_command)[0] + wyoming_info.snd = [ + SndProgram( + name=snd_program, + description=args.snd_command, + attribution=Attribution(name="", url=""), + installed=True, + version=None, + snd_format=AudioFormat( + rate=args.snd_command_rate, + width=args.snd_command_width, + channels=args.snd_command_channels, + ), + ) + ] + settings = SatelliteSettings( mic=MicSettings( uri=args.mic_uri, @@ -374,13 +413,12 @@ async def main() -> None: trigger_level=args.vad_trigger_level, buffer_seconds=args.vad_buffer_seconds, wake_word_timeout=args.vad_wake_word_timeout, + finished_speaking_seconds=args.vad_finished_speaking_seconds, ), wake=WakeSettings( uri=args.wake_uri, command=split_command(args.wake_command), - names=[ - WakeWordAndPipeline(*wake_name) for wake_name in args.wake_word_name - ], + names=args.wake_word_name, refractory_seconds=( args.wake_refractory_seconds if args.wake_refractory_seconds > 0 diff --git a/wyoming_satellite/event_handler.py b/wyoming_satellite/event_handler.py index 58f046b..315ef23 100644 --- a/wyoming_satellite/event_handler.py +++ b/wyoming_satellite/event_handler.py @@ -9,7 +9,7 @@ from wyoming.server import AsyncEventHandler from .satellite import SatelliteBase -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger(__name__) class SatelliteEventHandler(AsyncEventHandler): diff --git a/wyoming_satellite/example_event_client.py b/wyoming_satellite/example_event_client.py index 606236e..73672f2 100644 --- a/wyoming_satellite/example_event_client.py +++ b/wyoming_satellite/example_event_client.py @@ -8,7 +8,7 @@ from functools import partial from wyoming.event import Event from wyoming.server import AsyncEventHandler, AsyncServer -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger(__name__) async def main() -> None: diff --git a/wyoming_satellite/satellite.py b/wyoming_satellite/satellite.py index 7a1dd58..b4cccd3 100644 --- a/wyoming_satellite/satellite.py +++ b/wyoming_satellite/satellite.py @@ -1,4 +1,5 @@ """Satellite code.""" + import array import asyncio import logging @@ -8,11 +9,12 @@ import wave from dataclasses import dataclass from enum import Enum, auto from pathlib import Path +from queue import Queue from typing import Callable, Dict, Final, List, Optional, Set, Union from pyring_buffer import RingBuffer from wyoming.asr import Transcript -from wyoming.audio import AudioChunk, AudioFormat, AudioStart, AudioStop +from wyoming.audio import AudioChunk, AudioStart, AudioStop from wyoming.client import AsyncClient from wyoming.error import Error from wyoming.event import Event, async_write_event @@ -42,10 +44,10 @@ from .utils import ( run_event_command, wav_to_events, ) -from .vad import SileroVad +from .vad import SileroVad, VoiceCommandSegmenter from .webrtc import WebRtcAudio -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger(__name__) _PONG_TIMEOUT: Final = 5 _PING_SEND_DELAY: Final = 2 @@ -109,6 +111,11 @@ class SatelliteBase: settings.debug_recording_dir, "stt" ) + # Names of wake words to listen for + self._wake_names: Optional[List[str]] = None + if self.settings.wake.names: + self._wake_names = self.settings.wake.names + @property def is_running(self) -> bool: """True if not stopping/stopped.""" @@ -279,13 +286,6 @@ class SatelliteBase: # TTS stopped await self.event_to_snd(event) await self.trigger_tts_stop() - elif Detect.is_type(event.type): - # Wake word detection started - await self.trigger_detect() - elif Detection.is_type(event.type): - # Wake word detected - _LOGGER.debug("Wake word detected") - await self.trigger_detection(Detection.from_event(event)) elif VoiceStarted.is_type(event.type): # STT start await self.trigger_stt_start() @@ -320,34 +320,38 @@ class SatelliteBase: if forward_event: await self.forward_event(event) - async def _send_run_pipeline(self, pipeline_name: Optional[str] = None) -> None: + async def _send_run_pipeline( + self, + start_stage: Optional[PipelineStage] = None, + end_stage: Optional[PipelineStage] = None, + wake_word_name: Optional[str] = None, + ) -> None: """Sends a RunPipeline event with the correct stages.""" - if self.settings.wake.enabled: - # Local wake word detection - start_stage = PipelineStage.ASR - restart_on_end = False - else: - # Remote wake word detection - start_stage = PipelineStage.WAKE - restart_on_end = not self.settings.vad.enabled + restart_on_end = False - if self.settings.snd.enabled: - # Play TTS response - end_stage = PipelineStage.TTS - else: - # No audio output - end_stage = PipelineStage.HANDLE + if start_stage is None: + if self.settings.wake.enabled: + # Local wake word detection + start_stage = PipelineStage.ASR + restart_on_end = False + else: + # Remote wake word detection + start_stage = PipelineStage.WAKE + restart_on_end = not self.settings.vad.enabled + + if end_stage is None: + if self.settings.snd.enabled: + # Play TTS response + end_stage = PipelineStage.TTS + else: + # No audio output + end_stage = PipelineStage.HANDLE run_pipeline = RunPipeline( start_stage=start_stage, end_stage=end_stage, - name=pipeline_name, + wake_word_name=wake_word_name, restart_on_end=restart_on_end, - snd_format=AudioFormat( - rate=self.settings.snd.rate, - width=self.settings.snd.width, - channels=self.settings.snd.channels, - ), ).event() _LOGGER.debug(run_pipeline) await self.event_to_server(run_pipeline) @@ -476,7 +480,10 @@ class SatelliteBase: mic_client = self._make_mic_client() assert mic_client is not None await mic_client.connect() - _LOGGER.debug("Connected to mic service") + _LOGGER.info( + "Connected to mic service: %s", + self.settings.mic.uri or self.settings.mic.command, + ) event = await mic_client.read_event() if event is None: @@ -514,9 +521,11 @@ class SatelliteBase: event = AudioChunk( rate=chunk.rate, width=chunk.width, - channels=chunk.channels - if (self.settings.mic.channel_index is None) - else 1, + channels=( + chunk.channels + if (self.settings.mic.channel_index is None) + else 1 + ), audio=audio_bytes, ).event() else: @@ -595,7 +604,10 @@ class SatelliteBase: snd_client = self._make_snd_client() assert snd_client is not None await snd_client.connect() - _LOGGER.debug("Connected to snd service") + _LOGGER.info( + "Connected to snd service: %s", + self.settings.snd.uri or self.settings.snd.command, + ) # Audio processing if self.settings.snd.needs_processing and AudioChunk.is_type( @@ -618,6 +630,8 @@ class SatelliteBase: await _disconnect() if snd_event.is_tts: await self.trigger_played() + await self.event_to_server(Played().event()) + snd_client = None # reconnect on next event except asyncio.CancelledError: break @@ -736,7 +750,10 @@ class SatelliteBase: wake_client = self._make_wake_client() assert wake_client is not None await wake_client.connect() - _LOGGER.debug("Connected to wake service") + _LOGGER.info( + "Connected to wake service: %s", + self.settings.wake.uri or self.settings.wake.command, + ) # Reset from_client_task = None @@ -800,11 +817,7 @@ class SatelliteBase: async def _send_wake_detect(self) -> None: """Inform wake word service of which wake words to detect.""" - wake_names: Optional[List[str]] = None - if self.settings.wake.names: - wake_names = [w.name for w in self.settings.wake.names] - - await self.event_to_wake(Detect(names=wake_names).event()) + await self.event_to_wake(Detect(names=self._wake_names).event()) await self.trigger_detect() # ------------------------------------------------------------------------- @@ -935,7 +948,10 @@ class SatelliteBase: event_client = self._make_event_client() assert event_client is not None await event_client.connect() - _LOGGER.debug("Connected to event service") + _LOGGER.info( + "Connected to event service: %s", + self.settings.event.uri or self.settings.event.command, + ) await event_client.write_event(event) except asyncio.CancelledError: @@ -984,10 +1000,17 @@ class AlwaysStreamingSatellite(SatelliteBase): elif PauseSatellite.is_type(event.type): self.is_streaming = False _LOGGER.info("Satellite paused") + elif Detect.is_type(event.type): + # Wake word detection started + await self.trigger_detect() elif Detection.is_type(event.type): # Start debug recording if self.stt_audio_writer is not None: self.stt_audio_writer.start() + + # Wake word detected + _LOGGER.debug("Wake word detected") + await self.trigger_detection(Detection.from_event(event)) elif Transcript.is_type(event.type) or Error.is_type(event.type): # Stop debug recording if self.stt_audio_writer is not None: @@ -1059,10 +1082,16 @@ class VadStreamingSatellite(SatelliteBase): if RunSatellite.is_type(event.type): self._is_paused = False _LOGGER.info("Waiting for speech") + elif Detect.is_type(event.type): + # Wake word detection started + await self.trigger_detect() elif Detection.is_type(event.type): # Start debug recording if self.stt_audio_writer is not None: self.stt_audio_writer.start() + + _LOGGER.debug("Wake word detected") + await self.trigger_detection(Detection.from_event(event)) elif ( Transcript.is_type(event.type) or Error.is_type(event.type) @@ -1199,8 +1228,17 @@ class WakeStreamingSatellite(SatelliteBase): # wake word id -> seconds self.refractory_timestamp: Dict[Optional[str], float] = {} + # Local VAD for detecting the end of a voice command. + # If this is disabled, the server is expected to do it remotely. + self.vad: Optional[SileroVad] = None if settings.vad.enabled: - _LOGGER.warning("VAD is enabled but will not be used") + _LOGGER.debug("Loading VAD model") + self.vad = SileroVad(threshold=settings.vad.threshold, trigger_level=1) + _LOGGER.info("Loaded VAD model") + + self.segmenter = VoiceCommandSegmenter( + silence_seconds=settings.vad.finished_speaking_seconds + ) # Used for debug audio recording so both wake and stt WAV files have the # same timestamp. @@ -1211,6 +1249,9 @@ class WakeStreamingSatellite(SatelliteBase): self._wake_info: Optional[Info] = None self._wake_info_ready = asyncio.Event() + self._run_pipeline_queue: Queue[RunPipeline] = Queue() + self._next_run_pipeline: Optional[RunPipeline] = None + async def event_from_server(self, event: Event) -> None: # Only check event types once is_run_satellite = False @@ -1221,13 +1262,42 @@ class WakeStreamingSatellite(SatelliteBase): if RunSatellite.is_type(event.type): is_run_satellite = True self._is_paused = False - elif PauseSatellite.is_type(event.type): is_pause_satellite = True + elif Detect.is_type(event.type): + # Request to change active wake words + detect = Detect.from_event(event) + self._wake_names = detect.names + + if not self.is_streaming: + await self._send_wake_detect() + + _LOGGER.debug("Updated wake word names: %s", self._wake_names) elif Transcript.is_type(event.type): is_transcript = True + if (self._next_run_pipeline is not None) and ( + self._next_run_pipeline.end_stage == PipelineStage.ASR + ): + # Done with remotely triggered pipeline + self._next_run_pipeline = None elif Error.is_type(event.type): is_error = True + elif RunPipeline.is_type(event.type): + run_pipeline = RunPipeline.from_event(event) + if not self.is_streaming: + # Run right now + self._next_run_pipeline = run_pipeline + await self._trigger_pipeline(run_pipeline) + return + + # Run after streaming is finished + self._run_pipeline_queue.put(run_pipeline) + elif AudioStop.is_type(event.type): + if (self._next_run_pipeline is not None) and ( + self._next_run_pipeline.end_stage == PipelineStage.TTS + ): + # Done with remotely triggered pipeline + self._next_run_pipeline = None if is_transcript or is_pause_satellite: # Stop streaming before event_from_server is called because it will @@ -1248,20 +1318,20 @@ class WakeStreamingSatellite(SatelliteBase): self._is_paused = True _LOGGER.debug("Satellite is paused") else: - # Go back to wake word detection + # Go back to wake word detection or run next pipeline await self.trigger_streaming_stop() # It's possible to be paused in the middle of streaming - if not self._is_paused: - await self._send_wake_detect() - _LOGGER.info("Waiting for wake word") + if self._is_paused: + return - # Start debug recording (wake) - self._debug_recording_timestamp = time.monotonic_ns() - if self.wake_audio_writer is not None: - self.wake_audio_writer.start( - timestamp=self._debug_recording_timestamp - ) + if self._run_pipeline_queue.empty(): + # Back to wake word detection + await self._start_wake_word_detection() + else: + # Next run from queue + self._next_run_pipeline = self._run_pipeline_queue.get() + await self._trigger_pipeline(self._next_run_pipeline) async def trigger_server_disonnected(self) -> None: await super().trigger_server_disonnected() @@ -1297,13 +1367,26 @@ class WakeStreamingSatellite(SatelliteBase): self.stt_audio_writer.write(audio_bytes) if self.is_streaming: + if self.vad is not None: + if audio_bytes is None: + chunk = AudioChunk.from_event(event) + audio_bytes = chunk.audio + + is_speech = self.vad(audio_bytes) + chunk_seconds = (len(audio_bytes) // 2) / 16000 + if not self.segmenter.process(chunk_seconds, is_speech): + # Send empty audio chunk to indicate command is done + event = AudioChunk( + rate=16000, width=2, channels=1, audio=bytes() + ).event() + # Forward to server await self.event_to_server(event) else: # Forward to wake word service await self.event_to_wake(event) - async def event_from_wake(self, event: Event) -> None: + async def event_from_wake(self, event: Event, is_fake: bool = False) -> None: if Info.is_type(event.type): self._wake_info = Info.from_event(event) self._wake_info_ready.set() @@ -1313,53 +1396,85 @@ class WakeStreamingSatellite(SatelliteBase): # Not detecting or no server connected return - if Detection.is_type(event.type): - detection = Detection.from_event(event) + if not Detection.is_type(event.type): + return - # Check refractory period to avoid multiple back-to-back detections - refractory_timestamp = self.refractory_timestamp.get(detection.name) - if (refractory_timestamp is not None) and ( - refractory_timestamp > time.monotonic() + detection = Detection.from_event(event) + + if (self._next_run_pipeline is not None) and ( + self._next_run_pipeline.end_stage == PipelineStage.WAKE + ): + # Remote pipeline was triggered with only wake word detection + wake_word_name: Optional[str] = None + if detection.name: + wake_word_name = normalize_wake_word(detection.name) + + if (not self._next_run_pipeline.wake_word_names) or ( + wake_word_name in self._next_run_pipeline.wake_word_names ): - _LOGGER.debug("Wake word detection occurred during refractory period") - return - - # Stop debug recording (wake) - if self.wake_audio_writer is not None: - self.wake_audio_writer.stop() - - # Start debug recording (stt) - if self.stt_audio_writer is not None: - self.stt_audio_writer.start(timestamp=self._debug_recording_timestamp) - - _LOGGER.debug(detection) - - self.is_streaming = True - _LOGGER.debug("Streaming audio") - - if self.settings.wake.refractory_seconds is not None: - # Another detection may not occur for this wake word until - # refractory period is over. - self.refractory_timestamp[detection.name] = ( - time.monotonic() + self.settings.wake.refractory_seconds - ) + self._next_run_pipeline = None + await self.event_to_server(event) + await self._send_wake_detect() + _LOGGER.info("Waiting for wake word") else: - # No refractory period - self.refractory_timestamp.pop(detection.name, None) + _LOGGER.debug( + "Got %s but listening for %s", + wake_word_name, + self._next_run_pipeline.wake_word_names, + ) + return + # Check refractory period to avoid multiple back-to-back detections + refractory_timestamp = self.refractory_timestamp.get(detection.name) + if (refractory_timestamp is not None) and ( + refractory_timestamp > time.monotonic() + ): + _LOGGER.debug("Wake word detection occurred during refractory period") + return + + # Stop debug recording (wake) + if self.wake_audio_writer is not None: + self.wake_audio_writer.stop() + + # Start debug recording (stt) + if self.stt_audio_writer is not None: + self.stt_audio_writer.start(timestamp=self._debug_recording_timestamp) + + _LOGGER.debug(detection) + + self.segmenter.reset() + self.is_streaming = True + _LOGGER.debug("Streaming audio") + + if (not is_fake) and (self.settings.wake.refractory_seconds is not None): + # Another detection may not occur for this wake word until + # refractory period is over. + self.refractory_timestamp[detection.name] = ( + time.monotonic() + self.settings.wake.refractory_seconds + ) + else: + # No refractory period + self.refractory_timestamp.pop(detection.name, None) + + if not is_fake: # Forward to the server await self.event_to_server(event) - # Match detected wake word name with pipeline name - pipeline_name: Optional[str] = None - if self.settings.wake.names: - detection_name = normalize_wake_word(detection.name) - for wake_name in self.settings.wake.names: - if normalize_wake_word(wake_name.name) == detection_name: - pipeline_name = wake_name.pipeline - break + start_stage: Optional[PipelineStage] = None + end_stage: Optional[PipelineStage] = None - await self._send_run_pipeline(pipeline_name=pipeline_name) + if self._next_run_pipeline is not None: + # Remote pipeline has been triggered + start_stage = self._next_run_pipeline.start_stage + end_stage = self._next_run_pipeline.end_stage + + await self._send_run_pipeline( + start_stage=start_stage, + end_stage=end_stage, + wake_word_name=detection.name, + ) + + if not is_fake: await self.forward_event(event) # forward to event service await self.trigger_detection(Detection.from_event(event)) await self.trigger_streaming_start() @@ -1379,3 +1494,34 @@ class WakeStreamingSatellite(SatelliteBase): info.wake = self._wake_info.wake except asyncio.TimeoutError: _LOGGER.warning("Failed to get info from wake service") + + async def _start_wake_word_detection(self) -> None: + await self._send_wake_detect() + _LOGGER.info("Waiting for wake word") + + # Start debug recording (wake) + self._debug_recording_timestamp = time.monotonic_ns() + if self.wake_audio_writer is not None: + self.wake_audio_writer.start(timestamp=self._debug_recording_timestamp) + + async def _trigger_pipeline(self, run_pipeline: RunPipeline) -> None: + _LOGGER.debug("Pipeline remotely triggered: %s", run_pipeline) + + if run_pipeline.start_stage == PipelineStage.WAKE: + if run_pipeline.wake_word_names: + run_pipeline.wake_word_names = [ + normalize_wake_word(name) for name in run_pipeline.wake_word_names + ] + + _LOGGER.debug( + "Listening once for wake words: %s", run_pipeline.wake_word_names + ) + await self.event_to_wake(Detect(names=run_pipeline.wake_word_names).event()) + elif run_pipeline.start_stage == PipelineStage.ASR: + _LOGGER.debug("Listening for voice command once") + + # Fake a detection + await self.event_from_wake(Detection().event(), is_fake=True) + elif run_pipeline.start_stage == PipelineStage.TTS: + _LOGGER.debug("Speaking once: %s", run_pipeline.announce_text) + await self.event_to_server(run_pipeline.event()) diff --git a/wyoming_satellite/settings.py b/wyoming_satellite/settings.py index fd9c959..3ec1175 100644 --- a/wyoming_satellite/settings.py +++ b/wyoming_satellite/settings.py @@ -104,19 +104,11 @@ class SndSettings(ServiceSettings): return self.enabled and (self.volume_multiplier != 1.0) -@dataclass(frozen=True) -class WakeWordAndPipeline: - """Wake word name + optional pipeline name.""" - - name: str - pipeline: Optional[str] = None - - @dataclass(frozen=True) class WakeSettings(ServiceSettings): """Wake word service settings.""" - names: Optional[List[WakeWordAndPipeline]] = None + names: Optional[List[str]] = None """List of wake word names to listen for.""" rate: int = 16000 @@ -151,6 +143,9 @@ class VadSettings: wake_word_timeout: Optional[float] = 5.0 """Seconds before going back to sleep if wake word is not detected.""" + finished_speaking_seconds: float = 0.8 + """Seconds to wait after speech before voice command is finished.""" + @dataclass(frozen=True) class EventSettings(ServiceSettings): diff --git a/wyoming_satellite/utils/audio.py b/wyoming_satellite/utils/audio.py index fb74300..304a255 100644 --- a/wyoming_satellite/utils/audio.py +++ b/wyoming_satellite/utils/audio.py @@ -10,7 +10,7 @@ from pyring_buffer import RingBuffer from wyoming.audio import AudioChunk, AudioStart, AudioStop from wyoming.event import Event -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger(__name__) class AudioBuffer: diff --git a/wyoming_satellite/utils/misc.py b/wyoming_satellite/utils/misc.py index 0c8a7c5..5be07cd 100644 --- a/wyoming_satellite/utils/misc.py +++ b/wyoming_satellite/utils/misc.py @@ -12,7 +12,7 @@ from typing import List, Optional, Union from wyoming.event import Eventable -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger(__name__) async def run_event_command( diff --git a/wyoming_satellite/vad.py b/wyoming_satellite/vad.py index bd3a970..9cd4ed0 100644 --- a/wyoming_satellite/vad.py +++ b/wyoming_satellite/vad.py @@ -1,6 +1,10 @@ """Voice activity detection.""" +import logging +from dataclasses import dataclass from typing import Optional +_LOGGER = logging.getLogger(__name__) + class SileroVad: """Voice activity detection with silero VAD.""" @@ -31,3 +35,95 @@ class SileroVad: self._activation = max(0, self._activation - 1) return False + + +@dataclass +class VoiceCommandSegmenter: + """Segments an audio stream into voice commands.""" + + speech_seconds: float = 0.3 + """Seconds of speech before voice command has started.""" + + silence_seconds: float = 1.0 + """Seconds of silence after voice command has ended.""" + + timeout_seconds: float = 15.0 + """Maximum number of seconds before stopping with timeout=True.""" + + reset_seconds: float = 1.0 + """Seconds before reset start/stop time counters.""" + + in_command: bool = False + """True if inside voice command.""" + + _speech_seconds_left: float = 0.0 + """Seconds left before considering voice command as started.""" + + _silence_seconds_left: float = 0.0 + """Seconds left before considering voice command as stopped.""" + + _timeout_seconds_left: float = 0.0 + """Seconds left before considering voice command timed out.""" + + _reset_seconds_left: float = 0.0 + """Seconds left before resetting start/stop time counters.""" + + def __post_init__(self) -> None: + """Reset after initialization.""" + self.reset() + + def reset(self) -> None: + """Reset all counters and state.""" + self._speech_seconds_left = self.speech_seconds + self._silence_seconds_left = self.silence_seconds + self._timeout_seconds_left = self.timeout_seconds + self._reset_seconds_left = self.reset_seconds + self.in_command = False + + def process(self, chunk_seconds: float, is_speech: bool | None) -> bool: + """Process samples using external VAD. + + Returns False when command is done. + """ + self._timeout_seconds_left -= chunk_seconds + if self._timeout_seconds_left <= 0: + _LOGGER.warning( + "VAD end of speech detection timed out after %s seconds", + self.timeout_seconds, + ) + self.reset() + return False + + if not self.in_command: + if is_speech: + self._reset_seconds_left = self.reset_seconds + self._speech_seconds_left -= chunk_seconds + if self._speech_seconds_left <= 0: + # Inside voice command + self.in_command = True + self._silence_seconds_left = self.silence_seconds + _LOGGER.debug("Voice command started") + else: + # Reset if enough silence + self._reset_seconds_left -= chunk_seconds + if self._reset_seconds_left <= 0: + self._speech_seconds_left = self.speech_seconds + self._reset_seconds_left = self.reset_seconds + elif not is_speech: + # Silence in command + self._reset_seconds_left = self.reset_seconds + self._silence_seconds_left -= chunk_seconds + if self._silence_seconds_left <= 0: + # Command finished successfully + self.reset() + _LOGGER.debug("Voice command finished") + return False + else: + # Speech in command. + # Reset silence counter if enough speech. + self._reset_seconds_left -= chunk_seconds + if self._reset_seconds_left <= 0: + self._silence_seconds_left = self.silence_seconds + self._reset_seconds_left = self.reset_seconds + + return True