From 56b61003158f2b8359cc4dc440f9009ceaf6cd7d Mon Sep 17 00:00:00 2001 From: Derek Date: Sun, 5 Jan 2025 22:17:51 -0500 Subject: [PATCH 01/11] opt-in to bttv --- src/ovtk_audiencekit/chats/Twitch/TwitchProcess.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ovtk_audiencekit/chats/Twitch/TwitchProcess.py b/src/ovtk_audiencekit/chats/Twitch/TwitchProcess.py index f66734b..4dde8eb 100644 --- a/src/ovtk_audiencekit/chats/Twitch/TwitchProcess.py +++ b/src/ovtk_audiencekit/chats/Twitch/TwitchProcess.py @@ -27,6 +27,8 @@ class TwitchProcess(ChatProcess): botname=None, emote_res=4.0, # EventSub options eventsub=True, eventsub_host='wss://ovtk.skeh.site/twitch', + # BTTV integration + bttv=False, # Inheritance boilerplate **kwargs): super().__init__(*args, **kwargs) @@ -61,7 +63,7 @@ class TwitchProcess(ChatProcess): self.eventsub = TwitchEventSub(self.api, eventsub_host) self._sources.append(self.eventsub) - self.bttv = BTTV(target_data['user']['id']) + self.bttv = BTTV(target_data['user']['id']) if bttv else None def loop(self, next_state): @@ -98,7 +100,7 @@ class TwitchProcess(ChatProcess): for event in chain(*(source.read(0.1) for source in self._sources)): # Retarget event event.via = self._name - if isinstance(event, Message): + if self.bttv and isinstance(event, Message): event = self.bttv.hydrate(event) self.publish(event) return 0 -- 2.48.1 From 82691789f78b81dcc6c803bbf387b0ddb11388ec Mon Sep 17 00:00:00 2001 From: Derek Date: Sun, 5 Jan 2025 23:14:39 -0500 Subject: [PATCH 02/11] Make setup more sensible --- src/ovtk_audiencekit/core/MainProcess.py | 4 +- src/ovtk_audiencekit/core/PluginBase.py | 18 ++++++++- .../plugins/AudioAlert/AudioAlert.py | 38 ++++++++++++++----- .../plugins/Jail/JailPlugin.py | 3 +- src/ovtk_audiencekit/plugins/OBS/obs.py | 16 +++----- src/ovtk_audiencekit/plugins/OSC/osc.py | 3 +- .../plugins/PhraseCounter/PhraseCounter.py | 3 +- .../plugins/Shoutout/ShoutoutPlugin.py | 6 +-- src/ovtk_audiencekit/plugins/TTS/TTS.py | 6 +-- 9 files changed, 61 insertions(+), 36 deletions(-) diff --git a/src/ovtk_audiencekit/core/MainProcess.py b/src/ovtk_audiencekit/core/MainProcess.py index 863fd7c..9b9d692 100644 --- a/src/ovtk_audiencekit/core/MainProcess.py +++ b/src/ovtk_audiencekit/core/MainProcess.py @@ -219,9 +219,9 @@ class MainProcess: plugin_module = import_or_reload_mod(module_name, default_package='ovtk_audiencekit.plugins', external=False) - plugin = plugin_module.Plugin(self.chat_processes, self.event_queue, plugin_name, global_ctx, - **node.props, **secrets_for_mod, _children=node.nodes) + plugin = plugin_module.Plugin(self.chat_processes, self.event_queue, plugin_name, global_ctx) self.plugins[plugin_name] = plugin + await plugin._setup(*node.args[1:], **node.props, **secrets_for_mod) # Register UI with webserver self.webserver.register_blueprint(plugin.blueprint) except Exception as e: diff --git a/src/ovtk_audiencekit/core/PluginBase.py b/src/ovtk_audiencekit/core/PluginBase.py index e16cafd..cd814b3 100644 --- a/src/ovtk_audiencekit/core/PluginBase.py +++ b/src/ovtk_audiencekit/core/PluginBase.py @@ -86,6 +86,18 @@ class PluginBase(ABC): raise e raise PluginError(self._name, str(e)) from e + async def _setup(self, *args, **kwargs): + try: + res = self.setup(*args, **kwargs) + if asyncio.iscoroutinefunction(self.setup): + return await res + else: + return res + except Exception as e: + if isinstance(e, KeyboardInterrupt): + raise e + raise PluginError(self._name, str(e)) from e + # Base class helpers def broadcast(self, event): """Send event to every active chat""" @@ -123,6 +135,10 @@ class PluginBase(ABC): self._event_queue.put_nowait(event) # User-defined + async def setup(self, *args, **kwargs): + """Called when plugin is being loaded.""" + pass + def close(self): """Called when plugin is about to be unloaded. Use this to safely close any resouces if needed""" pass @@ -143,7 +159,7 @@ class PluginBase(ABC): pass @abstractmethod - async def run(self, _children=None, _ctx={}, **kwargs): + async def run(self, *args, _children=None, _ctx={}, **kwargs): """ Run plugin action, either due to a definition in the config, or due to another plugin """ diff --git a/src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py b/src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py index dd48ab8..5f9b8ea 100644 --- a/src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py +++ b/src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py @@ -1,11 +1,11 @@ import asyncio +from collections import deque from ovtk_audiencekit.plugins import PluginBase from ovtk_audiencekit.core import Clip class AudioAlert(PluginBase): - def __init__(self, *args, output=None, buffer_length=2048, cutoff_prevention_buffers=None, **kwargs): - super().__init__(*args, **kwargs) + def setup(self, output=None, buffer_length=2048, cutoff_prevention_buffers=None): if cutoff_prevention_buffers: self.logger.info('`cutoff_prevention_buffers` are depricated') @@ -13,13 +13,33 @@ class AudioAlert(PluginBase): self._buffer_length = int(buffer_length) self._output_index = Clip.find_output_index(output) - def run(self, path, speed=1, immediate=True, **kwargs): - if self.sounds.get(path) is None: - self.sounds[path] = Clip(path, - self._output_index, - buffer_length=self._buffer_length, - speed=speed) - sound = self.sounds.get(path) + def run(self, path, speed=1, immediate=True, poly=1, **kwargs): + sound = None + + if poly != 1: + poly = int(poly) + sound_dq = self.sounds.get(path) + if sound_dq is None or type(sound_dq) != deque or sound_dq.maxlen != poly: + sound_dq = deque(maxlen=poly) + self.sounds[path] = sound_dq + if len(sound_dq) != poly: + self.logger.debug("filling", len(sound_dq), poly, sound_dq) + sound = Clip(path, + self._output_index, + buffer_length=self._buffer_length, + speed=speed) + sound_dq.append(sound) + else: + self.logger.debug("rotate", len(sound_dq), poly, sound_dq) + sound_dq.rotate(1) + sound = sound_dq[0] + else: + if self.sounds.get(path) is None: + self.sounds[path] = Clip(path, + self._output_index, + buffer_length=self._buffer_length, + speed=speed) + sound = self.sounds.get(path) if immediate: asyncio.create_task(sound.aplay()) diff --git a/src/ovtk_audiencekit/plugins/Jail/JailPlugin.py b/src/ovtk_audiencekit/plugins/Jail/JailPlugin.py index d82b5c8..9426054 100644 --- a/src/ovtk_audiencekit/plugins/Jail/JailPlugin.py +++ b/src/ovtk_audiencekit/plugins/Jail/JailPlugin.py @@ -35,8 +35,7 @@ owomap = { } class JailPlugin(PluginBase): - def __init__(self, *args, min_level='vip', persist=True, **kwargs): - super().__init__(*args, **kwargs) + def setup(self, min_level='vip', persist=True): self.persist = persist self._cache = os.path.join(CACHE_DIR, 'Jail', 'sentences') os.makedirs(os.path.dirname(self._cache), exist_ok=True) diff --git a/src/ovtk_audiencekit/plugins/OBS/obs.py b/src/ovtk_audiencekit/plugins/OBS/obs.py index 644bbe2..eeae676 100644 --- a/src/ovtk_audiencekit/plugins/OBS/obs.py +++ b/src/ovtk_audiencekit/plugins/OBS/obs.py @@ -6,19 +6,15 @@ from ovtk_audiencekit.plugins import PluginBase class OBSWSPlugin(PluginBase): - def __init__(self, *args, password=None, uri='ws://localhost:4455', **kwargs): - super().__init__(*args, **kwargs) + async def setup(self, password=None, uri='ws://localhost:4455'): self.uri = uri self.obsws = simpleobsws.WebSocketClient(url=uri, password=password) - asyncio.get_event_loop().run_until_complete(self.setup()) - - async def setup(self): - await self.obsws.connect() - success = await self.obsws.wait_until_identified() - if not success: - await self.obsws.disconnect() - raise RuntimeError(f'Could not connect to OBS websocket at {self.uri}') + await self.obsws.connect() + success = await self.obsws.wait_until_identified() + if not success: + await self.obsws.disconnect() + raise RuntimeError(f'Could not connect to OBS websocket at {self.uri}') async def run(self, type, _children=None, _ctx={}, **kwargs): req = simpleobsws.Request(type, requestData=kwargs) diff --git a/src/ovtk_audiencekit/plugins/OSC/osc.py b/src/ovtk_audiencekit/plugins/OSC/osc.py index fc06234..0416b79 100644 --- a/src/ovtk_audiencekit/plugins/OSC/osc.py +++ b/src/ovtk_audiencekit/plugins/OSC/osc.py @@ -4,8 +4,7 @@ from ovtk_audiencekit.plugins import PluginBase class OSCPlugin(PluginBase): - def __init__(self, *args, ip='localhost', port=None, **kwargs): - super().__init__(*args, **kwargs) + def setup(self, ip='localhost', port=None): if port is None: raise RuntimeError('A unique port must be specified') self.client = SimpleUDPClient(ip, int(port)) diff --git a/src/ovtk_audiencekit/plugins/PhraseCounter/PhraseCounter.py b/src/ovtk_audiencekit/plugins/PhraseCounter/PhraseCounter.py index 0a9fec2..0868b00 100644 --- a/src/ovtk_audiencekit/plugins/PhraseCounter/PhraseCounter.py +++ b/src/ovtk_audiencekit/plugins/PhraseCounter/PhraseCounter.py @@ -64,8 +64,7 @@ class PhraseCounter: class PhraseCounterPlugin(PluginBase): - def __init__(self, *args, debounce_time=1, persist=False, **kwargs): - super().__init__(*args, **kwargs) + def setup(self, debounce_time=1, persist=False): self.debounce_time = debounce_time self.persist = persist diff --git a/src/ovtk_audiencekit/plugins/Shoutout/ShoutoutPlugin.py b/src/ovtk_audiencekit/plugins/Shoutout/ShoutoutPlugin.py index 0e20c43..61afd6e 100644 --- a/src/ovtk_audiencekit/plugins/Shoutout/ShoutoutPlugin.py +++ b/src/ovtk_audiencekit/plugins/Shoutout/ShoutoutPlugin.py @@ -9,10 +9,8 @@ from ovtk_audiencekit.chats.Twitch import Process as Twitch class ShoutoutPlugin(PluginBase): - def __init__(self, *args, command='so', min_level='vip', - text='Check out {link}!~ They were last streaming {last_game}', - **kwargs): - super().__init__(*args, **kwargs) + def setup(self, command='so', min_level='vip', + text='Check out {link}!~ They were last streaming {last_game}'): self.text = text if command: self.command = Command(name=command, help='Shoutout another user', required_level=min_level) diff --git a/src/ovtk_audiencekit/plugins/TTS/TTS.py b/src/ovtk_audiencekit/plugins/TTS/TTS.py index 124c61b..a311f95 100644 --- a/src/ovtk_audiencekit/plugins/TTS/TTS.py +++ b/src/ovtk_audiencekit/plugins/TTS/TTS.py @@ -13,10 +13,8 @@ from ovtk_audiencekit.core.Data import CACHE_DIR class TextToSpeechPlugin(PluginBase): - def __init__(self, *args, output=None, cuda=None, - engine="tts_models/en/ljspeech/tacotron2-DDC", speaker_wav=None, - _children=None, **kwargs): - super().__init__(*args, _children=_children) + def setup(self, output=None, cuda=None, + engine="tts_models/en/ljspeech/tacotron2-DDC", speaker_wav=None, **kwargs): self.speaker_wav = speaker_wav -- 2.48.1 From dfc87bb36f17a00b04af8b58303d9c50e778f4d7 Mon Sep 17 00:00:00 2001 From: Derek Date: Sun, 5 Jan 2025 23:20:02 -0500 Subject: [PATCH 03/11] Fix indent --- src/ovtk_audiencekit/plugins/OBS/obs.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ovtk_audiencekit/plugins/OBS/obs.py b/src/ovtk_audiencekit/plugins/OBS/obs.py index eeae676..3c56fd9 100644 --- a/src/ovtk_audiencekit/plugins/OBS/obs.py +++ b/src/ovtk_audiencekit/plugins/OBS/obs.py @@ -10,11 +10,11 @@ class OBSWSPlugin(PluginBase): self.uri = uri self.obsws = simpleobsws.WebSocketClient(url=uri, password=password) - await self.obsws.connect() - success = await self.obsws.wait_until_identified() - if not success: - await self.obsws.disconnect() - raise RuntimeError(f'Could not connect to OBS websocket at {self.uri}') + await self.obsws.connect() + success = await self.obsws.wait_until_identified() + if not success: + await self.obsws.disconnect() + raise RuntimeError(f'Could not connect to OBS websocket at {self.uri}') async def run(self, type, _children=None, _ctx={}, **kwargs): req = simpleobsws.Request(type, requestData=kwargs) -- 2.48.1 From 1e2b88f1c9d71d4c3bd2a546ecdd89fd874e9098 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 6 Jan 2025 00:41:05 -0500 Subject: [PATCH 04/11] [plugins/scene] allow oneshot scenes --- .../plugins/builtins/Scene/Plugin.py | 13 ++++++++----- .../plugins/builtins/Scene/templates/index.html | 6 ++++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/ovtk_audiencekit/plugins/builtins/Scene/Plugin.py b/src/ovtk_audiencekit/plugins/builtins/Scene/Plugin.py index 23119af..21b56c2 100644 --- a/src/ovtk_audiencekit/plugins/builtins/Scene/Plugin.py +++ b/src/ovtk_audiencekit/plugins/builtins/Scene/Plugin.py @@ -14,6 +14,7 @@ from ovtk_audiencekit.utils import format_exception class Scene: name: str group: str + oneshot: bool enter: Callable exit: Callable entry_context: dict = field(default_factory=dict) @@ -35,16 +36,16 @@ class ScenePlugin(PluginBase): self.blueprint.add_url_rule('//', 'api-sceneset', self.ui_setscene) self.blueprint.add_url_rule('/monitor', 'monitor', self.ui_monitor_ws, is_websocket=True) - async def run(self, name, _children=None, _ctx={}, active=None, group=None, immediate=True, **kwargs): + async def run(self, name, _children=None, _ctx={}, active=None, group=None, immediate=True, oneshot=False, **kwargs): if _children is None and active is None: raise UsageError('Either define a new scene or set `--active` to true / false') if _children: - await self.define(name, group, _children, default_active=active, ctx=_ctx) + await self.define(name, group, _children, default_active=active, oneshot=oneshot, ctx=_ctx) else: await self.switch(name, active, is_immediate=immediate, ctx=_ctx) - async def define(self, name, group, children, default_active=False, ctx={}): + async def define(self, name, group, children, default_active=False, oneshot=False, ctx={}): if self.scenes.get(name) is not None: raise UsageError(f'Scene with name "{name}" already exists!') @@ -63,7 +64,7 @@ class ScenePlugin(PluginBase): async def exit(ctx): await self.execute_kdl(exit_nodes, _ctx=ctx) - scene = Scene(name, group, enter, exit) + scene = Scene(name, group, oneshot, enter, exit) self.scenes[name] = scene if default_active: @@ -74,7 +75,9 @@ class ScenePlugin(PluginBase): if scene is None: raise UsageError(f'No defined scene with name "{name}"') - if active: + if scene.oneshot: + await self._execute(scene, 'enter', is_immediate, ctx) + elif active: if current := self.active.get(scene.group): if current == scene: return diff --git a/src/ovtk_audiencekit/plugins/builtins/Scene/templates/index.html b/src/ovtk_audiencekit/plugins/builtins/Scene/templates/index.html index 5e2de71..10f10f0 100644 --- a/src/ovtk_audiencekit/plugins/builtins/Scene/templates/index.html +++ b/src/ovtk_audiencekit/plugins/builtins/Scene/templates/index.html @@ -24,8 +24,10 @@ }) }) const toggle = async (group_name, scene_name) => { - if (inflight.value.includes(scene_name)) return - inflight.value.push(scene_name) + if (!groups.value[group_name][scene_name].oneshot) { + if (inflight.value.includes(scene_name)) return + inflight.value.push(scene_name) + } const next_state = !groups.value[group_name][scene_name] await fetch(`${scene_name}/${next_state ? 'activate' : 'deactivate'}`, { method: 'GET' }) } -- 2.48.1 From 50eb6b594d062db94bd0142622b8518fad01a8f6 Mon Sep 17 00:00:00 2001 From: Derek Schmidt Date: Tue, 7 Jan 2025 21:18:08 -0500 Subject: [PATCH 05/11] Fix cue cleanup behavior --- src/ovtk_audiencekit/plugins/builtins/Cue.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ovtk_audiencekit/plugins/builtins/Cue.py b/src/ovtk_audiencekit/plugins/builtins/Cue.py index ef3ebf9..2d7c701 100644 --- a/src/ovtk_audiencekit/plugins/builtins/Cue.py +++ b/src/ovtk_audiencekit/plugins/builtins/Cue.py @@ -74,7 +74,7 @@ class CuePlugin(PluginBase): cue = Cue(repeat, **kwargs) async def handler(): - # Repetion management + # Repetition management if not cue.enabled: return if cue.repeat: @@ -114,7 +114,7 @@ class CuePlugin(PluginBase): async def _cleanup(self): while True: await asyncio.sleep(60) - for name, (cue, _) in self.cues.items(): + for name, (cue, _) in list(self.cues.items()): if cue.is_obsolete(): del self.cues[name] if task := self.tasks.get(name): -- 2.48.1 From 27f0997d6a5c71afd17e5003047eb3e096a83f1c Mon Sep 17 00:00:00 2001 From: Derek Schmidt Date: Tue, 7 Jan 2025 21:35:50 -0500 Subject: [PATCH 06/11] Fix audio memory leak --- .../core/{Clip.py => Audio.py} | 114 +++++++++--------- src/ovtk_audiencekit/core/__init__.py | 2 +- .../plugins/AudioAlert/AudioAlert.py | 94 ++++++++++----- src/ovtk_audiencekit/plugins/TTS/TTS.py | 27 +++-- 4 files changed, 133 insertions(+), 104 deletions(-) rename src/ovtk_audiencekit/core/{Clip.py => Audio.py} (53%) diff --git a/src/ovtk_audiencekit/core/Clip.py b/src/ovtk_audiencekit/core/Audio.py similarity index 53% rename from src/ovtk_audiencekit/core/Clip.py rename to src/ovtk_audiencekit/core/Audio.py index 78d3f68..551b9b5 100644 --- a/src/ovtk_audiencekit/core/Clip.py +++ b/src/ovtk_audiencekit/core/Audio.py @@ -25,71 +25,59 @@ os.close(old_stderr) logger = logging.getLogger(__name__) - -def check_rate(index, channels, rate): - try: - return pyaudio.is_format_supported(rate, - output_channels=channels, - output_device=index, - output_format=pya.paFloat32) - except ValueError: - return False - -alt_rates = [44100, 48000] class Clip: - def __init__(self, path, output_index, buffer_length=2048, speed=1, force_stereo=True): - _raw, native_rate = librosa.load(path, sr=None, dtype='float32', mono=False) - self._channels = _raw.shape[0] if len(_raw.shape) == 2 else 1 - if force_stereo and self._channels == 1: - _raw = np.resize(_raw, (2,*_raw.shape)) - self._channels = 2 + def __init__(self, path, samplerate=None, speed=1, force_stereo=True): + self.path = path + raw, native_rate = librosa.load(self.path, sr=None, dtype='float32', mono=False) - target_samplerate = native_rate - if not check_rate(output_index, self._channels , native_rate): - try: - target_samplerate = next((rate for rate in alt_rates if check_rate(output_index, self._channels , rate))) - except StopIteration: - logger.warn('Target audio device does not claim to support any sample rates! Attempting playback at native rate') - self._samplerate = target_samplerate + self.channels = raw.shape[0] if len(raw.shape) == 2 else 1 + if force_stereo and self.channels == 1: + raw = np.resize(raw, (2,*raw.shape)) + self.channels = 2 - if native_rate != self._samplerate: - _raw = librosa.resample(_raw, native_rate, self._samplerate, fix=True, scale=True) + self.samplerate = samplerate or native_rate + if native_rate != self.samplerate: + raw = librosa.resample(raw, native_rate, self.samplerate, fix=True, scale=True) - self._raw = np.ascontiguousarray(self._stereo_transpose(_raw), dtype='float32') + self.raw = np.ascontiguousarray(self._stereo_transpose(raw), dtype='float32') if speed != 1: self.stretch(speed) - self._pos = 0 - self._playing = False + @property + def length(self): + return self.raw.shape[0] / self.samplerate + + def _stereo_transpose(self, ndata): + return ndata if self.channels == 1 else ndata.T + + def stretch(self, speed): + stretched = tsm.wsola(self._stereo_transpose(self.raw), speed) + self.raw = np.ascontiguousarray(self._stereo_transpose(stretched), dtype='float32') + + def save(self, filename): + soundfile.write(filename, self._stereo_transpose(self.raw), self.samplerate) + + +class Stream: + def __init__(self, clip, output_index, buffer_length=4096): + self.clip = clip + self.pos = 0 + self.playing = False self._end_event = AioEvent() self._stream = pyaudio.open( output_device_index=output_index, format=pya.paFloat32, - channels=self._channels, - rate=self._samplerate, + channels=self.clip.channels, + rate=self.clip.samplerate, frames_per_buffer=buffer_length, output=True, stream_callback=self._read_callback, start=False) - @property - def length(self): - return self._raw.shape[0] / self._samplerate - - def _stereo_transpose(self, ndata): - return ndata if self._channels == 1 else ndata.T - - def stretch(self, speed): - stretched = tsm.wsola(self._stereo_transpose(self._raw), speed) - self._raw = np.ascontiguousarray(self._stereo_transpose(stretched), dtype='float32') - - def save(self, filename): - soundfile.write(filename, self._stereo_transpose(self._raw), self._samplerate) - def _play(self): - self._playing = True - self._pos = 0 + self.playing = True + self.pos = 0 if not self._stream.is_active(): self._stream.start_stream() @@ -97,38 +85,48 @@ class Clip: def play(self): self._end_event.clear() self._play() - self._end_event.wait(timeout=self.length) + self._end_event.wait(timeout=self.clip.length) async def aplay(self): self._end_event.clear() self._play() try: - await self._end_event.coro_wait(timeout=self.length) + await self._end_event.coro_wait(timeout=self.clip.length) except asyncio.CancelledError: - self._playing = False + self.playing = False self._stream.stop_stream() def close(self): self._stream.close() def _read_callback(self, in_data, frame_count, time_info, status): - if self._channels > 1: - buffer = np.zeros((frame_count, self._channels), dtype='float32') + if self.clip.channels > 1: + buffer = np.zeros((frame_count, self.clip.channels), dtype='float32') else: buffer = np.zeros((frame_count,), dtype='float32') - if self._playing: - newpos = self._pos + frame_count - clip_chunk = self._raw[self._pos:newpos] - self._pos = newpos + if self.playing: + newpos = self.pos + frame_count + clip_chunk = self.clip.raw[self.pos:newpos] + self.pos = newpos buffer[0:clip_chunk.shape[0]] = clip_chunk - if self._pos >= self._raw.shape[0]: - self._playing = False + if self.pos >= self.clip.raw.shape[0]: + self.playing = False self._end_event.set() return buffer, pya.paContinue + @staticmethod + def check_rate(index, channels, rate): + try: + return pyaudio.is_format_supported(rate, + output_channels=channels, + output_device=index, + output_format=pya.paFloat32) + except ValueError: + return False + @staticmethod def find_output_index(output): if output is None: diff --git a/src/ovtk_audiencekit/core/__init__.py b/src/ovtk_audiencekit/core/__init__.py index 17c86a4..5de5ce3 100644 --- a/src/ovtk_audiencekit/core/__init__.py +++ b/src/ovtk_audiencekit/core/__init__.py @@ -1,3 +1,3 @@ from .WebsocketServerProcess import WebsocketServerProcess from .MainProcess import MainProcess -from .Clip import Clip +from .Audio import Clip, Stream diff --git a/src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py b/src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py index 5f9b8ea..2f29cb7 100644 --- a/src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py +++ b/src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py @@ -1,47 +1,75 @@ import asyncio from collections import deque +import maya + from ovtk_audiencekit.plugins import PluginBase -from ovtk_audiencekit.core import Clip +from ovtk_audiencekit.core import Clip, Stream class AudioAlert(PluginBase): - def setup(self, output=None, buffer_length=2048, cutoff_prevention_buffers=None): - if cutoff_prevention_buffers: - self.logger.info('`cutoff_prevention_buffers` are depricated') + def setup(self, output=None, timeout_min=1, sample_rate=None, buffer_length=4096, force_stereo=True): + self.force_stereo = force_stereo + self.timeout_min = timeout_min + self.clips = {} + self.streams = {} + self.buffer_length = int(buffer_length) + self.output_index = Stream.find_output_index(output) + if sample_rate is None: + try: + sample_rate = next((rate for rate in [44100, 48000] if Stream.check_rate(self.output_index, 1, rate))) + except StopIteration: + self.logger.warn('Target audio device does not claim to support common sample rates! Attempting playback at native rate of audio') - self.sounds = {} - self._buffer_length = int(buffer_length) - self._output_index = Clip.find_output_index(output) + self._cleanup_task = asyncio.create_task(self._cleanup()) def run(self, path, speed=1, immediate=True, poly=1, **kwargs): - sound = None + poly = int(poly) + key = f'{path}@{speed}x' + clip = self.clips.get(key, [None, None])[0] - if poly != 1: - poly = int(poly) - sound_dq = self.sounds.get(path) - if sound_dq is None or type(sound_dq) != deque or sound_dq.maxlen != poly: - sound_dq = deque(maxlen=poly) - self.sounds[path] = sound_dq - if len(sound_dq) != poly: - self.logger.debug("filling", len(sound_dq), poly, sound_dq) - sound = Clip(path, - self._output_index, - buffer_length=self._buffer_length, - speed=speed) - sound_dq.append(sound) - else: - self.logger.debug("rotate", len(sound_dq), poly, sound_dq) - sound_dq.rotate(1) - sound = sound_dq[0] + if clip is None: + clip = Clip(path, speed=speed, force_stereo=self.force_stereo) + self.clips[key] = [clip, maya.now()] else: - if self.sounds.get(path) is None: - self.sounds[path] = Clip(path, - self._output_index, - buffer_length=self._buffer_length, - speed=speed) - sound = self.sounds.get(path) + self.clips[key][1] = maya.now() + + stream_dq, refs = self.streams.get(path, (None, set())) + if stream_dq is None: + stream_dq = deque(maxlen=poly) + self.streams[path] = (stream_dq, refs) + refs.add(key) + + if stream_dq.maxlen != poly: + self.logger.warn('Cannot change poly while streams are active!') + + if len(stream_dq) != poly: + stream = Stream(clip, self.output_index, + buffer_length=self.buffer_length) + stream_dq.append(stream) + else: + stream_dq.rotate(1) + stream = stream_dq[0] if immediate: - asyncio.create_task(sound.aplay()) + asyncio.create_task(stream.aplay()) else: - sound.play() + stream.play() + + async def _cleanup(self): + while True: + await asyncio.sleep(60) + now = maya.now() + for key, [clip, last_used] in list(self.clips.items()): + if now >= last_used.add(minutes=self.timeout_min, seconds=clip.length): + del self.clips[key] + self.logger.debug(f'Dropping {key}') + + streams, refs = self.streams.get(clip.path, (None, None)) + if refs: + refs.remove(key) + self.logger.debug(f'Stream {clip.path} now refs {refs}') + if len(refs) == 0: + self.logger.debug('Closing streams...') + for stream in streams: + stream.close() + del self.streams[clip.path] diff --git a/src/ovtk_audiencekit/plugins/TTS/TTS.py b/src/ovtk_audiencekit/plugins/TTS/TTS.py index a311f95..328cb68 100644 --- a/src/ovtk_audiencekit/plugins/TTS/TTS.py +++ b/src/ovtk_audiencekit/plugins/TTS/TTS.py @@ -8,7 +8,7 @@ from TTS.config import load_config from ovtk_audiencekit.plugins import PluginBase from ovtk_audiencekit.events import Message, SysMessage -from ovtk_audiencekit.core import Clip +from ovtk_audiencekit.core import Clip, Stream from ovtk_audiencekit.core.Data import CACHE_DIR @@ -18,12 +18,12 @@ class TextToSpeechPlugin(PluginBase): self.speaker_wav = speaker_wav - self._output_index = Clip.find_output_index(output) + self.output_index = Stream.find_output_index(output) conf_overrides = {k[2:]: v for k, v in kwargs.items() if k.startswith('o_')} - self._cache = os.path.join(CACHE_DIR, 'tts') - os.makedirs(os.path.dirname(self._cache), exist_ok=True) + self.cache_dir = os.path.join(CACHE_DIR, 'tts') + os.makedirs(os.path.dirname(self.cache_dir), exist_ok=True) self.cuda = cuda @@ -36,7 +36,7 @@ class TextToSpeechPlugin(PluginBase): vocoder_path, vocoder_config_path = None, None if conf_overrides: - override_conf_path = os.path.join(self._cache, f'{self._name}_override.json') + override_conf_path = os.path.join(self.cache_dir, f'{self._name}_override.json') config = load_config(config_path) for key, value in conf_overrides.items(): @@ -55,7 +55,7 @@ class TextToSpeechPlugin(PluginBase): def make_tts_wav(self, text, filename=None): if filename is None: - filename = os.path.join(self._cache, f'{uuid.uuid1()}.wav') + filename = os.path.join(self.cache_dir, f'{uuid.uuid1()}.wav') if self.speaker_wav: wav = self.synthesizer.tts(text, None, 'en', self.speaker_wav) @@ -72,17 +72,20 @@ class TextToSpeechPlugin(PluginBase): text += '.' filename = self.make_tts_wav(text) # TODO: Play direct from memory - clip = Clip(filename, self._output_index, force_stereo=False) + clip = Clip(filename, force_stereo=True) + stream = Stream(clip, self.output_index) if wait: async def play(): - await clip.aplay() - clip.close() + await stream.aplay() + stream.close() + os.remove(os.path.join(self.cache_dir, filename)) asyncio.create_task(play()) else: - clip.play() - clip.close() + stream.play() + stream.close() + os.remove(os.path.join(self.cache_dir, filename)) except Exception as e: - print(e) + self.logger.error(f"Failed to make speech from input: {e}") if source_event := _ctx.get('event'): msg = SysMessage(self._name, 'Failed to make speech from input!!') -- 2.48.1 From 69fc34396fcff6a93094b9dcee7dcf91c23de4d7 Mon Sep 17 00:00:00 2001 From: Derek Schmidt Date: Tue, 7 Jan 2025 23:31:51 -0500 Subject: [PATCH 07/11] Pitch fun --- src/ovtk_audiencekit/core/Audio.py | 11 +++++--- .../plugins/AudioAlert/AudioAlert.py | 27 +++++++------------ 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/ovtk_audiencekit/core/Audio.py b/src/ovtk_audiencekit/core/Audio.py index 551b9b5..53e7d40 100644 --- a/src/ovtk_audiencekit/core/Audio.py +++ b/src/ovtk_audiencekit/core/Audio.py @@ -26,7 +26,7 @@ os.close(old_stderr) logger = logging.getLogger(__name__) class Clip: - def __init__(self, path, samplerate=None, speed=1, force_stereo=True): + def __init__(self, path, samplerate=None, speed=1, keep_pitch=True, force_stereo=True): self.path = path raw, native_rate = librosa.load(self.path, sr=None, dtype='float32', mono=False) @@ -42,7 +42,7 @@ class Clip: self.raw = np.ascontiguousarray(self._stereo_transpose(raw), dtype='float32') if speed != 1: - self.stretch(speed) + self.stretch(speed, keep_pitch=keep_pitch) @property def length(self): @@ -51,8 +51,11 @@ class Clip: def _stereo_transpose(self, ndata): return ndata if self.channels == 1 else ndata.T - def stretch(self, speed): - stretched = tsm.wsola(self._stereo_transpose(self.raw), speed) + def stretch(self, speed, keep_pitch=True): + if keep_pitch: + stretched = tsm.wsola(self._stereo_transpose(self.raw), speed) + else: + stretched = librosa.resample(self._stereo_transpose(self.raw), self.samplerate * (1 / speed), self.samplerate, fix=False, scale=True) self.raw = np.ascontiguousarray(self._stereo_transpose(stretched), dtype='float32') def save(self, filename): diff --git a/src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py b/src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py index 2f29cb7..bf445c7 100644 --- a/src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py +++ b/src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py @@ -22,22 +22,21 @@ class AudioAlert(PluginBase): self._cleanup_task = asyncio.create_task(self._cleanup()) - def run(self, path, speed=1, immediate=True, poly=1, **kwargs): + def run(self, path, speed=1, keep_pitch=False, immediate=True, poly=1, **kwargs): poly = int(poly) - key = f'{path}@{speed}x' + key = f'{path}@{speed}{"X" if keep_pitch else "x"}' clip = self.clips.get(key, [None, None])[0] if clip is None: - clip = Clip(path, speed=speed, force_stereo=self.force_stereo) + clip = Clip(path, speed=speed, keep_pitch=keep_pitch, force_stereo=self.force_stereo) self.clips[key] = [clip, maya.now()] else: self.clips[key][1] = maya.now() - stream_dq, refs = self.streams.get(path, (None, set())) + stream_dq = self.streams.get(path, None) if stream_dq is None: stream_dq = deque(maxlen=poly) - self.streams[path] = (stream_dq, refs) - refs.add(key) + self.streams[key] = stream_dq if stream_dq.maxlen != poly: self.logger.warn('Cannot change poly while streams are active!') @@ -61,15 +60,9 @@ class AudioAlert(PluginBase): now = maya.now() for key, [clip, last_used] in list(self.clips.items()): if now >= last_used.add(minutes=self.timeout_min, seconds=clip.length): - del self.clips[key] self.logger.debug(f'Dropping {key}') - - streams, refs = self.streams.get(clip.path, (None, None)) - if refs: - refs.remove(key) - self.logger.debug(f'Stream {clip.path} now refs {refs}') - if len(refs) == 0: - self.logger.debug('Closing streams...') - for stream in streams: - stream.close() - del self.streams[clip.path] + streams = self.streams.get(key, []) + for stream in streams: + stream.close() + del self.streams[key] + del self.clips[key] -- 2.48.1 From 4bbd2b1bb0db7c3724011cd3628ae160278710a6 Mon Sep 17 00:00:00 2001 From: Derek Schmidt Date: Tue, 7 Jan 2025 23:56:34 -0500 Subject: [PATCH 08/11] Fix poly --- src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py b/src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py index bf445c7..f694404 100644 --- a/src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py +++ b/src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py @@ -33,7 +33,7 @@ class AudioAlert(PluginBase): else: self.clips[key][1] = maya.now() - stream_dq = self.streams.get(path, None) + stream_dq = self.streams.get(key, None) if stream_dq is None: stream_dq = deque(maxlen=poly) self.streams[key] = stream_dq @@ -41,13 +41,14 @@ class AudioAlert(PluginBase): if stream_dq.maxlen != poly: self.logger.warn('Cannot change poly while streams are active!') - if len(stream_dq) != poly: + if len(stream_dq) == stream_dq.maxlen: + stream_dq.rotate(1) + stream = stream_dq[0] + else: stream = Stream(clip, self.output_index, buffer_length=self.buffer_length) stream_dq.append(stream) - else: - stream_dq.rotate(1) - stream = stream_dq[0] + if immediate: asyncio.create_task(stream.aplay()) -- 2.48.1 From 4bcb37030a42fffe9bc4d56cc68d922a469d9437 Mon Sep 17 00:00:00 2001 From: Derek Schmidt Date: Tue, 7 Jan 2025 23:56:47 -0500 Subject: [PATCH 09/11] Cleanup the cleanups --- src/ovtk_audiencekit/core/Audio.py | 1 + .../plugins/AudioAlert/AudioAlert.py | 13 ++++++++++++- src/ovtk_audiencekit/plugins/builtins/Cue.py | 6 ++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/ovtk_audiencekit/core/Audio.py b/src/ovtk_audiencekit/core/Audio.py index 53e7d40..e82c522 100644 --- a/src/ovtk_audiencekit/core/Audio.py +++ b/src/ovtk_audiencekit/core/Audio.py @@ -100,6 +100,7 @@ class Stream: self._stream.stop_stream() def close(self): + self._stream.stop_stream() self._stream.close() def _read_callback(self, in_data, frame_count, time_info, status): diff --git a/src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py b/src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py index f694404..5726803 100644 --- a/src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py +++ b/src/ovtk_audiencekit/plugins/AudioAlert/AudioAlert.py @@ -12,6 +12,7 @@ class AudioAlert(PluginBase): self.timeout_min = timeout_min self.clips = {} self.streams = {} + self.tasks = set() self.buffer_length = int(buffer_length) self.output_index = Stream.find_output_index(output) if sample_rate is None: @@ -51,10 +52,20 @@ class AudioAlert(PluginBase): if immediate: - asyncio.create_task(stream.aplay()) + task = asyncio.create_task(stream.aplay()) + task.add_done_callback(self.tasks.remove) + self.tasks.add(task) else: stream.play() + def close(self): + self._cleanup_task.cancel() + for task in self.tasks: + task.cancel() + for stream_dq in self.streams.values(): + for stream in stream_dq: + stream.close() + async def _cleanup(self): while True: await asyncio.sleep(60) diff --git a/src/ovtk_audiencekit/plugins/builtins/Cue.py b/src/ovtk_audiencekit/plugins/builtins/Cue.py index 2d7c701..6203814 100644 --- a/src/ovtk_audiencekit/plugins/builtins/Cue.py +++ b/src/ovtk_audiencekit/plugins/builtins/Cue.py @@ -111,6 +111,12 @@ class CuePlugin(PluginBase): except ValueError as e: self.logger.error(f'Cannot schedule cue {name} at {at}: {e}') + def close(self): + self._cleanup_task.cancel() + for task in self.tasks.values(): + self.scheduler.cancel(task) + self.scheduler._task.cancel() + async def _cleanup(self): while True: await asyncio.sleep(60) -- 2.48.1 From e89a0192f584bc9fe2a1e59eb87127080c969c3c Mon Sep 17 00:00:00 2001 From: Derek Schmidt Date: Sat, 11 Jan 2025 23:39:59 -0500 Subject: [PATCH 10/11] [bultins/chance] allow an 'or' child that runs on rand miss --- src/ovtk_audiencekit/plugins/builtins/Chance.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ovtk_audiencekit/plugins/builtins/Chance.py b/src/ovtk_audiencekit/plugins/builtins/Chance.py index a733b9f..5553fee 100644 --- a/src/ovtk_audiencekit/plugins/builtins/Chance.py +++ b/src/ovtk_audiencekit/plugins/builtins/Chance.py @@ -16,4 +16,7 @@ class ChancePlugin(PluginBase): raise ValueError('Chance must be a string (optionally ending in %) or number') if random.random() < chance / 100: - await self.execute_kdl(_children, _ctx=_ctx) + await self.execute_kdl([child for child in _children if child.name != 'or'], _ctx=_ctx) + else: + if elsenode := next((child for child in _children if child.name == 'or'), None): + await self.execute_kdl(elsenode.nodes, _ctx=_ctx) -- 2.48.1 From 23dda2fe902e898810e2d9f2fceb1f0fc4a2e468 Mon Sep 17 00:00:00 2001 From: Derek Date: Thu, 16 Jan 2025 23:59:52 -0500 Subject: [PATCH 11/11] [builtins/cue] Fix non-repeating cues never firing Oop --- src/ovtk_audiencekit/plugins/builtins/Cue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ovtk_audiencekit/plugins/builtins/Cue.py b/src/ovtk_audiencekit/plugins/builtins/Cue.py index 6203814..75bfe01 100644 --- a/src/ovtk_audiencekit/plugins/builtins/Cue.py +++ b/src/ovtk_audiencekit/plugins/builtins/Cue.py @@ -31,7 +31,7 @@ class Cue: def is_obsolete(self): if self.repeat: return False - if self._next <= maya.now(): + if self._next >= maya.now(): return False return True -- 2.48.1