From 44ba7d62736acafa792fb971251e0c70b4ac79c0 Mon Sep 17 00:00:00 2001 From: Derek Schmidt Date: Fri, 9 Apr 2021 21:25:48 -0700 Subject: [PATCH 1/3] Support proper configuration for chats! --- Pipfile | 1 + Pipfile.lock | 10 ++++++- chats/FakeChat.py | 17 +++++------- chats/Twitch.py | 16 ++++++----- chats/YoutubeLive.py | 29 ++++++++++++------- config.toml | 26 +++++++++++++++++ main.py | 66 +++++++++++++++++++++++++++----------------- 7 files changed, 111 insertions(+), 54 deletions(-) create mode 100644 config.toml diff --git a/Pipfile b/Pipfile index fdbb6de..f81d94f 100644 --- a/Pipfile +++ b/Pipfile @@ -8,6 +8,7 @@ requests = "*" pygame = "*" websocket-client = "*" miniirc = "*" +toml = "*" [requires] python_version = "3.9" diff --git a/Pipfile.lock b/Pipfile.lock index 9446338..82ddf2f 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "c6f4727f4fbf44c5155fd9276c2a575224bd4ce1914e7051b84c01dea7168336" + "sha256": "78eab561bb582e47361961316e00224a4e606589ca122fc14f1d0c527c1c5dbe" }, "pipfile-spec": 6, "requires": { @@ -110,6 +110,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.15.0" }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "index": "pypi", + "version": "==0.10.2" + }, "urllib3": { "hashes": [ "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df", diff --git a/chats/FakeChat.py b/chats/FakeChat.py index a692b2d..e5f103a 100644 --- a/chats/FakeChat.py +++ b/chats/FakeChat.py @@ -6,8 +6,6 @@ from .ChatProcess import ChatProcess class CONTROL_MESSAGES(Enum): - CHANGE_DELAY = auto() - CHANGE_CHUNK = auto() START_STOP = auto() @@ -16,13 +14,14 @@ class STATES(Enum): RUNNING = auto() -class FakeChat(ChatProcess): +class Process(ChatProcess): CHAT_NAME = 'Fake Chat' - def __init__(self, *args): + def __init__(self, *args, max_delay=10, max_messages_per_chunk=1, start_paused=True): super().__init__(*args) - self._max_messages_per_chunk = 1 - self._max_delay = 10 + self._max_delay = max_delay + self._max_messages_per_chunk = max_messages_per_chunk + self.state = STATES.PAUSED if start_paused else STATES.RUNNING @property def keybinds(self): @@ -35,15 +34,13 @@ class FakeChat(ChatProcess): running = not self.state == STATES.RUNNING text = 'Fake chat activated!' if running else 'Disabled fake chat' - sys_msg = Message(text, FakeChat.CHAT_NAME, self.__class__.__name__, AUTHOR_TYPES.SYSTEM) + sys_msg = Message(text, Process.CHAT_NAME, self.__class__.__name__, AUTHOR_TYPES.SYSTEM) self._message_queue.put(sys_msg) return STATES.RUNNING if running else STATES.PAUSED def loop(self, next_state): - if self.state is None: - return STATES.PAUSED - elif self.state == STATES.PAUSED: + if self.state == STATES.PAUSED: return None while range(int(random.random() * (self._max_messages_per_chunk + 1))): diff --git a/chats/Twitch.py b/chats/Twitch.py index d6a1e13..73f6ec4 100644 --- a/chats/Twitch.py +++ b/chats/Twitch.py @@ -49,15 +49,17 @@ class NonBlockingWebsocket(Process): return -1 -class TwitchIRCProcess(ChatProcess): +class Process(ChatProcess): CHAT_NAME = 'Twitch IRC' WEBSOCKET_ADDRESS = 'wss://irc-ws.chat.twitch.tv:443' - def __init__(self, username, token, *args): + def __init__(self, *args, channel_name=None, oauth_key=None): + if channel_name is None or oauth_key is None: + raise ValueError('Twitch chat is missing config requirements') super().__init__(*args) self._state_machine = self.bind_to_states(STATES) - self._username = username - self._token = token + self._username = channel_name + self._token = oauth_key self.state = STATES.DISCONNECTED @@ -78,12 +80,12 @@ class TwitchIRCProcess(ChatProcess): message = status_messages.get(new_state) if message is not None: - sys_msg = Message(message, TwitchIRCProcess.CHAT_NAME, self.__class__.__name__, AUTHOR_TYPES.SYSTEM) + sys_msg = Message(message, Process.CHAT_NAME, self.__class__.__name__, AUTHOR_TYPES.SYSTEM) self._message_queue.put(sys_msg) def on_disconnected(self, next_state): self._ws = NonBlockingWebsocket() - self._ws.connect(TwitchIRCProcess.WEBSOCKET_ADDRESS) + self._ws.connect(Process.WEBSOCKET_ADDRESS) self._ws.start() self._ws.send(f'PASS {self._token}') self._ws.send(f'NICK {self._username}') @@ -103,7 +105,7 @@ class TwitchIRCProcess(ChatProcess): for message in messages.splitlines(): cmd, hostmask, tags, args = ircv3_message_parser(message) if cmd == 'PRIVMSG': - normalized_message = TwitchIRCProcess.normalize_message(hostmask, tags, args) + normalized_message = Process.normalize_message(hostmask, tags, args) self._message_queue.put(normalized_message) elif cmd == 'PING': self._ws.send(f"PONG {' '.join(args)}") diff --git a/chats/YoutubeLive.py b/chats/YoutubeLive.py index 27dc47d..5b89201 100644 --- a/chats/YoutubeLive.py +++ b/chats/YoutubeLive.py @@ -1,4 +1,5 @@ import os +import json import webbrowser from enum import Enum, auto @@ -23,12 +24,20 @@ class CONTROL_MESSAGES(Enum): RESTART = auto() -class YoutubeLiveProcess(ChatProcess): +class Process(ChatProcess): CHAT_NAME = 'YouTube Live' GOOGLE_OAUTH_TOKEN_URL = 'https://accounts.google.com/o/oauth2/token' YOUTUBE_API_URL = 'https://www.googleapis.com/youtube/v3' - def __init__(self, client_secrets, *args): + def __init__(self, *args, client_secrets_path=None): + if client_secrets_path is None or not os.path.exists(client_secrets_path): + raise ValueError('Missing client secrets') + + with open(client_secrets_path, 'r') as f: + client_secrets = json.load(f).get('installed') + if client_secrets is None: + raise ValueError('Malformed client secrets file - missing installed section') + super().__init__(*args) self._client_secrets = client_secrets self._state_machine = self.bind_to_states(STATES) @@ -71,7 +80,7 @@ class YoutubeLiveProcess(ChatProcess): return next_state def on_waiting_for_broadcast(self, next_state): - response = requests.get(f'{YoutubeLiveProcess.YOUTUBE_API_URL}/liveBroadcasts', + response = requests.get(f'{Process.YOUTUBE_API_URL}/liveBroadcasts', params={'part': 'snippet', 'broadcastStatus': 'active'}, headers={'Authorization': f'Bearer {self._access_token}'}) @@ -88,7 +97,7 @@ class YoutubeLiveProcess(ChatProcess): return 30 def on_polling(self, next_state): - response = requests.get(f'{YoutubeLiveProcess.YOUTUBE_API_URL}/liveChat/messages', + response = requests.get(f'{Process.YOUTUBE_API_URL}/liveChat/messages', params={'liveChatId': self._live_chat_id, 'part': 'snippet,authorDetails', 'hl': 'en_US', @@ -104,7 +113,7 @@ class YoutubeLiveProcess(ChatProcess): self._page_token = data['nextPageToken'] if len(data['items']): for message in data['items']: - normalized = YoutubeLiveProcess.normalize_message(message) + normalized = Process.normalize_message(message) self._message_queue.put(normalized) sleep_milis = max(data['pollingIntervalMillis'], 5000) @@ -123,7 +132,7 @@ class YoutubeLiveProcess(ChatProcess): message = status_messages.get(new_state) if message is not None: - sys_msg = Message(message, YoutubeLiveProcess.CHAT_NAME, self.__class__.__name__, AUTHOR_TYPES.SYSTEM) + sys_msg = Message(message, Process.CHAT_NAME, self.__class__.__name__, AUTHOR_TYPES.SYSTEM) self._message_queue.put(sys_msg) def process_messages(self, message_type, args, next_state): @@ -133,10 +142,10 @@ class YoutubeLiveProcess(ChatProcess): elif message_type == CONTROL_MESSAGES.RESTART: if self.state == STATES.POLLING: self._message_queue.put(Message('Restart requested, but service is in healthy state! Ignoring...', - YoutubeLiveProcess.CHAT_NAME, self.__class__.__name__, AUTHOR_TYPES.SYSTEM)) + Process.CHAT_NAME, self.__class__.__name__, AUTHOR_TYPES.SYSTEM)) else: self._message_queue.put(Message('Restarting service...', - YoutubeLiveProcess.CHAT_NAME, self.__class__.__name__, AUTHOR_TYPES.SYSTEM)) + Process.CHAT_NAME, self.__class__.__name__, AUTHOR_TYPES.SYSTEM)) return None, None @classmethod @@ -167,7 +176,7 @@ class YoutubeLiveProcess(ChatProcess): webbrowser.open(url) def setup_oauth_consent(self, consent_code): - response = requests.post(YoutubeLiveProcess.GOOGLE_OAUTH_TOKEN_URL, data={ + response = requests.post(Process.GOOGLE_OAUTH_TOKEN_URL, data={ 'code': consent_code, 'client_id': self._client_secrets['client_id'], 'client_secret': self._client_secrets['client_secret'], @@ -182,7 +191,7 @@ class YoutubeLiveProcess(ChatProcess): self._refresh_token = auth['refresh_token'] def get_fresh_access_token(self): - response = requests.post(YoutubeLiveProcess.GOOGLE_OAUTH_TOKEN_URL, data={ + response = requests.post(Process.GOOGLE_OAUTH_TOKEN_URL, data={ 'client_id': self._client_secrets['client_id'], 'client_secret': self._client_secrets['client_secret'], 'refresh_token': self._refresh_token, diff --git a/config.toml b/config.toml new file mode 100644 index 0000000..426a003 --- /dev/null +++ b/config.toml @@ -0,0 +1,26 @@ +# How quickly to check for commands in the main event loop +# You shouldn't need to change this :) +event_poll_frequency = 0.1 + +[chat.FakeChat] +enabled = true +max_messages_per_chunk = 3 +max_delay = 15 +start_paused = false + +[chat.YoutubeLive] +enabled = false +# # Where you put the yotube client secrets file +# # See https://seo-michael.co.uk/how-to-create-your-own-youtube-api-key-id-and-secret +# # TODO: The above link isnt specific for this task - maybe make your own and link back to repo docs? +# client_secrets_path = "googleapi.secret.json" + +[chat.Twitch] +enabled = false +# # Your twitch username / name of your channel +# # REVIEW: Can these be different?? +# channel_name = "cool_streamer" +# # The oauth key for your account, including the "oauth:" prefix +# # You can get this from https://twitchapps.com/tmi/ +# oauth_key = "oauth:somethinglikethis" + diff --git a/main.py b/main.py index 259b0f4..9bf6313 100644 --- a/main.py +++ b/main.py @@ -1,50 +1,63 @@ -import json -import os import time +import importlib + +import toml from chats import YoutubeLive from chats.ChatProcess import LIFECYCLE_MESSAGES -from chats.FakeChat import FakeChat -from chats.Twitch import TwitchIRCProcess from ui.App import App -from plugins.PogCount import PogCounter - -EVENT_POLL_FREQ = 0.1 -GOOGLE_API_SECRETS_PATH = 'googleapi.secret.json' -TWITCH_SECRETS_PATH = 'twitch.secret.json' if __name__ == '__main__': - chat_processes = [FakeChat(EVENT_POLL_FREQ)] - if os.path.exists(GOOGLE_API_SECRETS_PATH): - with open(GOOGLE_API_SECRETS_PATH, 'r') as f: - client_secrets = json.load(f)['installed'] - chat_processes.append(YoutubeLive.YoutubeLiveProcess(client_secrets, EVENT_POLL_FREQ)) - else: - print('No client secrets - disabling youtube chat client') - if os.path.exists(TWITCH_SECRETS_PATH): - with open(TWITCH_SECRETS_PATH ,'r') as f: - secrets = json.load(f) - chat_processes.append(TwitchIRCProcess(secrets.get('username'), secrets.get('oauth'), EVENT_POLL_FREQ)) - else: - print('No twitch config - disabling twitch chat client') + with open('config.toml', 'r') as f: + config = toml.load(f) - if len(chat_processes) == 1: - print('Hit "f" with the window focused to enable a testing client') + EVENT_POLL_FREQUENCY = config['event_poll_frequency'] - plugins = [PogCounter('../live-status.txt', prefix='Pog count: ')] + # Load dynamic compoents + chat_processes = [] + for chat_name, chat_config in config.get('chat', {}).items(): + if not chat_config['enabled']: + continue + del chat_config['enabled'] + chat_module = importlib.import_module(f'.{chat_name}', package='chats') + try: + chat_process = chat_module.Process(EVENT_POLL_FREQUENCY, **chat_config) + chat_processes.append(chat_process) + except Exception as e: + print(f'Failed to initalize {chat_name} - {e}') + + if len(chat_processes) == 0: + print('No chats configured!') + + plugins = [] + for plugin_name, plugin_config in config.get('plugin', {}).items(): + if not plugin_config['enabled']: + continue + del plugin_config['enabled'] + + plugin_module = importlib.import_module(f'.{plugin_name}', package='plugins') + try: + plugin = plugin_module.Plugin(**plugin_config) + plugins.append(plugin) + except Exception as e: + print(f'Failed to initalize {plugin_name} - {e}') + + # Start the app and subprocesses app = App(chat_processes, plugins) for process in chat_processes: process.start() app.start() + # Event loop while True: if not app.is_alive(): break for process in chat_processes: + # HACK: need to find a generic way to do this if process.control_pipe.poll(): chat_control_msg = process.control_pipe.recv() if chat_control_msg == YoutubeLive.CONTROL_MESSAGES.NEEDS_OAUTH_CONSENT_TOKEN: @@ -53,8 +66,9 @@ if __name__ == '__main__': 'type': YoutubeLive.CONTROL_MESSAGES.OAUTH_CONSENT_TOKEN, 'token': code }) - time.sleep(EVENT_POLL_FREQ) + time.sleep(EVENT_POLL_FREQUENCY) + # Cleanup for process in chat_processes: process.control_pipe.send({'type': LIFECYCLE_MESSAGES.SHUTDOWN}) process.join(10) -- 2.45.1 From b1bf2fdbde7deb782be37ce8e1faf87a1c5247d6 Mon Sep 17 00:00:00 2001 From: Derek Schmidt Date: Fri, 9 Apr 2021 21:28:51 -0700 Subject: [PATCH 2/3] Add config support for plugins as well, generalize pogcounter Now a configurable phrase counter! --- Pipfile | 1 + Pipfile.lock | 18 ++++++++- config.toml | 28 ++++++++++++++ plugins/PhraseCounter.py | 84 ++++++++++++++++++++++++++++++++++++++++ plugins/PluginBase.py | 18 +++++++++ plugins/__init__.py | 1 - 6 files changed, 147 insertions(+), 3 deletions(-) create mode 100644 plugins/PhraseCounter.py create mode 100644 plugins/PluginBase.py diff --git a/Pipfile b/Pipfile index f81d94f..2d3d3c1 100644 --- a/Pipfile +++ b/Pipfile @@ -9,6 +9,7 @@ pygame = "*" websocket-client = "*" miniirc = "*" toml = "*" +num2words = "*" [requires] python_version = "3.9" diff --git a/Pipfile.lock b/Pipfile.lock index 82ddf2f..d466432 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "78eab561bb582e47361961316e00224a4e606589ca122fc14f1d0c527c1c5dbe" + "sha256": "b0076c1e84753068b73b209da13dce37cef7680589c6b54b14204240b893797b" }, "pipfile-spec": 6, "requires": { @@ -31,6 +31,12 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==4.0.0" }, + "docopt": { + "hashes": [ + "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" + ], + "version": "==0.6.2" + }, "idna": { "hashes": [ "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", @@ -47,6 +53,14 @@ "index": "pypi", "version": "==1.6.3" }, + "num2words": { + "hashes": [ + "sha256:0b6e5f53f11d3005787e206d9c03382f459ef048a43c544e3db3b1e05a961548", + "sha256:37cd4f60678f7e1045cdc3adf6acf93c8b41bf732da860f97d301f04e611cc57" + ], + "index": "pypi", + "version": "==0.5.10" + }, "pygame": { "hashes": [ "sha256:0571dde0277483f5060c8ee43cbfd8df5776b12505e3948eee241c8ce9b93371", @@ -107,7 +121,7 @@ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==1.15.0" }, "toml": { diff --git a/config.toml b/config.toml index 426a003..43fda1a 100644 --- a/config.toml +++ b/config.toml @@ -24,3 +24,31 @@ enabled = false # # You can get this from https://twitchapps.com/tmi/ # oauth_key = "oauth:somethinglikethis" +[plugin.PhraseCounter] +# # Dumps a file with the current count of mesasges containting a phrase, to be read with OBS or similar +enabled = false +# # The phrases to detect +# phrases = ["pog"] +# # Relative path to dump to +# out_path="pogcount.txt" +# # If the plugin should remove duplicate letters. For example, if a phrase is "pog", with this we would count "pooooggg" +# detect_extentions = false +# # Count regardless of the position of spaces +# detect_spaced = false +# # Min number of seconds to wait inbetween file updates +# debounce_time = 1 +# # Template of the text to dump, based on the standard python string format. +# # + {phrase} - the count as a number, ex "42" +# # + {phrase!s} - the count as a string, ex "fourty-two" +# # + {phrase!p:singular|plural} - chooses between the two strings depending on if the count is singular or plural in english +# # ex, `There's {pog} {pog!p:pog|pogs} in chat` would produce "There's 0 pogs in chat" and "There's 1 pog in chat" +# # A few special phrases are supported: +# # + "sum" - sum of all phrase counts +# # + "match" - number of messages containing any one of the phrases +# # + "nmatch" - number of messages containing NONE of the phrases +# # To make this multiline, use three quote marks and a newline, ie +# # template = """ +# # multiline content +# # goes here! +# # """ +# template="Pog count: {pog}" diff --git a/plugins/PhraseCounter.py b/plugins/PhraseCounter.py new file mode 100644 index 0000000..7421132 --- /dev/null +++ b/plugins/PhraseCounter.py @@ -0,0 +1,84 @@ +from .PluginBase import PluginBase +from string import Formatter + +from num2words import num2words + + +def remove_dups(text): + last_char = None + for char in list(text): + if last_char == char: + continue + last_char = char + yield char + + +class PhraseCounterFormatter(Formatter): + def convert_field(self, value, conversion): + if conversion == 's': + return num2words(value) + if conversion == 'p': + return value == 1 + else: + return super().convert_field(value, conversion) + + def format_field(self, value, format_spec): + if '|' in format_spec: + singular, plural = format_spec.split('|') + return singular if value else plural + else: + return super().format_field(value, format_spec) + + +class Plugin(PluginBase): + def __init__(self, out_path=None, phrases=None, template=None, + detect_spaced=False, detect_extentions=False, debounce_time=1): + if out_path is None or phrases is None or template is None: + raise ValueError('Missing config requirements') + self.out = out_path + self.phrases = phrases + self.template = template + self.detect_extentions = detect_extentions + self.detect_spaced = detect_spaced + self.debounce_time = debounce_time + + self._counts = {phrase: 0 for phrase in phrases + ['sum', 'match', 'nmatch']} + self._timer = 0 + self._dirty = True + self._formatter = PhraseCounterFormatter() + + def normalize(self, text): + if self.detect_extentions: + text = ''.join(remove_dups(text.lower())) + if self.detect_spaced: + text = ''.join(text.split(' ')) + return text + + def tick(self, dt): + self._timer += dt + if self._timer > self.debounce_time and self._dirty: + out_string = self._formatter.format(self.template, **self._counts) + with open(self.out, 'w') as f: + f.write(out_string) + self._timer = 0 + self._dirty = False + + def handle_event(self, event): + pass + + def on_message(self, message): + self._dirty = True + normalized = self.normalize(message.text) + + has_match = False + for phrase in self.phrases: + if phrase in normalized: + self._counts[phrase] += 1 + self._counts['sum'] += 1 + has_match = True + if has_match: + self._counts['match'] += 1 + else: + self._counts['nmatch'] += 1 + + return message diff --git a/plugins/PluginBase.py b/plugins/PluginBase.py new file mode 100644 index 0000000..2a333a9 --- /dev/null +++ b/plugins/PluginBase.py @@ -0,0 +1,18 @@ +from abc import ABC, abstractmethod + + +class PluginBase(ABC): + def __init__(self): + super().__init__() + + @abstractmethod + def tick(self, dt): + pass + + @abstractmethod + def handle_event(self, event): + pass + + @abstractmethod + def on_message(self, message): + pass diff --git a/plugins/__init__.py b/plugins/__init__.py index 64ec7ab..e69de29 100644 --- a/plugins/__init__.py +++ b/plugins/__init__.py @@ -1 +0,0 @@ -from chats import AUTHOR_TYPES -- 2.45.1 From f85d1f72e1d6e7301f8324ed23655dd2850828e8 Mon Sep 17 00:00:00 2001 From: Derek Schmidt Date: Fri, 9 Apr 2021 21:31:56 -0700 Subject: [PATCH 3/3] Minor config comment cleanup --- config.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config.toml b/config.toml index 43fda1a..38e729c 100644 --- a/config.toml +++ b/config.toml @@ -3,6 +3,7 @@ event_poll_frequency = 0.1 [chat.FakeChat] +# A fake chat for use when testing enabled = true max_messages_per_chunk = 3 max_delay = 15 @@ -25,7 +26,7 @@ enabled = false # oauth_key = "oauth:somethinglikethis" [plugin.PhraseCounter] -# # Dumps a file with the current count of mesasges containting a phrase, to be read with OBS or similar +# Dumps a file with the current count of mesasges containting a phrase, to be read with OBS or similar enabled = false # # The phrases to detect # phrases = ["pog"] -- 2.45.1