Merge pull request 'Add TOML configuration' (#7) from feature/config into main

Reviewed-on: #7

Pipfile

@@ -8,6 +8,8 @@ requests = "*"
 pygame = "*"
 websocket-client = "*"
 miniirc = "*"
+toml = "*"
+num2words = "*"
 
 [requires]
 python_version = "3.9"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "c6f4727f4fbf44c5155fd9276c2a575224bd4ce1914e7051b84c01dea7168336"
+            "sha256": "b0076c1e84753068b73b209da13dce37cef7680589c6b54b14204240b893797b"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -31,6 +31,12 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==4.0.0"
         },
+        "docopt": {
+            "hashes": [
+                "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"
+            ],
+            "version": "==0.6.2"
+        },
         "idna": {
             "hashes": [
                 "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
@@ -47,6 +53,14 @@
             "index": "pypi",
             "version": "==1.6.3"
         },
+        "num2words": {
+            "hashes": [
+                "sha256:0b6e5f53f11d3005787e206d9c03382f459ef048a43c544e3db3b1e05a961548",
+                "sha256:37cd4f60678f7e1045cdc3adf6acf93c8b41bf732da860f97d301f04e611cc57"
+            ],
+            "index": "pypi",
+            "version": "==0.5.10"
+        },
         "pygame": {
             "hashes": [
                 "sha256:0571dde0277483f5060c8ee43cbfd8df5776b12505e3948eee241c8ce9b93371",
@@ -107,9 +121,17 @@
                 "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
                 "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
             "version": "==1.15.0"
         },
+        "toml": {
+            "hashes": [
+                "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
+                "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
+            ],
+            "index": "pypi",
+            "version": "==0.10.2"
+        },
         "urllib3": {
             "hashes": [
                 "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df",

chats/FakeChat.py

@@ -6,8 +6,6 @@ from .ChatProcess import ChatProcess
 
 
 class CONTROL_MESSAGES(Enum):
-    CHANGE_DELAY = auto()
-    CHANGE_CHUNK = auto()
     START_STOP = auto()
 
 
@@ -16,13 +14,14 @@ class STATES(Enum):
     RUNNING = auto()
 
 
-class FakeChat(ChatProcess):
+class Process(ChatProcess):
     CHAT_NAME = 'Fake Chat'
 
-    def __init__(self, *args):
+    def __init__(self, *args, max_delay=10, max_messages_per_chunk=1, start_paused=True):
         super().__init__(*args)
-        self._max_messages_per_chunk = 1
+        self._max_delay = max_delay
-        self._max_delay = 10
+        self._max_messages_per_chunk = max_messages_per_chunk
+        self.state = STATES.PAUSED if start_paused else STATES.RUNNING
 
     @property
     def keybinds(self):
@@ -35,15 +34,13 @@ class FakeChat(ChatProcess):
             running = not self.state == STATES.RUNNING
             text = 'Fake chat activated!' if running else 'Disabled fake chat'
 
-            sys_msg = Message(text, FakeChat.CHAT_NAME, self.__class__.__name__, AUTHOR_TYPES.SYSTEM)
+            sys_msg = Message(text, Process.CHAT_NAME, self.__class__.__name__, AUTHOR_TYPES.SYSTEM)
             self._message_queue.put(sys_msg)
 
             return STATES.RUNNING if running else STATES.PAUSED
 
     def loop(self, next_state):
-        if self.state is None:
+        if self.state == STATES.PAUSED:
-            return STATES.PAUSED
-        elif self.state == STATES.PAUSED:
             return None
 
         while range(int(random.random() * (self._max_messages_per_chunk + 1))):

chats/Twitch.py

@@ -49,15 +49,17 @@ class NonBlockingWebsocket(Process):
             return -1
 
 
-class TwitchIRCProcess(ChatProcess):
+class Process(ChatProcess):
     CHAT_NAME = 'Twitch IRC'
     WEBSOCKET_ADDRESS = 'wss://irc-ws.chat.twitch.tv:443'
 
-    def __init__(self, username, token, *args):
+    def __init__(self, *args, channel_name=None, oauth_key=None):
+        if channel_name is None or oauth_key is None:
+            raise ValueError('Twitch chat is missing config requirements')
         super().__init__(*args)
         self._state_machine = self.bind_to_states(STATES)
-        self._username = username
+        self._username = channel_name
-        self._token = token
+        self._token = oauth_key
 
         self.state = STATES.DISCONNECTED
 
@@ -78,12 +80,12 @@ class TwitchIRCProcess(ChatProcess):
 
         message = status_messages.get(new_state)
         if message is not None:
-            sys_msg = Message(message, TwitchIRCProcess.CHAT_NAME, self.__class__.__name__, AUTHOR_TYPES.SYSTEM)
+            sys_msg = Message(message, Process.CHAT_NAME, self.__class__.__name__, AUTHOR_TYPES.SYSTEM)
             self._message_queue.put(sys_msg)
 
     def on_disconnected(self, next_state):
         self._ws = NonBlockingWebsocket()
-        self._ws.connect(TwitchIRCProcess.WEBSOCKET_ADDRESS)
+        self._ws.connect(Process.WEBSOCKET_ADDRESS)
         self._ws.start()
         self._ws.send(f'PASS {self._token}')
         self._ws.send(f'NICK {self._username}')
@@ -103,7 +105,7 @@ class TwitchIRCProcess(ChatProcess):
             for message in messages.splitlines():
                 cmd, hostmask, tags, args = ircv3_message_parser(message)
                 if cmd == 'PRIVMSG':
-                    normalized_message = TwitchIRCProcess.normalize_message(hostmask, tags, args)
+                    normalized_message = Process.normalize_message(hostmask, tags, args)
                     self._message_queue.put(normalized_message)
                 elif cmd == 'PING':
                     self._ws.send(f"PONG {' '.join(args)}")

chats/YoutubeLive.py

@@ -1,4 +1,5 @@
 import os
+import json
 import webbrowser
 from enum import Enum, auto
 
@@ -23,12 +24,20 @@ class CONTROL_MESSAGES(Enum):
     RESTART = auto()
 
 
-class YoutubeLiveProcess(ChatProcess):
+class Process(ChatProcess):
     CHAT_NAME = 'YouTube Live'
     GOOGLE_OAUTH_TOKEN_URL = 'https://accounts.google.com/o/oauth2/token'
     YOUTUBE_API_URL = 'https://www.googleapis.com/youtube/v3'
 
-    def __init__(self, client_secrets, *args):
+    def __init__(self, *args, client_secrets_path=None):
+        if client_secrets_path is None or not os.path.exists(client_secrets_path):
+            raise ValueError('Missing client secrets')
+
+        with open(client_secrets_path, 'r') as f:
+            client_secrets = json.load(f).get('installed')
+            if client_secrets is None:
+                raise ValueError('Malformed client secrets file - missing installed section')
+
         super().__init__(*args)
         self._client_secrets = client_secrets
         self._state_machine = self.bind_to_states(STATES)
@@ -71,7 +80,7 @@ class YoutubeLiveProcess(ChatProcess):
         return next_state
 
     def on_waiting_for_broadcast(self, next_state):
-        response = requests.get(f'{YoutubeLiveProcess.YOUTUBE_API_URL}/liveBroadcasts',
+        response = requests.get(f'{Process.YOUTUBE_API_URL}/liveBroadcasts',
                                 params={'part': 'snippet', 'broadcastStatus': 'active'},
                                 headers={'Authorization': f'Bearer {self._access_token}'})
 
@@ -88,7 +97,7 @@ class YoutubeLiveProcess(ChatProcess):
         return 30
 
     def on_polling(self, next_state):
-        response = requests.get(f'{YoutubeLiveProcess.YOUTUBE_API_URL}/liveChat/messages',
+        response = requests.get(f'{Process.YOUTUBE_API_URL}/liveChat/messages',
                                 params={'liveChatId': self._live_chat_id,
                                         'part': 'snippet,authorDetails',
                                         'hl': 'en_US',
@@ -104,7 +113,7 @@ class YoutubeLiveProcess(ChatProcess):
             self._page_token = data['nextPageToken']
             if len(data['items']):
                 for message in data['items']:
-                    normalized = YoutubeLiveProcess.normalize_message(message)
+                    normalized = Process.normalize_message(message)
                     self._message_queue.put(normalized)
 
             sleep_milis = max(data['pollingIntervalMillis'], 5000)
@@ -123,7 +132,7 @@ class YoutubeLiveProcess(ChatProcess):
 
         message = status_messages.get(new_state)
         if message is not None:
-            sys_msg = Message(message, YoutubeLiveProcess.CHAT_NAME, self.__class__.__name__, AUTHOR_TYPES.SYSTEM)
+            sys_msg = Message(message, Process.CHAT_NAME, self.__class__.__name__, AUTHOR_TYPES.SYSTEM)
             self._message_queue.put(sys_msg)
 
     def process_messages(self, message_type, args, next_state):
@@ -133,10 +142,10 @@ class YoutubeLiveProcess(ChatProcess):
         elif message_type == CONTROL_MESSAGES.RESTART:
             if self.state == STATES.POLLING:
                 self._message_queue.put(Message('Restart requested, but service is in healthy state! Ignoring...',
-                                                YoutubeLiveProcess.CHAT_NAME, self.__class__.__name__, AUTHOR_TYPES.SYSTEM))
+                                                Process.CHAT_NAME, self.__class__.__name__, AUTHOR_TYPES.SYSTEM))
             else:
                 self._message_queue.put(Message('Restarting service...',
-                                                YoutubeLiveProcess.CHAT_NAME, self.__class__.__name__, AUTHOR_TYPES.SYSTEM))
+                                                Process.CHAT_NAME, self.__class__.__name__, AUTHOR_TYPES.SYSTEM))
                 return None, None
 
     @classmethod
@@ -167,7 +176,7 @@ class YoutubeLiveProcess(ChatProcess):
         webbrowser.open(url)
 
     def setup_oauth_consent(self, consent_code):
-        response = requests.post(YoutubeLiveProcess.GOOGLE_OAUTH_TOKEN_URL, data={
+        response = requests.post(Process.GOOGLE_OAUTH_TOKEN_URL, data={
             'code': consent_code,
             'client_id': self._client_secrets['client_id'],
             'client_secret': self._client_secrets['client_secret'],
@@ -182,7 +191,7 @@ class YoutubeLiveProcess(ChatProcess):
         self._refresh_token = auth['refresh_token']
 
     def get_fresh_access_token(self):
-        response = requests.post(YoutubeLiveProcess.GOOGLE_OAUTH_TOKEN_URL, data={
+        response = requests.post(Process.GOOGLE_OAUTH_TOKEN_URL, data={
             'client_id': self._client_secrets['client_id'],
             'client_secret': self._client_secrets['client_secret'],
             'refresh_token': self._refresh_token,

config.toml

@@ -0,0 +1,55 @@
+# How quickly to check for commands in the main event loop
+# You shouldn't need to change this :)
+event_poll_frequency = 0.1
+
+[chat.FakeChat]
+# A fake chat for use when testing
+enabled = true
+max_messages_per_chunk = 3
+max_delay = 15
+start_paused = false
+
+[chat.YoutubeLive]
+enabled = false
+# # Where you put the yotube client secrets file
+# # See https://seo-michael.co.uk/how-to-create-your-own-youtube-api-key-id-and-secret
+# # TODO: The above link isnt specific for this task - maybe make your own and link back to repo docs?
+# client_secrets_path = "googleapi.secret.json"
+
+[chat.Twitch]
+enabled = false
+# # Your twitch username / name of your channel
+# # REVIEW: Can these be different??
+# channel_name = "cool_streamer"
+# # The oauth key for your account, including the "oauth:" prefix
+# # You can get this from https://twitchapps.com/tmi/
+# oauth_key = "oauth:somethinglikethis"
+
+[plugin.PhraseCounter]
+# Dumps a file with the current count of mesasges containting a phrase, to be read with OBS or similar
+enabled = false
+# # The phrases to detect
+# phrases = ["pog"]
+# # Relative path to dump to
+# out_path="pogcount.txt"
+# # If the plugin should remove duplicate letters. For example, if a phrase is "pog", with this we would count "pooooggg"
+# detect_extentions = false
+# # Count regardless of the position of spaces
+# detect_spaced = false
+# # Min number of seconds to wait inbetween file updates
+# debounce_time = 1
+# # Template of the text to dump, based on the standard python string format.
+# # + {phrase} - the count as a number, ex "42"
+# # + {phrase!s} - the count as a string, ex "fourty-two"
+# # + {phrase!p:singular|plural} - chooses between the two strings depending on if the count is singular or plural in english
+# #                                ex, `There's {pog} {pog!p:pog|pogs} in chat` would produce "There's 0 pogs in chat" and "There's 1 pog in chat"
+# # A few special phrases are supported:
+# # + "sum" - sum of all phrase counts
+# # + "match" - number of messages containing any one of the phrases
+# # + "nmatch" - number of messages containing NONE of the phrases
+# # To make this multiline, use three quote marks and a newline, ie
+# # template = """
+# # multiline content
+# # goes here!
+# # """
+# template="Pog count: {pog}"

main.py

@@ -1,50 +1,63 @@
-import json
-import os
 import time
+import importlib
+
+import toml
 
 from chats import YoutubeLive
 from chats.ChatProcess import LIFECYCLE_MESSAGES
-from chats.FakeChat import FakeChat
-from chats.Twitch import TwitchIRCProcess
 
 from ui.App import App
-from plugins.PogCount import PogCounter
-
-EVENT_POLL_FREQ = 0.1
-GOOGLE_API_SECRETS_PATH = 'googleapi.secret.json'
-TWITCH_SECRETS_PATH = 'twitch.secret.json'
 
 if __name__ == '__main__':
-    chat_processes = [FakeChat(EVENT_POLL_FREQ)]
+    with open('config.toml', 'r') as f:
-    if os.path.exists(GOOGLE_API_SECRETS_PATH):
+        config = toml.load(f)
-        with open(GOOGLE_API_SECRETS_PATH, 'r') as f:
-            client_secrets = json.load(f)['installed']
-        chat_processes.append(YoutubeLive.YoutubeLiveProcess(client_secrets, EVENT_POLL_FREQ))
-    else:
-        print('No client secrets - disabling youtube chat client')
-    if os.path.exists(TWITCH_SECRETS_PATH):
-        with open(TWITCH_SECRETS_PATH ,'r') as f:
-            secrets = json.load(f)
-        chat_processes.append(TwitchIRCProcess(secrets.get('username'), secrets.get('oauth'), EVENT_POLL_FREQ))
-    else:
-        print('No twitch config - disabling twitch chat client')
 
-    if len(chat_processes) == 1:
+    EVENT_POLL_FREQUENCY = config['event_poll_frequency']
-        print('Hit "f" with the window focused to enable a testing client')
 
-    plugins = [PogCounter('../live-status.txt', prefix='Pog count: ')]
+    # Load dynamic compoents
+    chat_processes = []
+    for chat_name, chat_config in config.get('chat', {}).items():
+        if not chat_config['enabled']:
+            continue
+        del chat_config['enabled']
 
+        chat_module = importlib.import_module(f'.{chat_name}', package='chats')
+        try:
+            chat_process = chat_module.Process(EVENT_POLL_FREQUENCY, **chat_config)
+            chat_processes.append(chat_process)
+        except Exception as e:
+            print(f'Failed to initalize {chat_name} - {e}')
+
+    if len(chat_processes) == 0:
+        print('No chats configured!')
+
+    plugins = []
+    for plugin_name, plugin_config in config.get('plugin', {}).items():
+        if not plugin_config['enabled']:
+            continue
+        del plugin_config['enabled']
+
+        plugin_module = importlib.import_module(f'.{plugin_name}', package='plugins')
+        try:
+            plugin = plugin_module.Plugin(**plugin_config)
+            plugins.append(plugin)
+        except Exception as e:
+            print(f'Failed to initalize {plugin_name} - {e}')
+
+    # Start the app and subprocesses
     app = App(chat_processes, plugins)
 
     for process in chat_processes:
         process.start()
     app.start()
 
+    # Event loop
     while True:
         if not app.is_alive():
             break
 
         for process in chat_processes:
+            # HACK: need to find a generic way to do this
             if process.control_pipe.poll():
                 chat_control_msg = process.control_pipe.recv()
                 if chat_control_msg == YoutubeLive.CONTROL_MESSAGES.NEEDS_OAUTH_CONSENT_TOKEN:
@@ -53,8 +66,9 @@ if __name__ == '__main__':
                         'type': YoutubeLive.CONTROL_MESSAGES.OAUTH_CONSENT_TOKEN,
                         'token': code
                     })
-        time.sleep(EVENT_POLL_FREQ)
+        time.sleep(EVENT_POLL_FREQUENCY)
 
+    # Cleanup
     for process in chat_processes:
         process.control_pipe.send({'type': LIFECYCLE_MESSAGES.SHUTDOWN})
         process.join(10)

plugins/PhraseCounter.py

@@ -0,0 +1,84 @@
+from .PluginBase import PluginBase
+from string import Formatter
+
+from num2words import num2words
+
+
+def remove_dups(text):
+    last_char = None
+    for char in list(text):
+        if last_char == char:
+            continue
+        last_char = char
+        yield char
+
+
+class PhraseCounterFormatter(Formatter):
+    def convert_field(self, value, conversion):
+        if conversion == 's':
+            return num2words(value)
+        if conversion == 'p':
+            return value == 1
+        else:
+            return super().convert_field(value, conversion)
+
+    def format_field(self, value, format_spec):
+        if '|' in format_spec:
+            singular, plural = format_spec.split('|')
+            return singular if value else plural
+        else:
+            return super().format_field(value, format_spec)
+
+
+class Plugin(PluginBase):
+    def __init__(self, out_path=None, phrases=None, template=None,
+                 detect_spaced=False, detect_extentions=False, debounce_time=1):
+        if out_path is None or phrases is None or template is None:
+            raise ValueError('Missing config requirements')
+        self.out = out_path
+        self.phrases = phrases
+        self.template = template
+        self.detect_extentions = detect_extentions
+        self.detect_spaced = detect_spaced
+        self.debounce_time = debounce_time
+
+        self._counts = {phrase: 0 for phrase in phrases + ['sum', 'match', 'nmatch']}
+        self._timer = 0
+        self._dirty = True
+        self._formatter = PhraseCounterFormatter()
+
+    def normalize(self, text):
+        if self.detect_extentions:
+            text = ''.join(remove_dups(text.lower()))
+        if self.detect_spaced:
+            text = ''.join(text.split(' '))
+        return text
+
+    def tick(self, dt):
+        self._timer += dt
+        if self._timer > self.debounce_time and self._dirty:
+            out_string = self._formatter.format(self.template, **self._counts)
+            with open(self.out, 'w') as f:
+                f.write(out_string)
+            self._timer = 0
+            self._dirty = False
+
+    def handle_event(self, event):
+        pass
+
+    def on_message(self, message):
+        self._dirty = True
+        normalized = self.normalize(message.text)
+
+        has_match = False
+        for phrase in self.phrases:
+            if phrase in normalized:
+                self._counts[phrase] += 1
+                self._counts['sum'] += 1
+                has_match = True
+        if has_match:
+            self._counts['match'] += 1
+        else:
+            self._counts['nmatch'] += 1
+
+        return message

plugins/PluginBase.py

@@ -0,0 +1,18 @@
+from abc import ABC, abstractmethod
+
+
+class PluginBase(ABC):
+    def __init__(self):
+        super().__init__()
+
+    @abstractmethod
+    def tick(self, dt):
+        pass
+
+    @abstractmethod
+    def handle_event(self, event):
+        pass
+
+    @abstractmethod
+    def on_message(self, message):
+        pass

plugins/__init__.py

@@ -1 +0,0 @@
-from chats import AUTHOR_TYPES