From fb2637a14ef64ec0fdc12bdfa9e3487692308458 Mon Sep 17 00:00:00 2001 From: iou1name Date: Tue, 13 Feb 2018 16:31:23 -0500 Subject: [PATCH] thread watcher writes to database incase of disconnect --- db.py | 405 ++++++++++++++++++++++----------------------- modules/watcher.py | 75 +++++++-- 2 files changed, 259 insertions(+), 221 deletions(-) diff --git a/db.py b/db.py index a7c0afb..e5e1a2b 100755 --- a/db.py +++ b/db.py @@ -1,6 +1,3 @@ -# coding=utf-8 -from __future__ import unicode_literals, absolute_import, print_function, division - import json import os.path import sys @@ -8,240 +5,236 @@ import sqlite3 from tools import Identifier -if sys.version_info.major >= 3: - unicode = str - basestring = str - def _deserialize(value): - if value is None: - return None - # sqlite likes to return ints for strings that look like ints, even though - # the column type is string. That's how you do dynamic typing wrong. - value = unicode(value) - # Just in case someone's mucking with the DB in a way we can't account for, - # ignore json parsing errors - try: - value = json.loads(value) - except: - pass - return value + if value is None: + return None + # sqlite likes to return ints for strings that look like ints, even though + # the column type is string. That's how you do dynamic typing wrong. + value = str(value) + # Just in case someone's mucking with the DB in a way we can't account for, + # ignore json parsing errors + try: + value = json.loads(value) + except: + pass + return value class SopelDB(object): - """*Availability: 5.0+* + """*Availability: 5.0+* - This defines an interface for basic, common operations on a sqlite - database. It simplifies those common operations, and allows direct access - to the database, wherever the user has configured it to be. + This defines an interface for basic, common operations on a sqlite + database. It simplifies those common operations, and allows direct access + to the database, wherever the user has configured it to be. - When configured with a relative filename, it is assumed to be in the same - directory as the config.""" + When configured with a relative filename, it is assumed to be in the same + directory as the config.""" - def __init__(self, config): - path = config.core.db_filename - config_dir, config_file = os.path.split(config.filename) - config_name, _ = os.path.splitext(config_file) - if path is None: - path = os.path.join(config_dir, config_name + '.db') - path = os.path.expanduser(path) - if not os.path.isabs(path): - path = os.path.normpath(os.path.join(config_dir, path)) - self.filename = path - self._create() + def __init__(self, config): + path = config.core.db_filename + config_dir, config_file = os.path.split(config.filename) + config_name, _ = os.path.splitext(config_file) + if path is None: + path = os.path.join(config_dir, config_name + '.db') + path = os.path.expanduser(path) + if not os.path.isabs(path): + path = os.path.normpath(os.path.join(config_dir, path)) + self.filename = path + self._create() - def connect(self): - """Return a raw database connection object.""" - return sqlite3.connect(self.filename, timeout=10) + def connect(self): + """Return a raw database connection object.""" + return sqlite3.connect(self.filename, timeout=10) - def execute(self, *args, **kwargs): - """Execute an arbitrary SQL query against the database. + def execute(self, *args, **kwargs): + """Execute an arbitrary SQL query against the database. - Returns a cursor object, on which things like `.fetchall()` can be - called per PEP 249.""" - with self.connect() as conn: - cur = conn.cursor() - return cur.execute(*args, **kwargs) + Returns a cursor object, on which things like `.fetchall()` can be + called per PEP 249.""" + with self.connect() as conn: + cur = conn.cursor() + return cur.execute(*args, **kwargs) - def _create(self): - """Create the basic database structure.""" - # Do nothing if the db already exists. - try: - self.execute('SELECT * FROM nick_ids;') - self.execute('SELECT * FROM nicknames;') - self.execute('SELECT * FROM nick_values;') - self.execute('SELECT * FROM channel_values;') - except: - pass - else: - return + def _create(self): + """Create the basic database structure.""" + # Do nothing if the db already exists. + try: + self.execute('SELECT * FROM nick_ids;') + self.execute('SELECT * FROM nicknames;') + self.execute('SELECT * FROM nick_values;') + self.execute('SELECT * FROM channel_values;') + except: + pass + else: + return - self.execute( - 'CREATE TABLE nick_ids (nick_id INTEGER PRIMARY KEY AUTOINCREMENT)' - ) - self.execute( - 'CREATE TABLE nicknames ' - '(nick_id INTEGER REFERENCES nick_ids, ' - 'slug STRING PRIMARY KEY, canonical string)' - ) - self.execute( - 'CREATE TABLE nick_values ' - '(nick_id INTEGER REFERENCES nick_ids(nick_id), ' - 'key STRING, value STRING, ' - 'PRIMARY KEY (nick_id, key))' - ) - self.execute( - 'CREATE TABLE channel_values ' - '(channel STRING, key STRING, value STRING, ' - 'PRIMARY KEY (channel, key))' - ) + self.execute( + 'CREATE TABLE nick_ids (nick_id INTEGER PRIMARY KEY AUTOINCREMENT)' + ) + self.execute( + 'CREATE TABLE nicknames ' + '(nick_id INTEGER REFERENCES nick_ids, ' + 'slug STRING PRIMARY KEY, canonical string)' + ) + self.execute( + 'CREATE TABLE nick_values ' + '(nick_id INTEGER REFERENCES nick_ids(nick_id), ' + 'key STRING, value STRING, ' + 'PRIMARY KEY (nick_id, key))' + ) + self.execute( + 'CREATE TABLE channel_values ' + '(channel STRING, key STRING, value STRING, ' + 'PRIMARY KEY (channel, key))' + ) - def get_uri(self): - """Returns a URL for the database, usable to connect with SQLAlchemy. - """ - return 'sqlite://{}'.format(self.filename) + def get_uri(self): + """Returns a URL for the database, usable to connect with SQLAlchemy. + """ + return 'sqlite://{}'.format(self.filename) - # NICK FUNCTIONS + # NICK FUNCTIONS - def get_nick_id(self, nick, create=True): - """Return the internal identifier for a given nick. + def get_nick_id(self, nick, create=True): + """Return the internal identifier for a given nick. - This identifier is unique to a user, and shared across all of that - user's aliases. If create is True, a new ID will be created if one does - not already exist""" - slug = nick.lower() - nick_id = self.execute('SELECT nick_id from nicknames where slug = ?', - [slug]).fetchone() - if nick_id is None: - if not create: - raise ValueError('No ID exists for the given nick') - with self.connect() as conn: - cur = conn.cursor() - cur.execute('INSERT INTO nick_ids VALUES (NULL)') - nick_id = cur.execute('SELECT last_insert_rowid()').fetchone()[0] - cur.execute( - 'INSERT INTO nicknames (nick_id, slug, canonical) VALUES ' - '(?, ?, ?)', - [nick_id, slug, nick] - ) - nick_id = self.execute('SELECT nick_id from nicknames where slug = ?', - [slug]).fetchone() - return nick_id[0] + This identifier is unique to a user, and shared across all of that + user's aliases. If create is True, a new ID will be created if one does + not already exist""" + slug = nick.lower() + nick_id = self.execute('SELECT nick_id from nicknames where slug = ?', + [slug]).fetchone() + if nick_id is None: + if not create: + raise ValueError('No ID exists for the given nick') + with self.connect() as conn: + cur = conn.cursor() + cur.execute('INSERT INTO nick_ids VALUES (NULL)') + nick_id = cur.execute('SELECT last_insert_rowid()').fetchone()[0] + cur.execute( + 'INSERT INTO nicknames (nick_id, slug, canonical) VALUES ' + '(?, ?, ?)', + [nick_id, slug, nick] + ) + nick_id = self.execute('SELECT nick_id from nicknames where slug = ?', + [slug]).fetchone() + return nick_id[0] - def alias_nick(self, nick, alias): - """Create an alias for a nick. + def alias_nick(self, nick, alias): + """Create an alias for a nick. - Raises ValueError if the alias already exists. If nick does not already - exist, it will be added along with the alias.""" - nick = Identifier(nick) - alias = Identifier(alias) - nick_id = self.get_nick_id(nick) - sql = 'INSERT INTO nicknames (nick_id, slug, canonical) VALUES (?, ?, ?)' - values = [nick_id, alias.lower(), alias] - try: - self.execute(sql, values) - except sqlite3.IntegrityError: - raise ValueError('Alias already exists.') + Raises ValueError if the alias already exists. If nick does not already + exist, it will be added along with the alias.""" + nick = Identifier(nick) + alias = Identifier(alias) + nick_id = self.get_nick_id(nick) + sql = 'INSERT INTO nicknames (nick_id, slug, canonical) VALUES (?, ?, ?)' + values = [nick_id, alias.lower(), alias] + try: + self.execute(sql, values) + except sqlite3.IntegrityError: + raise ValueError('Alias already exists.') - def set_nick_value(self, nick, key, value): - """Sets the value for a given key to be associated with the nick.""" - nick = Identifier(nick) - value = json.dumps(value, ensure_ascii=False) - nick_id = self.get_nick_id(nick) - self.execute('INSERT OR REPLACE INTO nick_values VALUES (?, ?, ?)', - [nick_id, key, value]) + def set_nick_value(self, nick, key, value): + """Sets the value for a given key to be associated with the nick.""" + nick = Identifier(nick) + value = json.dumps(value, ensure_ascii=False) + nick_id = self.get_nick_id(nick) + self.execute('INSERT OR REPLACE INTO nick_values VALUES (?, ?, ?)', + [nick_id, key, value]) - def get_nick_value(self, nick, key): - """Retrieves the value for a given key associated with a nick.""" - nick = Identifier(nick) - result = self.execute( - 'SELECT value FROM nicknames JOIN nick_values ' - 'ON nicknames.nick_id = nick_values.nick_id ' - 'WHERE slug = ? AND key = ?', - [nick.lower(), key] - ).fetchone() - if result is not None: - result = result[0] - return _deserialize(result) + def get_nick_value(self, nick, key): + """Retrieves the value for a given key associated with a nick.""" + nick = Identifier(nick) + result = self.execute( + 'SELECT value FROM nicknames JOIN nick_values ' + 'ON nicknames.nick_id = nick_values.nick_id ' + 'WHERE slug = ? AND key = ?', + [nick.lower(), key] + ).fetchone() + if result is not None: + result = result[0] + return _deserialize(result) - def unalias_nick(self, alias): - """Removes an alias. + def unalias_nick(self, alias): + """Removes an alias. - Raises ValueError if there is not at least one other nick in the group. - To delete an entire group, use `delete_group`. - """ - alias = Identifier(alias) - nick_id = self.get_nick_id(alias, False) - count = self.execute('SELECT COUNT(*) FROM nicknames WHERE nick_id = ?', - [nick_id]).fetchone()[0] - if count <= 1: - raise ValueError('Given alias is the only entry in its group.') - self.execute('DELETE FROM nicknames WHERE slug = ?', [alias.lower()]) + Raises ValueError if there is not at least one other nick in the group. + To delete an entire group, use `delete_group`. + """ + alias = Identifier(alias) + nick_id = self.get_nick_id(alias, False) + count = self.execute('SELECT COUNT(*) FROM nicknames WHERE nick_id = ?', + [nick_id]).fetchone()[0] + if count <= 1: + raise ValueError('Given alias is the only entry in its group.') + self.execute('DELETE FROM nicknames WHERE slug = ?', [alias.lower()]) - def delete_nick_group(self, nick): - """Removes a nickname, and all associated aliases and settings. - """ - nick = Identifier(nick) - nick_id = self.get_nick_id(nick, False) - self.execute('DELETE FROM nicknames WHERE nick_id = ?', [nick_id]) - self.execute('DELETE FROM nick_values WHERE nick_id = ?', [nick_id]) + def delete_nick_group(self, nick): + """Removes a nickname, and all associated aliases and settings. + """ + nick = Identifier(nick) + nick_id = self.get_nick_id(nick, False) + self.execute('DELETE FROM nicknames WHERE nick_id = ?', [nick_id]) + self.execute('DELETE FROM nick_values WHERE nick_id = ?', [nick_id]) - def merge_nick_groups(self, first_nick, second_nick): - """Merges the nick groups for the specified nicks. + def merge_nick_groups(self, first_nick, second_nick): + """Merges the nick groups for the specified nicks. - Takes two nicks, which may or may not be registered. Unregistered - nicks will be registered. Keys which are set for only one of the given - nicks will be preserved. Where multiple nicks have values for a given - key, the value set for the first nick will be used. + Takes two nicks, which may or may not be registered. Unregistered + nicks will be registered. Keys which are set for only one of the given + nicks will be preserved. Where multiple nicks have values for a given + key, the value set for the first nick will be used. - Note that merging of data only applies to the native key-value store. - If modules define their own tables which rely on the nick table, they - will need to have their merging done separately.""" - first_id = self.get_nick_id(Identifier(first_nick)) - second_id = self.get_nick_id(Identifier(second_nick)) - self.execute( - 'UPDATE OR IGNORE nick_values SET nick_id = ? WHERE nick_id = ?', - [first_id, second_id]) - self.execute('DELETE FROM nick_values WHERE nick_id = ?', [second_id]) - self.execute('UPDATE nicknames SET nick_id = ? WHERE nick_id = ?', - [first_id, second_id]) + Note that merging of data only applies to the native key-value store. + If modules define their own tables which rely on the nick table, they + will need to have their merging done separately.""" + first_id = self.get_nick_id(Identifier(first_nick)) + second_id = self.get_nick_id(Identifier(second_nick)) + self.execute( + 'UPDATE OR IGNORE nick_values SET nick_id = ? WHERE nick_id = ?', + [first_id, second_id]) + self.execute('DELETE FROM nick_values WHERE nick_id = ?', [second_id]) + self.execute('UPDATE nicknames SET nick_id = ? WHERE nick_id = ?', + [first_id, second_id]) - # CHANNEL FUNCTIONS + # CHANNEL FUNCTIONS - def set_channel_value(self, channel, key, value): - channel = Identifier(channel).lower() - value = json.dumps(value, ensure_ascii=False) - self.execute('INSERT OR REPLACE INTO channel_values VALUES (?, ?, ?)', - [channel, key, value]) + def set_channel_value(self, channel, key, value): + channel = Identifier(channel).lower() + value = json.dumps(value, ensure_ascii=False) + self.execute('INSERT OR REPLACE INTO channel_values VALUES (?, ?, ?)', + [channel, key, value]) - def get_channel_value(self, channel, key): - """Retrieves the value for a given key associated with a channel.""" - channel = Identifier(channel).lower() - result = self.execute( - 'SELECT value FROM channel_values WHERE channel = ? AND key = ?', - [channel, key] - ).fetchone() - if result is not None: - result = result[0] - return _deserialize(result) + def get_channel_value(self, channel, key): + """Retrieves the value for a given key associated with a channel.""" + channel = Identifier(channel).lower() + result = self.execute( + 'SELECT value FROM channel_values WHERE channel = ? AND key = ?', + [channel, key] + ).fetchone() + if result is not None: + result = result[0] + return _deserialize(result) - # NICK AND CHANNEL FUNCTIONS + # NICK AND CHANNEL FUNCTIONS - def get_nick_or_channel_value(self, name, key): - """Gets the value `key` associated to the nick or channel `name`. - """ - name = Identifier(name) - if name.is_nick(): - return self.get_nick_value(name, key) - else: - return self.get_channel_value(name, key) + def get_nick_or_channel_value(self, name, key): + """Gets the value `key` associated to the nick or channel `name`. + """ + name = Identifier(name) + if name.is_nick(): + return self.get_nick_value(name, key) + else: + return self.get_channel_value(name, key) - def get_preferred_value(self, names, key): - """Gets the value for the first name which has it set. + def get_preferred_value(self, names, key): + """Gets the value for the first name which has it set. - `names` is a list of channel and/or user names. Returns None if none of - the names have the key set.""" - for name in names: - value = self.get_nick_or_channel_value(name, key) - if value is not None: - return value + `names` is a list of channel and/or user names. Returns None if none of + the names have the key set.""" + for name in names: + value = self.get_nick_or_channel_value(name, key) + if value is not None: + return value diff --git a/modules/watcher.py b/modules/watcher.py index 7b54a26..820618a 100755 --- a/modules/watcher.py +++ b/modules/watcher.py @@ -15,6 +15,27 @@ def setup(bot): """ bot.memory["watcher"] = {} + con = bot.db.connect() + cur = con.cursor() + try: + watching = cur.execute("SELECT * FROM watcher").fetchall() + except: + cur.execute("CREATE TABLE watcher(" + "api_url STRING PRIMARY KEY," + "name STRING DEFAULT 'Anonymous'," + "last_post INT," + "time_since STRING," + "channel STRING)") + cur.commit() + con.close() + else: + for thread in watching: + t = WatcherThread(bot, thread[0], thread[1], thread[2], thread[3], + thread[4]) + t.start() + bot.memory["watcher"][get_thread_url(thread[0])] = t + con.close() + def get_time(): """ @@ -64,7 +85,9 @@ def watch(bot, trigger): A thread watcher for 4chan. """ url = trigger.group(3) - op_name = trigger.group(4) + name = trigger.group(4) + if not name: + name = "Anonymous" if url in bot.memory["watcher"].keys(): return bot.say("Error: I'm already watching that thread.") @@ -75,12 +98,21 @@ def watch(bot, trigger): return bot.say("404: thread not found") thread = res.json() - last_post = get_last_post(thread, op_name) + last_post = get_last_post(thread, name) time_since = get_time() - t = WatcherThread(bot, api_url, op_name, last_post, time_since) + t = WatcherThread(bot, api_url, name, last_post, time_since, trigger.sender) t.start() bot.memory["watcher"][url] = t + + con = bot.db.connect() + cur = con.cursor() + cur.execute("INSERT INTO watcher(api_url, name, last_post, time_since, channel)" + " VALUES(?,?,?,?,?)", (api_url, name, last_post, time_since, + trigger.sender)) + con.commit() + con.close() + bot.say("[\x0304Watcher\x03] Watching thread: \x0307" + url) @@ -96,51 +128,64 @@ def unwatch(bot, trigger): bot.memory["watcher"].pop(url) except KeyError: return bot.say("Error: I'm not watching that thread.") - return bot.say("[\x0304Watcher\x03] No longer watching: \x0307" + url) + + con = bot.db.connect() + cur = con.cursor() + cur.execute("DELETE FROM watcher WHERE api_url = ?", (get_api_url(url),)) + con.commit() + con.close() + + bot.say("[\x0304Watcher\x03] No longer watching: \x0307" + url) class WatcherThread(threading.Thread): - def __init__(self, bot, api_url, op_name, last_post, time_since): + def __init__(self, bot, api_url, name, last_post, time_since, channel): threading.Thread.__init__(self) self.stop = threading.Event() self.period = 20 self.bot = bot self.api_url = api_url - self.op_name = op_name + self.name = name self.last_post = last_post self.time_since = time_since + self.channel = channel def run(self): while not self.stop.is_set(): + self.stop.wait(self.period) + headers = {"If-Modified-Since": self.time_since} - res = requests.get(self.api_url, headers=headers, verify=True) - self.time_since = get_time() + try: + res = requests.get(self.api_url, headers=headers, verify=True) + self.time_since = get_time() + except urllib3.exceptions.NewConnectionError: + print(f"Watcher: Thread {self.api_url}: Connection error") + continue if res.status_code == 404: msg = "[\x0304Watcher\x03] Thread deleted: " \ + f"\x0307{get_thread_url(self.api_url)}" - self.bot.say(msg) + self.bot.say(msg, channel) self.stop.set() continue if res.status_code == 304: - self.stop.wait(self.period) continue thread = res.json() if thread["posts"][0].get("closed"): msg = "[\x0304Watcher\x03] Thread closed: " \ + f"\x0307{get_thread_url(self.api_url)}" - self.bot.say(msg) + self.bot.say(msg, channel) self.stop.set() continue - new_last_post = get_last_post(thread, self.op_name) + new_last_post = get_last_post(thread, self.name) if new_last_post > self.last_post: self.last_post = new_last_post msg = "[\x0304Watcher\x03] New post from \x0308" \ - + f"{self.op_name}\x03 in \x0307{get_thread_url(self.api_url)}" - self.bot.say(msg) - self.stop.wait(self.period) + + f"{self.name}\x03 in \x0307{get_thread_url(self.api_url)}" + self.bot.say(msg, channel) +