From 7f025abd5b75ed0982ada81475c40cfd0635bc1d Mon Sep 17 00:00:00 2001 From: iou1name Date: Wed, 11 Dec 2019 19:47:19 -0500 Subject: [PATCH] third commit --- filters.py | 6 +-- scrape_quest.py | 17 +++++--- static/voyage.css | 8 ++++ static/voyage.js | 45 +++++++++++++++++++++ templates/thread.html | 19 ++++++--- voyage.py | 91 +++++++++++++++++++++++++++++-------------- voyage.sql | 25 +++++++----- 7 files changed, 156 insertions(+), 55 deletions(-) diff --git a/filters.py b/filters.py index d6f1fbd..28fa91b 100644 --- a/filters.py +++ b/filters.py @@ -4,13 +4,13 @@ Jinja2 template filters. """ import re -from markupsafe import Markup +from markupsafe import Markup, escape -def quotelink(line, post): +def quotelink(line, post_links): """Checks if the quotelink is valid and adds an anchor tag if so.""" links = re.findall(r'>>(\d+)', line) for link in links: - if int(link) in post['link_tos']: + if int(link) in post_links: span = f'>>{link}' else: span = f'>>{link}' diff --git a/scrape_quest.py b/scrape_quest.py index 8b74937..7ec9113 100644 --- a/scrape_quest.py +++ b/scrape_quest.py @@ -51,9 +51,9 @@ def scrape_posts(root_dir): # information gathering post_id = int(post.get('id')[2:]) name = post.find(class_='name').text - trip_code = post.find(class_='postertrip') - if trip_code: - trip_code = trip_code.text + tripcode = post.find(class_='postertrip') + if tripcode: + tripcode = tripcode.text subject = post.find(class_='subject') if subject: subject = subject.text @@ -73,6 +73,7 @@ def scrape_posts(root_dir): links = post.find_all(class_='quotelink') links = [l for l in links if l.get('href').startswith('#')] links = [int(link.text[2:]) for link in links] + links = list(set(links)) # heuristics tags = [] @@ -110,9 +111,9 @@ def scrape_posts(root_dir): # database insert cur.execute( - "INSERT INTO post VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", - (thread_id, post_id, name, trip_code, subject, - post_time, file_url, file_name, file_md5, post_body) + "INSERT INTO post VALUES (%s,%s,%s,%s,%s,%s,%s)", + (thread_id, post_id, name, tripcode, subject, + post_time, post_body) ) for link in links: cur.execute("INSERT INTO link VALUES (%s,%s)", @@ -122,6 +123,10 @@ def scrape_posts(root_dir): cur.execute("INSERT INTO tag VALUES (%s,%s)", (post_id, tag) ) + if file_text: + cur.execute("INSERT INTO file VALUES (%s,%s,%s)", + (file_url, file_name, file_md5) + ) if __name__ == '__main__': diff --git a/static/voyage.css b/static/voyage.css index 3cc940a..b2bd219 100644 --- a/static/voyage.css +++ b/static/voyage.css @@ -10,6 +10,14 @@ body { padding: 0.5em; } +.faded { + opacity: 0.33; +} + +.tag { + font-size: 0.8em; +} + .header { margin-bottom: 0.5em; } diff --git a/static/voyage.js b/static/voyage.js index e69de29..8c462e2 100644 --- a/static/voyage.js +++ b/static/voyage.js @@ -0,0 +1,45 @@ +function add_tag(input) { + post_id = input.closest('.post_container').id; + tag_name = input.children[0].value; + fetch(url_prefix + '/add_tag', { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({ + 'post_id': post_id, + 'tag_name': tag_name, + }) + }).then(function(response) { + if(!response.ok) { throw new Error('Error adding tag!'); } + return response.json(); + }).then(function(json) { + if (!json.ok) { throw new Error('Could not add tag.') } + let tag = document.createElement('span'); + tag.innerText = tag_name; + tag.className = 'tag'; + let anchor = document.createElement('a'); + anchor.href = "javascript:void(0)"; + anchor.setAttribute('onlick', "remove_tag(this.closest('.tag'))"); + anchor.innerText = '-'; + tag.appendChild(anchor); + input.closest('.tags').insertBefore(tag, input); + }); +} + +function remove_tag(tag) { + post_id = tag.closest('.post_container').id; + tag_name = tag.innerText.replace(/ .*/, ''); + fetch(url_prefix + '/remove_tag', { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({ + 'post_id': post_id, + 'tag_name': tag_name, + }) + }).then(function(response) { + if(!response.ok) { throw new Error('Error removing tag!'); } + return response.json(); + }).then(function(json) { + if (!json.ok) { throw new Error('Could not remove tag.') } + tag.remove(); + }); +} diff --git a/templates/thread.html b/templates/thread.html index d7d9528..e260464 100644 --- a/templates/thread.html +++ b/templates/thread.html @@ -3,6 +3,7 @@ Voyage + @@ -12,15 +13,21 @@

Voyage

- {% for post_id, post in posts.items() %} -
+ {% for post in posts %} +
+
+ {% for tag in tags.get(post.id, []) %} + {{ tag }} - + {% endfor %} + + +
{% if post.subject %} {{ post.subject }} {% endif %} {{ post.name }} - {% if post.trip_code %} - {{ post.trip_code }} + {% if post.tripcode %} + {{ post.tripcode }} {% endif %} {{ post.time.strftime('%Y-%m-%d %H:%M') }} No.{{ post.id }} @@ -35,9 +42,9 @@
{% for line in post.body.splitlines() %} {% if line.startswith('>') %} - {{ line|quotelink(post) }}
+ {{ line|quotelink(links[post.id]) }}
{% else %} - {{ line|quotelink(post) }}
+ {{ line|quotelink(links[post.id]) }}
{% endif %} {% endfor %}
diff --git a/voyage.py b/voyage.py index 77e9c30..416a28e 100644 --- a/voyage.py +++ b/voyage.py @@ -2,6 +2,8 @@ """ A display interface for archived 4chan quest threads. """ +from collections import defaultdict + from aiohttp import web import jinja2 import aiohttp_jinja2 @@ -26,41 +28,70 @@ async def index(request): @routes.get(r'/{thread_id:\d+}', name='thread') async def thread(request): """A single thread page.""" + thread_id = int(request.match_info['thread_id']) async with request.app['pool'].acquire() as conn: - data = await conn.fetch( - "SELECT post.id, post.name, post.trip_code, post.subject, " - "post.time, post.body, " - "tag.name AS tag_name, link.link_to, link.link_from " - "FROM post " - "LEFT JOIN tag ON (post.id = tag.post_id) " - "LEFT JOIN link ON (post.id = link.link_from) " - "WHERE post.thread_id = $1 " - "ORDER BY post.id ASC", - int(request.match_info['thread_id'])) - posts = {} - backlinks = {} - for row in data: - post = posts.get(row['id'], {}) - for key, value in row.items(): - if key == 'tag_name': - tags = post.get('tags', []) - tags.append(value) - post['tags'] = tags - elif key == 'link_to': - link_tos = post.get('link_tos', []) - link_tos.append(value) - post['link_tos'] = link_tos - elif key == 'link_from': - back_post = backlinks.get(row['link_to'], []) - back_post.append(value) - backlinks[row['link_to']] = back_post - post[key] = value - post['tags'] = [t for t in post['tags'] if t] - posts[row['id']] = post + posts = await conn.fetch( + "SELECT * FROM post WHERE thread_id = $1 ORDER BY id ASC", + thread_id) + tags_raw = await conn.fetch( + "SELECT tag.post_id, tag.name FROM tag " + "LEFT JOIN post ON (tag.post_id = post.id) " + "WHERE post.thread_id = $1", + thread_id) + links_raw = await conn.fetch( + "SELECT link.link_from, link.link_to FROM link " + "LEFT JOIN post ON (link.link_from = post.id) " + "WHERE post.thread_id = $1", + thread_id) + tags = defaultdict(list) + links = defaultdict(list) + backlinks = defaultdict(list) + + for tag_raw in tags_raw: + tag = tags[tag_raw['post_id']] + tag.append(tag_raw['name']) + tags[tag_raw['post_id']] = tag + + for link_raw in links_raw: + link = links[link_raw['link_from']] + link.append(link_raw['link_to']) + links[link_raw['link_from']] = link + + backlink = backlinks[link_raw['link_to']] + backlink.append(link_raw['link_from']) + backlinks[link_raw['link_to']] = backlink + + url_prefix = config.url_prefix return render_template("thread.html", request, locals()) +@routes.post('/add_tag', name='add_tag') +async def add_tag(request): + """Adds a tag to a post.""" + data = await request.json() + post_id = int(data.get('post_id')) + tag_name = data.get('tag_name') + async with request.app['pool'].acquire() as conn: + await conn.execute( + "INSERT INTO tag (post_id, name) VALUES ($1, $2)", + post_id, tag_name) + return web.json_response({'ok': True}) + + +@routes.post('/remove_tag', name='remove_tag') +async def remove_tag(request): + """Removes a tag to a post.""" + data = await request.json() + post_id = int(data.get('post_id')) + tag_name = data.get('tag_name') + async with request.app['pool'].acquire() as conn: + await conn.execute( + "DELETE FROM tag WHERE post_id = $1 AND name = $2", + post_id, tag_name) + return web.json_response({'ok': True}) + + async def init_app(): """Initializes the application.""" app = web.Application() diff --git a/voyage.sql b/voyage.sql index 2818cdc..9db64f5 100644 --- a/voyage.sql +++ b/voyage.sql @@ -1,28 +1,33 @@ -CREATE TABLE IF NOT EXISTS thread( +CREATE TABLE IF NOT EXISTS thread ( id INTEGER PRIMARY KEY, title TEXT NOT NULL, time TIMESTAMP WITH TIME ZONE NOT NULL ); -CREATE TABLE IF NOT EXISTS post( +CREATE TABLE IF NOT EXISTS post ( thread_id INTEGER REFERENCES thread(id) ON DELETE CASCADE NOT NULL, id INTEGER PRIMARY KEY, name TEXT NOT NULL, - trip_code TEXT, + tripcode TEXT, subject TEXT, time TIMESTAMP WITH TIME ZONE NOT NULL, - file_url TEXT, - file_name TEXT, - file_md5 TEXT, body TEXT NOT NULL ); -CREATE TABLE IF NOT EXISTS link( +CREATE TABLE IF NOT EXISTS link ( link_from INTEGER REFERENCES post(id) ON DELETE CASCADE NOT NULL, - link_to INTEGER REFERENCES post(id) ON DELETE CASCADE NOT NULL + link_to INTEGER REFERENCES post(id) ON DELETE CASCADE NOT NULL, + PRIMARY KEY (link_from, link_to) ); -CREATE TABLE IF NOT EXISTS tag( +CREATE TABLE IF NOT EXISTS tag ( post_id INTEGER REFERENCES post(id) ON DELETE CASCADE NOT NULL, - name TEXT NOT NULL + name TEXT NOT NULL, + PRIMARY KEY (post_id, name) +); + +CREATE TABLE IF NOT EXISTS file ( + file_url TEXT, + file_name TEXT, + file_md5 TEXT );