iou1name/Voyage
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

third commit

Browse Source
This commit is contained in:
iou1name 2019-12-11 19:47:19 -05:00
parent 3beb5af2c9
commit 7f025abd5b
7 changed files with 156 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
filters.py
View File

@ -4,13 +4,13 @@ Jinja2 template filters.
""" """
import re import re
from markupsafe import Markup from markupsafe import Markup, escape
def quotelink(line, post): def quotelink(line, post_links):
"""Checks if the quotelink is valid and adds an anchor tag if so.""" """Checks if the quotelink is valid and adds an anchor tag if so."""
links = re.findall(r'>>(\d+)', line) links = re.findall(r'>>(\d+)', line)
for link in links: for link in links:
if int(link) in post['link_tos']: if int(link) in post_links:
span = f'<a class="quotelink" href="#{link}">>>{link}</a>' span = f'<a class="quotelink" href="#{link}">>>{link}</a>'
else: else:
span = f'<span class="deadlink">>>{link}</span>' span = f'<span class="deadlink">>>{link}</span>'

17
scrape_quest.py
View File

@ -51,9 +51,9 @@ def scrape_posts(root_dir):
# information gathering # information gathering
post_id = int(post.get('id')[2:]) post_id = int(post.get('id')[2:])
name = post.find(class_='name').text name = post.find(class_='name').text
trip_code = post.find(class_='postertrip') tripcode = post.find(class_='postertrip')
if trip_code: if tripcode:
trip_code = trip_code.text tripcode = tripcode.text
subject = post.find(class_='subject') subject = post.find(class_='subject')
if subject: if subject:
subject = subject.text subject = subject.text
@ -73,6 +73,7 @@ def scrape_posts(root_dir):
links = post.find_all(class_='quotelink') links = post.find_all(class_='quotelink')
links = [l for l in links if l.get('href').startswith('#')] links = [l for l in links if l.get('href').startswith('#')]
links = [int(link.text[2:]) for link in links] links = [int(link.text[2:]) for link in links]
links = list(set(links))
# heuristics # heuristics
tags = [] tags = []
@ -110,9 +111,9 @@ def scrape_posts(root_dir):
# database insert # database insert
cur.execute( cur.execute(
"INSERT INTO post VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", "INSERT INTO post VALUES (%s,%s,%s,%s,%s,%s,%s)",
(thread_id, post_id, name, trip_code, subject, (thread_id, post_id, name, tripcode, subject,
post_time, file_url, file_name, file_md5, post_body) post_time, post_body)
) )
for link in links: for link in links:
cur.execute("INSERT INTO link VALUES (%s,%s)", cur.execute("INSERT INTO link VALUES (%s,%s)",
@ -122,6 +123,10 @@ def scrape_posts(root_dir):
cur.execute("INSERT INTO tag VALUES (%s,%s)", cur.execute("INSERT INTO tag VALUES (%s,%s)",
(post_id, tag) (post_id, tag)
) )
if file_text:
cur.execute("INSERT INTO file VALUES (%s,%s,%s)",
(file_url, file_name, file_md5)
)
if __name__ == '__main__': if __name__ == '__main__':

8
static/voyage.css
View File

@ -10,6 +10,14 @@ body {
padding: 0.5em; padding: 0.5em;
} }
.faded {
opacity: 0.33;
}
.tag {
font-size: 0.8em;
}
.header { .header {
margin-bottom: 0.5em; margin-bottom: 0.5em;
} }

45
static/voyage.js
View File

@ -0,0 +1,45 @@
function add_tag(input) {
post_id = input.closest('.post_container').id;
tag_name = input.children[0].value;
fetch(url_prefix + '/add_tag', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({
'post_id': post_id,
'tag_name': tag_name,
})
}).then(function(response) {
if(!response.ok) { throw new Error('Error adding tag!'); }
return response.json();
}).then(function(json) {
if (!json.ok) { throw new Error('Could not add tag.') }
let tag = document.createElement('span');
tag.innerText = tag_name;
tag.className = 'tag';
let anchor = document.createElement('a');
anchor.href = "javascript:void(0)";
anchor.setAttribute('onlick', "remove_tag(this.closest('.tag'))");
anchor.innerText = '-';
tag.appendChild(anchor);
input.closest('.tags').insertBefore(tag, input);
});
}
function remove_tag(tag) {
post_id = tag.closest('.post_container').id;
tag_name = tag.innerText.replace(/ .*/, '');
fetch(url_prefix + '/remove_tag', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({
'post_id': post_id,
'tag_name': tag_name,
})
}).then(function(response) {
if(!response.ok) { throw new Error('Error removing tag!'); }
return response.json();
}).then(function(json) {
if (!json.ok) { throw new Error('Could not remove tag.') }
tag.remove();
});
}

19
templates/thread.html
View File

@ -3,6 +3,7 @@
<head> <head>
<title>Voyage</title> <title>Voyage</title>
<link rel="stylesheet" type="text/css" href="/static/voyage.css"> <link rel="stylesheet" type="text/css" href="/static/voyage.css">
<script>const url_prefix = "{{ url_prefix }}";</script>
<script type="text/javascript" src="/static/voyage.js"></script> <script type="text/javascript" src="/static/voyage.js"></script>
<meta name="viewport" content="width=device-width, initial-scale=0.8"> <meta name="viewport" content="width=device-width, initial-scale=0.8">
<meta name="description" content="A quest archive viewer."> <meta name="description" content="A quest archive viewer.">
@ -12,15 +13,21 @@
<h1>Voyage</h1> <h1>Voyage</h1>
</header> </header>
<main> <main>
{% for post_id, post in posts.items() %} {% for post in posts %}
<div id="{{ post.id }}" class="post_container {{ ' '.join(post.tags) }}"> <div id="{{ post.id }}" class="post_container{% if tags[post.id] %} {{ ' '.join(tags[post.id]) }}{% endif %}{% if 'story_post' not in tags[post.id] %} faded{% endif %}">
<div class="tags">
{% for tag in tags.get(post.id, []) %}
<span class="tag">{{ tag }} <a href="javascript:void(0)" onclick="remove_tag(this.closest('.tag'))">-</a></span>
{% endfor %}
<span class="tag"><input type="text"><a href="javascript:void(0)" onclick="add_tag(this.closest('.tag'))">+</a></span>
</div>
<div class="header"> <div class="header">
{% if post.subject %} {% if post.subject %}
<span class="subject">{{ post.subject }}</span> <span class="subject">{{ post.subject }}</span>
{% endif %} {% endif %}
<span class="name">{{ post.name }}</span> <span class="name">{{ post.name }}</span>
{% if post.trip_code %} {% if post.tripcode %}
<span class="tripcode">{{ post.trip_code }}</span> <span class="tripcode">{{ post.tripcode }}</span>
{% endif %} {% endif %}
<span class="time">{{ post.time.strftime('%Y-%m-%d %H:%M') }}</span> <span class="time">{{ post.time.strftime('%Y-%m-%d %H:%M') }}</span>
<span class="id">No.{{ post.id }}</span> <span class="id">No.{{ post.id }}</span>
@ -35,9 +42,9 @@
<div class="body"> <div class="body">
{% for line in post.body.splitlines() %} {% for line in post.body.splitlines() %}
{% if line.startswith('>') %} {% if line.startswith('>') %}
<span class="quote">{{ line|quotelink(post) }}</span><br> <span class="quote">{{ line|quotelink(links[post.id]) }}</span><br>
{% else %} {% else %}
{{ line|quotelink(post) }}<br> {{ line|quotelink(links[post.id]) }}<br>
{% endif %} {% endif %}
{% endfor %} {% endfor %}
</div> </div>

91
voyage.py
View File

@ -2,6 +2,8 @@
""" """
A display interface for archived 4chan quest threads. A display interface for archived 4chan quest threads.
""" """
from collections import defaultdict
from aiohttp import web from aiohttp import web
import jinja2 import jinja2
import aiohttp_jinja2 import aiohttp_jinja2
@ -26,41 +28,70 @@ async def index(request):
@routes.get(r'/{thread_id:\d+}', name='thread') @routes.get(r'/{thread_id:\d+}', name='thread')
async def thread(request): async def thread(request):
"""A single thread page.""" """A single thread page."""
thread_id = int(request.match_info['thread_id'])
async with request.app['pool'].acquire() as conn: async with request.app['pool'].acquire() as conn:
data = await conn.fetch( posts = await conn.fetch(
"SELECT post.id, post.name, post.trip_code, post.subject, " "SELECT * FROM post WHERE thread_id = $1 ORDER BY id ASC",
"post.time, post.body, " thread_id)
"tag.name AS tag_name, link.link_to, link.link_from " tags_raw = await conn.fetch(
"FROM post " "SELECT tag.post_id, tag.name FROM tag "
"LEFT JOIN tag ON (post.id = tag.post_id) " "LEFT JOIN post ON (tag.post_id = post.id) "
"LEFT JOIN link ON (post.id = link.link_from) " "WHERE post.thread_id = $1",
"WHERE post.thread_id = $1 " thread_id)
"ORDER BY post.id ASC", links_raw = await conn.fetch(
int(request.match_info['thread_id'])) "SELECT link.link_from, link.link_to FROM link "
posts = {} "LEFT JOIN post ON (link.link_from = post.id) "
backlinks = {} "WHERE post.thread_id = $1",
for row in data: thread_id)
post = posts.get(row['id'], {}) tags = defaultdict(list)
for key, value in row.items(): links = defaultdict(list)
if key == 'tag_name': backlinks = defaultdict(list)
tags = post.get('tags', [])
tags.append(value) for tag_raw in tags_raw:
post['tags'] = tags tag = tags[tag_raw['post_id']]
elif key == 'link_to': tag.append(tag_raw['name'])
link_tos = post.get('link_tos', []) tags[tag_raw['post_id']] = tag
link_tos.append(value)
post['link_tos'] = link_tos for link_raw in links_raw:
elif key == 'link_from': link = links[link_raw['link_from']]
back_post = backlinks.get(row['link_to'], []) link.append(link_raw['link_to'])
back_post.append(value) links[link_raw['link_from']] = link
backlinks[row['link_to']] = back_post
post[key] = value backlink = backlinks[link_raw['link_to']]
post['tags'] = [t for t in post['tags'] if t] backlink.append(link_raw['link_from'])
posts[row['id']] = post backlinks[link_raw['link_to']] = backlink
url_prefix = config.url_prefix
return render_template("thread.html", request, locals()) return render_template("thread.html", request, locals())
@routes.post('/add_tag', name='add_tag')
async def add_tag(request):
"""Adds a tag to a post."""
data = await request.json()
post_id = int(data.get('post_id'))
tag_name = data.get('tag_name')
async with request.app['pool'].acquire() as conn:
await conn.execute(
"INSERT INTO tag (post_id, name) VALUES ($1, $2)",
post_id, tag_name)
return web.json_response({'ok': True})
@routes.post('/remove_tag', name='remove_tag')
async def remove_tag(request):
"""Removes a tag to a post."""
data = await request.json()
post_id = int(data.get('post_id'))
tag_name = data.get('tag_name')
async with request.app['pool'].acquire() as conn:
await conn.execute(
"DELETE FROM tag WHERE post_id = $1 AND name = $2",
post_id, tag_name)
return web.json_response({'ok': True})
async def init_app(): async def init_app():
"""Initializes the application.""" """Initializes the application."""
app = web.Application() app = web.Application()

25
voyage.sql
View File

@ -1,28 +1,33 @@
CREATE TABLE IF NOT EXISTS thread( CREATE TABLE IF NOT EXISTS thread (
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
title TEXT NOT NULL, title TEXT NOT NULL,
time TIMESTAMP WITH TIME ZONE NOT NULL time TIMESTAMP WITH TIME ZONE NOT NULL
); );
CREATE TABLE IF NOT EXISTS post( CREATE TABLE IF NOT EXISTS post (
thread_id INTEGER REFERENCES thread(id) ON DELETE CASCADE NOT NULL, thread_id INTEGER REFERENCES thread(id) ON DELETE CASCADE NOT NULL,
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
name TEXT NOT NULL, name TEXT NOT NULL,
trip_code TEXT, tripcode TEXT,
subject TEXT, subject TEXT,
time TIMESTAMP WITH TIME ZONE NOT NULL, time TIMESTAMP WITH TIME ZONE NOT NULL,
file_url TEXT,
file_name TEXT,
file_md5 TEXT,
body TEXT NOT NULL body TEXT NOT NULL
); );
CREATE TABLE IF NOT EXISTS link( CREATE TABLE IF NOT EXISTS link (
link_from INTEGER REFERENCES post(id) ON DELETE CASCADE NOT NULL, link_from INTEGER REFERENCES post(id) ON DELETE CASCADE NOT NULL,
link_to INTEGER REFERENCES post(id) ON DELETE CASCADE NOT NULL link_to INTEGER REFERENCES post(id) ON DELETE CASCADE NOT NULL,
PRIMARY KEY (link_from, link_to)
); );
CREATE TABLE IF NOT EXISTS tag( CREATE TABLE IF NOT EXISTS tag (
post_id INTEGER REFERENCES post(id) ON DELETE CASCADE NOT NULL, post_id INTEGER REFERENCES post(id) ON DELETE CASCADE NOT NULL,
name TEXT NOT NULL name TEXT NOT NULL,
PRIMARY KEY (post_id, name)
);
CREATE TABLE IF NOT EXISTS file (
file_url TEXT,
file_name TEXT,
file_md5 TEXT
); );