iou1name/Voyage
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

third commit

Browse Source
This commit is contained in:
iou1name 2019-12-11 19:47:19 -05:00
parent 3beb5af2c9
commit 7f025abd5b
7 changed files with 156 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
filters.py
View File

@ -4,13 +4,13 @@ Jinja2 template filters.
"""
import re
from markupsafe import Markup
from markupsafe import Markup, escape
def quotelink(line, post):
def quotelink(line, post_links):
"""Checks if the quotelink is valid and adds an anchor tag if so."""
links = re.findall(r'>>(\d+)', line)
for link in links:
if int(link) in post['link_tos']:
if int(link) in post_links:
span = f'<a class="quotelink" href="#{link}">>>{link}</a>'
else:
span = f'<span class="deadlink">>>{link}</span>'

17
scrape_quest.py
View File

@ -51,9 +51,9 @@ def scrape_posts(root_dir):
# information gathering
post_id = int(post.get('id')[2:])
name = post.find(class_='name').text
trip_code = post.find(class_='postertrip')
if trip_code:
trip_code = trip_code.text
tripcode = post.find(class_='postertrip')
if tripcode:
tripcode = tripcode.text
subject = post.find(class_='subject')
if subject:
subject = subject.text
@ -73,6 +73,7 @@ def scrape_posts(root_dir):
links = post.find_all(class_='quotelink')
links = [l for l in links if l.get('href').startswith('#')]
links = [int(link.text[2:]) for link in links]
links = list(set(links))
# heuristics
tags = []
@ -110,9 +111,9 @@ def scrape_posts(root_dir):
# database insert
cur.execute(
"INSERT INTO post VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
(thread_id, post_id, name, trip_code, subject,
post_time, file_url, file_name, file_md5, post_body)
"INSERT INTO post VALUES (%s,%s,%s,%s,%s,%s,%s)",
(thread_id, post_id, name, tripcode, subject,
post_time, post_body)
)
for link in links:
cur.execute("INSERT INTO link VALUES (%s,%s)",
@ -122,6 +123,10 @@ def scrape_posts(root_dir):
cur.execute("INSERT INTO tag VALUES (%s,%s)",
(post_id, tag)
)
if file_text:
cur.execute("INSERT INTO file VALUES (%s,%s,%s)",
(file_url, file_name, file_md5)
)
if __name__ == '__main__':

8
static/voyage.css
View File

@ -10,6 +10,14 @@ body {
padding: 0.5em;
}
.faded {
opacity: 0.33;
}
.tag {
font-size: 0.8em;
}
.header {
margin-bottom: 0.5em;
}

45
static/voyage.js
View File

@ -0,0 +1,45 @@
function add_tag(input) {
post_id = input.closest('.post_container').id;
tag_name = input.children[0].value;
fetch(url_prefix + '/add_tag', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({
'post_id': post_id,
'tag_name': tag_name,
})
}).then(function(response) {
if(!response.ok) { throw new Error('Error adding tag!'); }
return response.json();
}).then(function(json) {
if (!json.ok) { throw new Error('Could not add tag.') }
let tag = document.createElement('span');
tag.innerText = tag_name;
tag.className = 'tag';
let anchor = document.createElement('a');
anchor.href = "javascript:void(0)";
anchor.setAttribute('onlick', "remove_tag(this.closest('.tag'))");
anchor.innerText = '-';
tag.appendChild(anchor);
input.closest('.tags').insertBefore(tag, input);
});
}
function remove_tag(tag) {
post_id = tag.closest('.post_container').id;
tag_name = tag.innerText.replace(/ .*/, '');
fetch(url_prefix + '/remove_tag', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({
'post_id': post_id,
'tag_name': tag_name,
})
}).then(function(response) {
if(!response.ok) { throw new Error('Error removing tag!'); }
return response.json();
}).then(function(json) {
if (!json.ok) { throw new Error('Could not remove tag.') }
tag.remove();
});
}

19
templates/thread.html
View File

@ -3,6 +3,7 @@
<head>
<title>Voyage</title>
<link rel="stylesheet" type="text/css" href="/static/voyage.css">
<script>const url_prefix = "{{ url_prefix }}";</script>
<script type="text/javascript" src="/static/voyage.js"></script>
<meta name="viewport" content="width=device-width, initial-scale=0.8">
<meta name="description" content="A quest archive viewer.">
@ -12,15 +13,21 @@
<h1>Voyage</h1>
</header>
<main>
{% for post_id, post in posts.items() %}
<div id="{{ post.id }}" class="post_container {{ ' '.join(post.tags) }}">
{% for post in posts %}
<div id="{{ post.id }}" class="post_container{% if tags[post.id] %} {{ ' '.join(tags[post.id]) }}{% endif %}{% if 'story_post' not in tags[post.id] %} faded{% endif %}">
<div class="tags">
{% for tag in tags.get(post.id, []) %}
<span class="tag">{{ tag }} <a href="javascript:void(0)" onclick="remove_tag(this.closest('.tag'))">-</a></span>
{% endfor %}
<span class="tag"><input type="text"><a href="javascript:void(0)" onclick="add_tag(this.closest('.tag'))">+</a></span>
</div>
<div class="header">
{% if post.subject %}
<span class="subject">{{ post.subject }}</span>
{% endif %}
<span class="name">{{ post.name }}</span>
{% if post.trip_code %}
<span class="tripcode">{{ post.trip_code }}</span>
{% if post.tripcode %}
<span class="tripcode">{{ post.tripcode }}</span>
{% endif %}
<span class="time">{{ post.time.strftime('%Y-%m-%d %H:%M') }}</span>
<span class="id">No.{{ post.id }}</span>
@ -35,9 +42,9 @@
<div class="body">
{% for line in post.body.splitlines() %}
{% if line.startswith('>') %}
<span class="quote">{{ line|quotelink(post) }}</span><br>
<span class="quote">{{ line|quotelink(links[post.id]) }}</span><br>
{% else %}
{{ line|quotelink(post) }}<br>
{{ line|quotelink(links[post.id]) }}<br>
{% endif %}
{% endfor %}
</div>

91
voyage.py
View File

@ -2,6 +2,8 @@
"""
A display interface for archived 4chan quest threads.
"""
from collections import defaultdict
from aiohttp import web
import jinja2
import aiohttp_jinja2
@ -26,41 +28,70 @@ async def index(request):
@routes.get(r'/{thread_id:\d+}', name='thread')
async def thread(request):
"""A single thread page."""
thread_id = int(request.match_info['thread_id'])
async with request.app['pool'].acquire() as conn:
data = await conn.fetch(
"SELECT post.id, post.name, post.trip_code, post.subject, "
"post.time, post.body, "
"tag.name AS tag_name, link.link_to, link.link_from "
"FROM post "
"LEFT JOIN tag ON (post.id = tag.post_id) "
"LEFT JOIN link ON (post.id = link.link_from) "
"WHERE post.thread_id = $1 "
"ORDER BY post.id ASC",
int(request.match_info['thread_id']))
posts = {}
backlinks = {}
for row in data:
post = posts.get(row['id'], {})
for key, value in row.items():
if key == 'tag_name':
tags = post.get('tags', [])
tags.append(value)
post['tags'] = tags
elif key == 'link_to':
link_tos = post.get('link_tos', [])
link_tos.append(value)
post['link_tos'] = link_tos
elif key == 'link_from':
back_post = backlinks.get(row['link_to'], [])
back_post.append(value)
backlinks[row['link_to']] = back_post
post[key] = value
post['tags'] = [t for t in post['tags'] if t]
posts[row['id']] = post
posts = await conn.fetch(
"SELECT * FROM post WHERE thread_id = $1 ORDER BY id ASC",
thread_id)
tags_raw = await conn.fetch(
"SELECT tag.post_id, tag.name FROM tag "
"LEFT JOIN post ON (tag.post_id = post.id) "
"WHERE post.thread_id = $1",
thread_id)
links_raw = await conn.fetch(
"SELECT link.link_from, link.link_to FROM link "
"LEFT JOIN post ON (link.link_from = post.id) "
"WHERE post.thread_id = $1",
thread_id)
tags = defaultdict(list)
links = defaultdict(list)
backlinks = defaultdict(list)
for tag_raw in tags_raw:
tag = tags[tag_raw['post_id']]
tag.append(tag_raw['name'])
tags[tag_raw['post_id']] = tag
for link_raw in links_raw:
link = links[link_raw['link_from']]
link.append(link_raw['link_to'])
links[link_raw['link_from']] = link
backlink = backlinks[link_raw['link_to']]
backlink.append(link_raw['link_from'])
backlinks[link_raw['link_to']] = backlink
url_prefix = config.url_prefix
return render_template("thread.html", request, locals())
@routes.post('/add_tag', name='add_tag')
async def add_tag(request):
"""Adds a tag to a post."""
data = await request.json()
post_id = int(data.get('post_id'))
tag_name = data.get('tag_name')
async with request.app['pool'].acquire() as conn:
await conn.execute(
"INSERT INTO tag (post_id, name) VALUES ($1, $2)",
post_id, tag_name)
return web.json_response({'ok': True})
@routes.post('/remove_tag', name='remove_tag')
async def remove_tag(request):
"""Removes a tag to a post."""
data = await request.json()
post_id = int(data.get('post_id'))
tag_name = data.get('tag_name')
async with request.app['pool'].acquire() as conn:
await conn.execute(
"DELETE FROM tag WHERE post_id = $1 AND name = $2",
post_id, tag_name)
return web.json_response({'ok': True})
async def init_app():
"""Initializes the application."""
app = web.Application()

25
voyage.sql
View File

@ -1,28 +1,33 @@
CREATE TABLE IF NOT EXISTS thread(
CREATE TABLE IF NOT EXISTS thread (
id INTEGER PRIMARY KEY,
title TEXT NOT NULL,
time TIMESTAMP WITH TIME ZONE NOT NULL
);
CREATE TABLE IF NOT EXISTS post(
CREATE TABLE IF NOT EXISTS post (
thread_id INTEGER REFERENCES thread(id) ON DELETE CASCADE NOT NULL,
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
trip_code TEXT,
tripcode TEXT,
subject TEXT,
time TIMESTAMP WITH TIME ZONE NOT NULL,
file_url TEXT,
file_name TEXT,
file_md5 TEXT,
body TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS link(
CREATE TABLE IF NOT EXISTS link (
link_from INTEGER REFERENCES post(id) ON DELETE CASCADE NOT NULL,
link_to INTEGER REFERENCES post(id) ON DELETE CASCADE NOT NULL
link_to INTEGER REFERENCES post(id) ON DELETE CASCADE NOT NULL,
PRIMARY KEY (link_from, link_to)
);
CREATE TABLE IF NOT EXISTS tag(
CREATE TABLE IF NOT EXISTS tag (
post_id INTEGER REFERENCES post(id) ON DELETE CASCADE NOT NULL,
name TEXT NOT NULL
name TEXT NOT NULL,
PRIMARY KEY (post_id, name)
);
CREATE TABLE IF NOT EXISTS file (
file_url TEXT,
file_name TEXT,
file_md5 TEXT
);