diff --git a/database.py b/database.py index c69ea3c..8e0f618 100644 --- a/database.py +++ b/database.py @@ -19,12 +19,10 @@ _r = redis.Redis( def scrape_feed(feed_url): """Scrapes an RSS feed and extract all relevant data from it.""" - try: - res = requests.get(feed_url) - res.raise_for_status() - except requests.exceptions.RequestException as e: - # TODO: log the error - return + headers = {"User-Agent": "Feed me RSS"} + res = requests.get(feed_url, headers=headers, timeout=10) + res.raise_for_status() + encoding = re.search(r'encoding="(.*)"', res.text) if encoding: encoding = encoding.group(1) @@ -113,14 +111,23 @@ def scrape_factorio(soup): def update_feed(feed_url): """Updates the given feed_id.""" - feed = scrape_feed(feed_url) + try: + feed = scrape_feed(feed_url) + except Exception as e: + feed = { + 'meta': { + 'title': feed_url, + 'html_url': feed_url, + 'description': "" + }, + 'error': str(e), + } _r.set(feed_url, json.dumps(feed)) def update_all_feeds(): """Updates all feeds being watched.""" for feed_url in config.FEEDS: - #print(feed_url) update_feed(feed_url) diff --git a/templates/index.html b/templates/index.html index 7254ccc..4a4b3b4 100644 --- a/templates/index.html +++ b/templates/index.html @@ -21,6 +21,9 @@ {% for feed in feeds %}
Error loading feed: {{ feed['error'] }}
+ {% endif %}
{% for entry in feed['entries'] %}
{{ entry['date'] }}{{ entry['title'] }}