From 155518b78701671031bd70701c1f09fe991f1948 Mon Sep 17 00:00:00 2001 From: iou1name Date: Sun, 2 Jun 2019 15:27:13 -0400 Subject: [PATCH] first commit --- .gitignore | 6 + README.md | 15 ++ config_master.py.template | 9 ++ overwrought_master.py | 286 ++++++++++++++++++++++++++++++++++++++ template.html | 39 ++++++ 5 files changed, 355 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 config_master.py.template create mode 100644 overwrought_master.py create mode 100644 template.html diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f3897c6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +*.swp +*.swo +config_master.py +config_server.py +config_slave.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..1836374 --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ +# Mine +It manages Minecraft modpacks. + +## Requirements +Python 3.6+ +Python packages: `requests bs4` + +## Usage +`$ python3 mine.py action [options] [target]` +__actions__ +* `add` Add a new mod to the modpack. `target` must be a url to a curseforge mod project page. By default, it will prioritize release phases as Release, Beta and then Alpha. `--phase` can alter this. Will always grab the latest version of whichever phase it targets. +* `add_all` Add a list of mods to the modpack. `target` must be a file containing a newline-delineated list of curseforge mod project page urls. Will prioritize release phases similar to `add`, however `--phase` will not affect it. +* `update` Update a specific mod in the modpack. `target` must be the url associated with the mod. By default, it will prioritize the release phase previously added. `--phase` can alter this. Will only update if the targeted mod is more recent than the one previously downloaded. +* `update_all` Update every mod in the modpack. No `target` needed. Will prioritize release phases similar to `update`, however `--phase` will will not affect it. +* `summary` Generate a summary page as html. Summary page displays a list of all mods in the modpack and a change log that is updated as changes are made to the modpack. diff --git a/config_master.py.template b/config_master.py.template new file mode 100644 index 0000000..ac01c5c --- /dev/null +++ b/config_master.py.template @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +""" +Configuration settings for overwrought_master.py. +`mc_ver` is used literally when searching for an appropriate verison of +a mod to download. Should not include the minor version. +`modpack_name` is used for vanity purposes when naming the modpack. +""" +mc_ver = "1.12" +modpack_name = "motherlode1" diff --git a/overwrought_master.py b/overwrought_master.py new file mode 100644 index 0000000..7067860 --- /dev/null +++ b/overwrought_master.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +""" +A script for updating mods in a minecraft modpack. +""" +import os +import re +import json +import sqlite3 +from datetime import datetime + +import bs4 +import requests + +import config_master + +_con = sqlite3.connect("modpack.db", detect_types=sqlite3.PARSE_DECLTYPES) +_cur = _con.cursor() + +class Mod: + """Represents a mod.""" + title = "" + url = "" + release_phase = "" + dl_link = "" + filename = "" + upload_date = "" + + def __init__(self, data=()): + if data: + self.from_tup(data) + + def from_tup(self, data): + """ + Updates the mod's information with that retrieved from the database. + """ + self.title = data[0] + self.url = data[1] + self.release_phase = data[2] + self.filename = data[3] + self.upload_date = data[4] + + def save(self): + """Inserts the mod into the `mod` table of the database.""" + _cur.execute( + "INSERT OR REPLACE INTO mod VALUES (?, ?, ?, ?, ?)", + ( + self.title, + self.url, + self.release_phase, + self.filename, + self.upload_date + ), + ) + _con.commit() + + +def init_db(): + """ + Initializes the database. + """ + try: + _cur.execute("SELECT * FROM `mod`").fetchone() + _cur.execute("SELECT * FROM `change_item`").fetchone() + except sqlite3.OperationalError: + _cur.execute( + "CREATE TABLE mod(" + + "title TEXT PRIMARY KEY," + + "url TEXT," + + "release_phase TEXT," + + "filename TEXT," + + "upload_date TIMESTAMP" + + ")" + ) + _cur.execute( + "CREATE TABLE change_item(" + + "date TIMESTAMP DEFAULT CURRENT_TIMESTAMP," + + "text TEXT" + + ")" + ) + _con.commit() + + +def scrape_curse_forge(url, phase=None): + """ + Scrapes all relevent info from the provided mod page and returns a + mod object. + """ + print("Scraping:", url) + res = requests.get(url, verify=True) + res.raise_for_status() + soup = bs4.BeautifulSoup(res.text, "html.parser") + mod = Mod() + + bar = soup.find("h3", text="Recent Files").parent.parent + bar = bar.find("div", class_="cf-sidebar-inner") + hs = bar.find_all("h4") + h = [h for h in hs if h.text.strip() ==f"Minecraft {config_master.mc_ver}"] + if not h: + print("No valid files for this version found.") + return False + h = h[0] + files = bar.contents[bar.contents.index(h)+2].find_all("li") + if phase: + for li in files: + if li.div.div.get("title") == phase: + break + else: + print("No valid files for this release phase found.") + return False + else: + li = files[0] + + mod.title = re.search(r"-(.*?)- Mods", soup.title.text).group(1).strip() + mod.url = url + mod.release_phase = li.div.div.get("title") + mod.dl_link = "https://minecraft.curseforge.com" + mod.dl_link += li.contents[3].a.get("href") + mod.filename = li.contents[3].contents[3].a.get("data-name") + mod.upload_date = datetime.utcfromtimestamp(int(li.abbr.get("data-epoch"))) + return mod + + +def download_mod(mod): + """Downloads the mod.""" + print("Downloading:", mod.filename) + res = requests.get(mod.dl_link) + res.raise_for_status() + fname = res.headers.get('content-disposition') + if fname: + mod.filename = re.search(r'filename="(.+)"', fname).group(1) + else: + mod.filename = res.url.split('/')[-1] + with open(os.path.join("mods", mod.filename), 'wb') as file: + for chunk in res.iter_content(100000): + file.write(chunk) + + +def get_mod(target): + """Gets the specified mod in the modpack.""" + data = _cur.execute( + "SELECT * FROM mod WHERE url = ?", + (target,) + ).fetchone() + mod = Mod(data) + return mod + + +def get_all_mods(): + """Retrieves all mods from the database.""" + data = _cur.execute("SELECT * FROM mod ORDER BY title ASC").fetchall() + mods = [] + for line in data: + mods.append(Mod(line)) + return mods + + +def log_change(text): + """Logs the provided message as a change under today's date.""" + _cur.execute( + "INSERT INTO change_item(text) VALUES(?)", (text,) + ) + _con.commit() + + +def get_all_changes(): + """Retrieves all change log items.""" + data = _cur.execute( + "SELECT date(date), text FROM change_item ORDER BY date DESC" + ).fetchall() + dates = [tup[0] for tup in data] + data = {date: [tup[1] for tup in data if tup[0] == date] for date in dates} + return data + + +def add_mod(target, phase=None): + """Adds the specified mod to the modpack.""" + mod = scrape_curse_forge(target, phase) + download_mod(mod) + mod.save() + log_change(f"Added {mod.title}") + + +def add_mod_all(target): + """Reads a list of URLs from a textfile and adds them to the modpack.""" + with open(target, 'r') as file: + urls = file.read().splitlines() + for url in urls: + mod = scrape_curse_forge(url) + download_mod(mod) + mod.save() + log_change(f"Added mods in bulk from {os.path.basename(target)}") + + +def update_mod(target, delete=False, phase=None): + """Updates the specified mod.""" + mod_current = get_mod(target) + if not phase: + phase = mod_current.release_phase + mod_latest = scrape_curse_forge(mod_current.url, phase) + if mod_latest.upload_date > mod_current.upload_date: + print(f"Updating {mod_latest.title} to {mod_latest.filename}") + download_mod(mod_latest) + mod_latest.save() + log_change(f"Updated {mod_latest.title} to {mod_latest.filename}") + if delete: + os.remove(os.path.join("mods", mod_current.filename)) + + +def update_mod_all(delete=False): + """Updates all mods in the pack.""" + for mod in get_all_mods(): + update_mod(mod.url, delete) + + +def generate_summary(): + """Generates the summary HTML file.""" + mods = get_all_mods() + changes = get_all_changes() + with open("template.html", 'r') as file: + html = file.read() + + change_log = "" + for date, items in changes.items(): + change_log += f"{date}" + + mod_table = "" + for mod in mods: + mod_table += f'{mod.title}' + mod_table += f'{mod.filename}' + mod_table += f'{mod.release_phase[0]}' + mod_table += f'{mod.upload_date.strftime("%Y-%m-%d")}' + + html = html.replace("$$MODPACK$$", config_master.modpack_name) + html = html.replace("$$CHANGELOG$$", change_log) + html = html.replace("$$MODTABLE$$", mod_table) + + with open(config_master.modpack_name + ".html", "w") as file: + file.write(html) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + description="A series of tools and functions for managing a" \ + + "Minecraft modpack.", + ) + parser.add_argument( + "action", + choices=["add", "add_all", "update", "update_all", "summary"], + help="What action to perform.", + ) + parser.add_argument( + "target", + nargs='?', + help="The mod, mod url, or file containing a list of urls to " \ + + "work with.", + ) + parser.add_argument( + "-p", + "--phase", + choices=["Release", "Beta", "Alpha"], + help="Which release phase to look for." + ) + parser.add_argument( + "--delete", + action="store_true", + help="Delete the old mod file after downloading the new.", + ) + args = parser.parse_args() + os.makedirs("mods", exist_ok=True) + init_db() + + if args.action == "add": + add_mod(args.target, args.phase) + elif args.action == "add_all": + add_mod_all(args.target) + elif args.action == "update": + update_mod(args.target, args.delete, args.phase) + elif args.action == "update_all": + update_mod_all(args.delete) + elif args.action == "summary": + generate_summary() diff --git a/template.html b/template.html new file mode 100644 index 0000000..0023ace --- /dev/null +++ b/template.html @@ -0,0 +1,39 @@ + + + + + $$MODPACK$$ Modlist + + + +

Changelog:

+ $$CHANGELOG$$ +
+ + + + + + + + + + + $$MODTABLE$$ + +
ModFileReleaseRelease Date
+ +