first commit

.gitignore

@@ -0,0 +1,6 @@
+__pycache__/
+*.swp
+*.swo
+config_master.py
+config_server.py
+config_slave.py

README.md

@@ -0,0 +1,15 @@
+# Mine  
+It manages Minecraft modpacks.  
+
+## Requirements  
+Python 3.6+  
+Python packages: `requests bs4`  
+
+## Usage  
+`$ python3 mine.py action [options] [target]`  
+__actions__  
+* `add` Add a new mod to the modpack. `target` must be a url to a curseforge mod project page. By default, it will prioritize release phases as Release, Beta and then Alpha. `--phase` can alter this. Will always grab the latest version of whichever phase it targets.  
+* `add_all` Add a list of mods to the modpack. `target` must be a file containing a newline-delineated list of curseforge mod project page urls. Will prioritize release phases similar to `add`, however `--phase` will not affect it.  
+* `update` Update a specific mod in the modpack. `target` must be the url associated with the mod. By default, it will prioritize the release phase previously added. `--phase` can alter this. Will only update if the targeted mod is more recent than the one previously downloaded.  
+* `update_all` Update every mod in the modpack. No `target` needed. Will prioritize release phases similar to `update`, however `--phase` will will not affect it.  
+* `summary` Generate a summary page as html. Summary page displays a list of all mods in the modpack and a change log that is updated as changes are made to the modpack.  

config_master.py.template

@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+"""
+Configuration settings for overwrought_master.py.
+`mc_ver` is used literally when searching for an appropriate verison of
+a mod to download. Should not include the minor version.
+`modpack_name` is used for vanity purposes when naming the modpack.
+"""
+mc_ver = "1.12"
+modpack_name = "motherlode1"

overwrought_master.py

@@ -0,0 +1,286 @@
+#!/usr/bin/env python3
+"""
+A script for updating mods in a minecraft modpack.
+"""
+import os
+import re
+import json
+import sqlite3
+from datetime import datetime
+
+import bs4
+import requests
+
+import config_master
+
+_con = sqlite3.connect("modpack.db", detect_types=sqlite3.PARSE_DECLTYPES)
+_cur = _con.cursor()
+
+class Mod:
+	"""Represents a mod."""
+	title = ""
+	url = ""
+	release_phase = ""
+	dl_link = ""
+	filename = ""
+	upload_date = ""
+
+	def __init__(self, data=()):
+		if data:
+			self.from_tup(data)
+
+	def from_tup(self, data):
+		"""
+		Updates the mod's information with that retrieved from the database.
+		"""
+		self.title = data[0]
+		self.url = data[1]
+		self.release_phase = data[2]
+		self.filename = data[3]
+		self.upload_date = data[4]
+	
+	def save(self):
+		"""Inserts the mod into the `mod` table of the database."""
+		_cur.execute(
+			"INSERT OR REPLACE INTO mod VALUES (?, ?, ?, ?, ?)",
+			(
+				self.title,
+				self.url,
+				self.release_phase,
+				self.filename,
+				self.upload_date
+			),
+		)
+		_con.commit()
+
+
+def init_db():
+	"""
+	Initializes the database.
+	"""
+	try:
+		_cur.execute("SELECT * FROM `mod`").fetchone()
+		_cur.execute("SELECT * FROM `change_item`").fetchone()
+	except sqlite3.OperationalError:
+		_cur.execute(
+			"CREATE TABLE mod("
+			+ "title TEXT PRIMARY KEY,"
+			+ "url TEXT,"
+			+ "release_phase TEXT,"
+			+ "filename TEXT,"
+			+ "upload_date TIMESTAMP"
+			+ ")"
+		)
+		_cur.execute(
+			"CREATE TABLE change_item("
+			+ "date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,"
+			+ "text TEXT"
+			+ ")"
+		)
+		_con.commit()
+
+
+def scrape_curse_forge(url, phase=None):
+	"""
+	Scrapes all relevent info from the provided mod page and returns a
+	mod object.
+	"""
+	print("Scraping:", url)
+	res = requests.get(url, verify=True)
+	res.raise_for_status()
+	soup = bs4.BeautifulSoup(res.text, "html.parser")
+	mod = Mod()
+
+	bar = soup.find("h3", text="Recent Files").parent.parent
+	bar = bar.find("div", class_="cf-sidebar-inner")
+	hs = bar.find_all("h4")
+	h = [h for h in hs if h.text.strip() ==f"Minecraft {config_master.mc_ver}"]
+	if not h:
+		print("No valid files for this version found.")
+		return False
+	h = h[0]
+	files = bar.contents[bar.contents.index(h)+2].find_all("li")
+	if phase:
+		for li in files:
+			if li.div.div.get("title") == phase:
+				break
+		else:
+			print("No valid files for this release phase found.")
+			return False
+	else:
+		li = files[0]
+
+	mod.title = re.search(r"-(.*?)- Mods", soup.title.text).group(1).strip()
+	mod.url = url
+	mod.release_phase = li.div.div.get("title")
+	mod.dl_link = "https://minecraft.curseforge.com"
+	mod.dl_link += li.contents[3].a.get("href")
+	mod.filename = li.contents[3].contents[3].a.get("data-name")
+	mod.upload_date = datetime.utcfromtimestamp(int(li.abbr.get("data-epoch")))
+	return mod
+
+
+def download_mod(mod):
+	"""Downloads the mod."""
+	print("Downloading:", mod.filename)
+	res = requests.get(mod.dl_link)
+	res.raise_for_status()
+	fname = res.headers.get('content-disposition')
+	if fname:
+		mod.filename = re.search(r'filename="(.+)"', fname).group(1)
+	else:
+		mod.filename = res.url.split('/')[-1]
+	with open(os.path.join("mods", mod.filename), 'wb') as file:
+		for chunk in res.iter_content(100000):
+			file.write(chunk)
+
+
+def get_mod(target):
+	"""Gets the specified mod in the modpack."""
+	data = _cur.execute(
+		"SELECT * FROM mod WHERE url = ?",
+		(target,)
+	).fetchone()
+	mod = Mod(data)
+	return mod
+
+
+def get_all_mods():
+	"""Retrieves all mods from the database."""
+	data = _cur.execute("SELECT * FROM mod ORDER BY title ASC").fetchall()
+	mods = []
+	for line in data:
+		mods.append(Mod(line))
+	return mods
+
+
+def log_change(text):
+	"""Logs the provided message as a change under today's date."""
+	_cur.execute(
+		"INSERT INTO change_item(text) VALUES(?)", (text,)
+	)
+	_con.commit()
+
+
+def get_all_changes():
+	"""Retrieves all change log items."""
+	data = _cur.execute(
+		"SELECT date(date), text FROM change_item ORDER BY date DESC"
+	).fetchall()
+	dates = [tup[0] for tup in data]
+	data = {date: [tup[1] for tup in data if tup[0] == date] for date in dates}
+	return data
+
+
+def add_mod(target, phase=None):
+	"""Adds the specified mod to the modpack."""
+	mod = scrape_curse_forge(target, phase)
+	download_mod(mod)
+	mod.save()
+	log_change(f"Added {mod.title}")
+
+
+def add_mod_all(target):
+	"""Reads a list of URLs from a textfile and adds them to the modpack."""
+	with open(target, 'r') as file:
+		urls = file.read().splitlines()
+	for url in urls:
+		mod = scrape_curse_forge(url)
+		download_mod(mod)
+		mod.save()
+	log_change(f"Added mods in bulk from {os.path.basename(target)}")
+
+
+def update_mod(target, delete=False, phase=None):
+	"""Updates the specified mod."""
+	mod_current = get_mod(target)
+	if not phase:
+		phase = mod_current.release_phase
+	mod_latest = scrape_curse_forge(mod_current.url, phase)
+	if mod_latest.upload_date > mod_current.upload_date:
+		print(f"Updating {mod_latest.title} to {mod_latest.filename}")
+		download_mod(mod_latest)
+		mod_latest.save()
+		log_change(f"Updated {mod_latest.title} to {mod_latest.filename}")
+		if delete:
+			os.remove(os.path.join("mods", mod_current.filename))
+
+
+def update_mod_all(delete=False):
+	"""Updates all mods in the pack."""
+	for mod in get_all_mods():
+		update_mod(mod.url, delete)
+
+
+def generate_summary():
+	"""Generates the summary HTML file."""
+	mods = get_all_mods()
+	changes = get_all_changes()
+	with open("template.html", 'r') as file:
+		html = file.read()
+
+	change_log = ""
+	for date, items in changes.items():
+		change_log += f"<i>{date}</i><ul>"
+		for item in items:
+			change_log += f"<li>{item}</li>"
+		change_log += "</ul>"
+
+	mod_table = ""
+	for mod in mods:
+		mod_table += f'<tr><td><a href="{mod.url}">{mod.title}</a></td>'
+		mod_table += f'<td>{mod.filename}</td>'
+		mod_table += f'<td>{mod.release_phase[0]}</td>'
+		mod_table += f'<td>{mod.upload_date.strftime("%Y-%m-%d")}</td></tr>'
+
+	html = html.replace("$$MODPACK$$", config_master.modpack_name)
+	html = html.replace("$$CHANGELOG$$", change_log)
+	html = html.replace("$$MODTABLE$$", mod_table)
+
+	with open(config_master.modpack_name + ".html", "w") as file:
+		file.write(html)
+
+
+if __name__ == "__main__":
+	import argparse
+
+	parser = argparse.ArgumentParser(
+		description="A series of tools and functions for managing a" \
+			+ "Minecraft modpack.",
+	)
+	parser.add_argument(
+		"action",
+		choices=["add", "add_all", "update", "update_all", "summary"],
+		help="What action to perform.",
+	)
+	parser.add_argument(
+		"target",
+		nargs='?',
+		help="The mod, mod url, or file containing a list of urls to " \
+			+ "work with.",
+	)
+	parser.add_argument(
+		"-p",
+		"--phase",
+		choices=["Release", "Beta", "Alpha"],
+		help="Which release phase to look for."
+	)
+	parser.add_argument(
+		"--delete",
+		action="store_true",
+		help="Delete the old mod file after downloading the new.",
+	)
+	args = parser.parse_args()
+	os.makedirs("mods", exist_ok=True)
+	init_db()
+
+	if args.action == "add":
+		add_mod(args.target, args.phase)
+	elif args.action == "add_all":
+		add_mod_all(args.target)
+	elif args.action == "update":
+		update_mod(args.target, args.delete, args.phase)
+	elif args.action == "update_all":
+		update_mod_all(args.delete)
+	elif args.action == "summary":
+		generate_summary()

template.html

@@ -0,0 +1,39 @@
+<!DOCTYPE html>
+<html>
+<head>
+	<meta charset="utf-8">
+	<title>$$MODPACK$$ Modlist</title>
+	<style>
+		table {
+			border-collapse: collapse;
+			border: 1px solid #ddd;
+		}
+		th, td {
+			border: 1px solid #ddd;
+			padding-right: 0.5em;
+			padding-left: 0.5em;
+		}
+		td:nth-child(3), td:nth-child(4) {
+			text-align: center;
+		}
+	</style>
+</head>
+<body>
+	<h3>Changelog:</h3>
+	$$CHANGELOG$$
+	<br>
+	<table>
+		<thead>
+			<tr>
+				<th>Mod</th>
+				<th>File</th>
+				<th>Release</th>
+				<th>Release Date</th>
+			</tr>
+		</thead>
+		<tbody>
+			$$MODTABLE$$
+		</tbody>
+	</table>
+</body>
+</html>