From ac1d9bb6b3aa337c0156293e780d153eedf4f44a Mon Sep 17 00:00:00 2001 From: agatha Date: Fri, 25 Jul 2025 16:11:17 -0400 Subject: [PATCH] initial commit --- .gitignore | 4 + README.md | 98 ++++++++++++++++++++ downloader.py | 232 +++++++++++++++++++++++++++++++++++++++++++++++ patch_roms.sh | 91 +++++++++++++++++++ requirements.txt | 2 + 5 files changed, 427 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 downloader.py create mode 100755 patch_roms.sh create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0e70e19 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +venv/ +download_history.json +patches/ +patched/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..1781374 --- /dev/null +++ b/README.md @@ -0,0 +1,98 @@ +# RomHack Races Downloader and Patcher + +Scripts to download and patch Super Mario World ROM hacks from RomHack Races. + +## Requirements + +### Python Dependencies +- Python 3.6+ +- Required packages in `requirements.txt` + +Install Python dependencies with: +```bash +pip install -r requirements.txt +``` + +### Other Requirements +- [Flips](https://github.com/Alcaro/Flips) - ROM patching utility +- Original Super Mario World ROM (.smc format) + +## Setup + +1. Clone this repository +2. Install Python dependencies +3. Ensure `flips` is installed and available in your PATH +4. Place your original Super Mario World ROM somewhere accessible + +## Usage + +### Downloading Patches + +The `downloader.py` script will download all ROM patches from RomHack Races and organize them by season: + +```bash +python downloader.py +``` + +Patches will be downloaded to the `patches` directory, organized in season folders (Season1, Season2, etc.). + +### Patching ROMs + +The `patch_roms.sh` script applies the downloaded patches to your original ROM: + +```bash +# Show help +./patch_roms.sh + +# Do a dry run (shows what would be patched without actually patching) +./patch_roms.sh --dry-run /path/to/original.smc + +# Actually patch the ROMs +./patch_roms.sh /path/to/original.smc +``` + +Patched ROMs will be created in the `patched` directory with filenames like `Week001.smc`, `Week002.smc`, etc. + +The script will: +- Only patch ROMs that don't already exist in the output directory +- Show a summary of new and skipped patches +- Maintain correct week numbering from the original patches + +## Directory Structure + +``` +. +├── downloader.py # Patch downloader script +├── patch_roms.sh # ROM patcher script +├── requirements.txt # Python dependencies +├── patches/ # Downloaded patches (created by downloader.py) +│ ├── Season1/ +│ ├── Season2/ +│ └── ... +└── patched/ # Patched ROMs (created by patch_roms.sh) + ├── Week001.smc + ├── Week002.smc + └── ... +``` + +## Updating + +To get new patches and ROMs: + +1. Run the downloader: +```bash +python downloader.py +``` + +2. Patch new ROMs: +```bash +./patch_roms.sh /path/to/original.smc +``` + +The scripts will only download new patches and create new ROMs that don't already exist. + +## Notes + +- The original ROM is not included and must be provided separately +- Patches are downloaded from RomHack Races (https://www.romhackraces.com/) +- ROMs are organized by week numbers for easy reference diff --git a/downloader.py b/downloader.py new file mode 100644 index 0000000..0d4d34a --- /dev/null +++ b/downloader.py @@ -0,0 +1,232 @@ +import requests +from bs4 import BeautifulSoup +import os +import re +import json +import time +from urllib.parse import urljoin +from datetime import datetime + +class RomhackRaceScraper: + def __init__(self): + self.base_url = "https://www.romhackraces.com/levels.php" + self.session = requests.Session() + self.rate_limit = 1 + self.last_request = 0 + self.download_history_file = "download_history.json" + self.download_history = self.load_download_history() + self.debug = True + + os.makedirs('patches', exist_ok=True) + + def debug_print(self, message): + if self.debug: + print(f"DEBUG: {message}") + + def load_download_history(self): + try: + with open(self.download_history_file, 'r') as f: + return json.load(f) + except FileNotFoundError: + return { + "last_update": "", + "downloaded_patches": {}, + "last_season_checked": 0 + } + + def save_download_history(self): + with open(self.download_history_file, 'w') as f: + json.dump(self.download_history, f, indent=2) + + def rate_limited_request(self, url): + self.debug_print(f"Making request to: {url}") + current_time = time.time() + time_since_last = current_time - self.last_request + if time_since_last < self.rate_limit: + time.sleep(self.rate_limit - time_since_last) + + response = self.session.get(url) + self.last_request = time.time() + self.debug_print(f"Response status code: {response.status_code}") + return response + + def get_week_number(self, element): + """Extract week number from span element containing number images""" + # First find the span with font-size:18px that contains 'Week' + week_span = element.find('span', style='font-size:18px;') + if not week_span or 'Week' not in week_span.text: + return None + + # Get all number images in this span + number_images = week_span.find_all('img') + if not number_images: + self.debug_print("Found week span but no number images") + return None + + try: + # Extract numbers from image filenames + numbers = [img['src'].split('/')[-1].split('.')[0] for img in number_images] + week_num = int(''.join(numbers)) + self.debug_print(f"Found week number: {week_num}") + return week_num + except (ValueError, KeyError, IndexError) as e: + self.debug_print(f"Error parsing week number: {e}") + return None + + def download_patch(self, url, week_number, season_number): + patch_id = f"s{season_number}_w{week_number}" + + if patch_id in self.download_history["downloaded_patches"]: + self.debug_print(f"Patch {patch_id} already downloaded") + return False + + response = self.rate_limited_request(url) + if response.status_code == 200: + season_dir = os.path.join('patches', f"Season{season_number}") + os.makedirs(season_dir, exist_ok=True) + + filename = f"Week{week_number}.bps" + filepath = os.path.join(season_dir, filename) + + with open(filepath, 'wb') as f: + f.write(response.content) + + self.download_history["downloaded_patches"][patch_id] = { + "filename": filepath, + "downloaded_at": datetime.now().isoformat(), + "url": url + } + self.save_download_history() + + print(f"Downloaded Season {season_number} Week {week_number}") + return True + else: + print(f"Failed to download Season {season_number} Week {week_number} - Status code: {response.status_code}") + return False + + def get_seasons(self): + response = self.rate_limited_request(self.base_url) + soup = BeautifulSoup(response.text, 'html.parser') + + season_div = soup.find('div', class_='info leaders', style=lambda s: s and '300px' in s) + if not season_div: + self.debug_print("Could not find season navigation div") + return [] + + season_links = season_div.find_all('a') + seasons = [] + for link in season_links: + season_num = re.search(r'season=(\d+)', link['href']) + if season_num: + seasons.append(int(season_num.group(1))) + + if 1 not in seasons: + seasons.append(1) + + self.debug_print(f"Found seasons: {seasons}") + return sorted(seasons) + + def test_parse(self): + """Test parsing on the first page""" + response = self.rate_limited_request(self.base_url) + soup = BeautifulSoup(response.text, 'html.parser') + + # Find first patch link + first_patch = soup.find('a', href=lambda href: href and href.endswith('.bps')) + if first_patch: + self.debug_print(f"Test found patch link: {first_patch['href']}") + else: + self.debug_print("Test could not find any patch links") + + # Find first week span + first_week = soup.find('span', style='font-size:18px;') + if first_week: + self.debug_print(f"Test found week span: {first_week.text}") + number_images = first_week.find_all('img') + self.debug_print(f"Number images found: {len(number_images)}") + for img in number_images: + self.debug_print(f"Image source: {img['src']}") + else: + self.debug_print("Test could not find any week spans") + + def scrape_season(self, season_number): + url = f"{self.base_url}?season={season_number}" + print(f"\nScraping Season {season_number}") + + response = self.rate_limited_request(url) + soup = BeautifulSoup(response.text, 'html.parser') + + downloads_this_season = 0 + + # Find all info divs + info_divs = soup.find_all('div', class_='info') + + for info_div in info_divs: + # Check if this div contains a week number + week_num = self.get_week_number(info_div) + if week_num is None: + continue + + self.debug_print(f"Processing Week {week_num}") + + # Look for the patch link in the next table cell + table_cell = info_div.find_next('td', valign='top', align='right') + if table_cell: + patch_link = table_cell.find('a', href=lambda href: href and href.endswith('.bps')) + if patch_link: + self.debug_print(f"Found patch link: {patch_link['href']}") + patch_url = urljoin("https://www.romhackraces.com/", patch_link['href']) + self.debug_print(f"Full patch URL: {patch_url}") + + if self.download_patch(patch_url, week_num, season_number): + downloads_this_season += 1 + else: + self.debug_print(f"No patch link found for Week {week_num}") + else: + self.debug_print(f"No table cell found for Week {week_num}") + + self.debug_print(f"Downloads this season: {downloads_this_season}") + return downloads_this_season + + def scrape_all_seasons(self): + self.test_parse() + seasons = self.get_seasons() + print(f"Found {len(seasons)} seasons to scrape") + + total_downloads = 0 + last_season_checked = self.download_history["last_season_checked"] + + for season in seasons: + if season < last_season_checked: + self.debug_print(f"Skipping Season {season} - already checked") + continue + + downloads = self.scrape_season(season) + total_downloads += downloads + + self.download_history["last_season_checked"] = max( + season, + self.download_history["last_season_checked"] + ) + self.download_history["last_update"] = datetime.now().isoformat() + self.save_download_history() + + print(f"\nDownload session complete. Downloaded {total_downloads} new patches.") + +def main(): + scraper = RomhackRaceScraper() + + # Check if we have existing downloads + if os.path.exists("download_history.json"): + print("Found existing download history") + print(f"Last update: {scraper.download_history['last_update']}") + print(f"Previously downloaded patches: {len(scraper.download_history['downloaded_patches'])}") + print("Checking for new patches...\n") + else: + print("No download history found. Will download all patches.\n") + + scraper.scrape_all_seasons() + +if __name__ == '__main__': + main() + diff --git a/patch_roms.sh b/patch_roms.sh new file mode 100755 index 0000000..540ab96 --- /dev/null +++ b/patch_roms.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +# Function to show usage +show_usage() { + echo "Usage: $0 [-d|--dry-run] " + echo "Options:" + echo " -d, --dry-run Show what would be patched without actually patching" + exit 1 +} + +# Parse command line arguments +DRY_RUN=0 +ORIGINAL_ROM="" + +while [[ $# -gt 0 ]]; do + case $1 in + -d|--dry-run) + DRY_RUN=1 + shift + ;; + *) + ORIGINAL_ROM="$1" + shift + ;; + esac +done + +# Check if original ROM was provided +if [ -z "$ORIGINAL_ROM" ]; then + show_usage +fi + +PATCHES_DIR="./patches" +OUTPUT_DIR="./patched" + +# Check if original ROM exists +if [ ! -f "$ORIGINAL_ROM" ]; then + echo "Error: Original ROM file not found: $ORIGINAL_ROM" + exit 1 +fi + +# Check if flips is installed (skip check in dry run) +if [ $DRY_RUN -eq 0 ]; then + if ! command -v flips &> /dev/null; then + echo "Error: flips is not installed or not in PATH" + exit 1 + fi +fi + +# Create output directory if it doesn't exist (skip in dry run) +if [ $DRY_RUN -eq 0 ]; then + mkdir -p "$OUTPUT_DIR" +fi + +echo "$([ $DRY_RUN -eq 1 ] && echo '[DRY RUN] ')Processing patches..." +echo + +# Find all .bps files and extract week numbers +while IFS= read -r patch_file; do + # Extract week number from filename + if [[ $patch_file =~ Week([0-9]+)\.bps$ ]]; then + week_num="${BASH_REMATCH[1]}" + # Create zero-padded week number + padded_week=$(printf "Week%03d.smc" "$week_num") + output_file="$OUTPUT_DIR/$padded_week" + + if [ $DRY_RUN -eq 1 ]; then + echo "[DRY RUN] Would patch: $patch_file" + echo "[DRY RUN] Would create: $output_file" + echo + else + echo "Patching: $patch_file -> $padded_week" + + # Apply patch using flips + flips -a "$patch_file" "$ORIGINAL_ROM" "$output_file" + + # Check if patching was successful + if [ $? -eq 0 ]; then + echo "Successfully created $padded_week" + else + echo "Error patching $patch_file" + fi + echo + fi + else + echo "Warning: Could not extract week number from filename: $patch_file" + fi +done < <(find "$PATCHES_DIR" -type f -name "Week*.bps" | sort -V) + +total_files=$(find "$PATCHES_DIR" -type f -name "Week*.bps" | wc -l) +echo "$([ $DRY_RUN -eq 1 ] && echo '[DRY RUN] ')Would create $total_files patched ROMs in $OUTPUT_DIR" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1190bd8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +requests +beautifulsoup4