diff --git a/archive-rom/.idea/.gitignore b/archive-rom/.idea/.gitignore new file mode 100644 index 0000000..1c2fda5 --- /dev/null +++ b/archive-rom/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/archive-rom/.idea/archive-rom.iml b/archive-rom/.idea/archive-rom.iml new file mode 100644 index 0000000..6828f76 --- /dev/null +++ b/archive-rom/.idea/archive-rom.iml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/archive-rom/.idea/inspectionProfiles/Project_Default.xml b/archive-rom/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..9277599 --- /dev/null +++ b/archive-rom/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,12 @@ + + + + \ No newline at end of file diff --git a/archive-rom/.idea/inspectionProfiles/profiles_settings.xml b/archive-rom/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/archive-rom/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/archive-rom/.idea/misc.xml b/archive-rom/.idea/misc.xml new file mode 100644 index 0000000..2cfde97 --- /dev/null +++ b/archive-rom/.idea/misc.xml @@ -0,0 +1,11 @@ + + + + + + + + \ No newline at end of file diff --git a/archive-rom/.idea/modules.xml b/archive-rom/.idea/modules.xml new file mode 100644 index 0000000..45be94b --- /dev/null +++ b/archive-rom/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/archive-rom/download-urls.py b/archive-rom/download-urls.py new file mode 100644 index 0000000..7ce2c63 --- /dev/null +++ b/archive-rom/download-urls.py @@ -0,0 +1,60 @@ +import os +import sys +import requests +import humanize +from urllib.parse import unquote + + +def parse_file(file_name): + urls = [] + with open(file_name, 'r') as f: + for line in f: + urls.append(unquote(line.strip())) + return urls + + +def get_filename(url): + return url.split('/')[-1] + + +def print_progress(amount_written, file_size, sizes): + for size in sizes: + if amount_written >= size: + sizes.remove(size) + percentage = (10 - len(sizes)) * 10 + print(f"{percentage}% Written {humanize.naturalsize(amount_written)} / {humanize.naturalsize(file_size)}") + return sizes + + +def download_url(url, filename): + # Constant + temp_file = filename + ".tmp" + chunk_size = 1024 * 1024 * 8 # 8MB + # Request + r = requests.get(url, stream=True) + # More constants + file_size = int(r.headers.get('Content-Length')) + sizes = [file_size * (i / 100) for i in range(0, 101, 10)] # 101, because range is exclusive lmao + # Statistics + amount_written = 0 + with open(temp_file, 'wb') as f: + for chunk in r.iter_content(chunk_size=chunk_size): + if chunk: + f.write(chunk) + amount_written += chunk_size + sizes = print_progress(amount_written, file_size, sizes) + # Rename + os.rename(temp_file, filename) + + +def main(): + file = sys.argv[1] + urls = parse_file(file) + for url in urls: + filename = get_filename(url) + print(f"Downloading {filename}") + download_url(url, filename) + + +if __name__ == '__main__': + main() diff --git a/archive-rom/run.ps1 b/archive-rom/run.ps1 new file mode 100644 index 0000000..41b118d --- /dev/null +++ b/archive-rom/run.ps1 @@ -0,0 +1,2 @@ +.\venv\Scripts\activate.ps1 +python .\download-urls.py .\urls.txt \ No newline at end of file diff --git a/archive-rom/todo.url b/archive-rom/todo.url new file mode 100644 index 0000000..bbeba0a --- /dev/null +++ b/archive-rom/todo.url @@ -0,0 +1,5 @@ +[{000214A0-0000-0000-C000-000000000046}] +Prop3=19,11 +[InternetShortcut] +IDList= +URL=https://stackoverflow.com/questions/62007674/multi-thread-requests-python3 diff --git a/archive-rom/urls.txt b/archive-rom/urls.txt new file mode 100644 index 0000000..b78323e --- /dev/null +++ b/archive-rom/urls.txt @@ -0,0 +1,36 @@ +https://archive.org/download/wii_rvz_usa/wii_rvz_usa/Animal%20Crossing%20-%20City%20Folk%20%28USA%2C%20Asia%29%20%28En%2CFr%2CEs%29%20%28Rev%201%29.rvz +https://archive.org/download/wii_rvz_usa/wii_rvz_usa/Donkey%20Kong%20-%20Barrel%20Blast%20%28USA%29.rvz +https://archive.org/download/wii_rvz_usa/wii_rvz_usa/Donkey%20Kong%20Country%20Returns%20%28USA%29%20%28En%2CFr%2CEs%29%20%28Rev%201%29.rvz +https://archive.org/download/wii_rvz_usa_p2/wii_rvz_usa/GoldenEye%20007%20%28USA%29%20%28En%2CFr%29.rvz +https://archive.org/download/wii_rvz_usa_p2/wii_rvz_usa/Kirby%27s%20Dream%20Collection%20-%20Special%20Edition%20%28USA%29.rvz +https://archive.org/download/wii_rvz_usa_p2/wii_rvz_usa/Kirby%27s%20Epic%20Yarn%20%28USA%29%20%28En%2CFr%2CEs%29.rvz +https://archive.org/download/wii_rvz_usa_p2/wii_rvz_usa/Kirby%27s%20Return%20to%20Dream%20Land%20%28USA%29%20%28En%2CFr%2CEs%29.rvz +https://archive.org/download/wii_rvz_usa_p2/wii_rvz_usa/Last%20Story%2C%20The%20%28USA%29%20%28En%2CFr%2CEs%29.rvz +https://archive.org/download/wii_rvz_usa_p2/wii_rvz_usa/Legend%20of%20Zelda%2C%20The%20-%20Skyward%20Sword%20%28USA%29%20%28En%2CFr%2CEs%29%20%28Rev%202%29.rvz +https://archive.org/download/wii_rvz_usa_p2/wii_rvz_usa/Legend%20of%20Zelda%2C%20The%20-%20Twilight%20Princess%20%28USA%29%20%28En%2CFr%2CEs%29%20%28Rev%202%29.rvz +https://archive.org/download/wii_rvz_usa_p2/wii_rvz_usa/Mario%20Kart%20Wii%20%28USA%29%20%28En%2CFr%2CEs%29.rvz +https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/New%20Super%20Mario%20Bros.%20Wii%20%28USA%29%20%28En%2CFr%2CEs%29%20%28Rev%202%29.rvz +https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/No%20More%20Heroes%20%28USA%29%20%28En%2CFr%2CEs%29.rvz +https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/No%20More%20Heroes%202%20-%20Desperate%20Struggle%20%28USA%29%20%28En%2CFr%2CEs%29.rvz +https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/Ookami%20%28USA%29.rvz +https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/Pikmin%202%20%28USA%29%20%28En%2CFr%2CEs%29.rvz +https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/Pokemon%20Battle%20Revolution%20%28USA%29.rvz +https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/Punch-Out%21%21%20%28USA%29%20%28En%2CFr%2CEs%29%20%28Rev%201%29.rvz +https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/Rhythm%20Heaven%20Fever%20%28USA%29.rvz +https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/Super%20Mario%20All-Stars%20%28USA%29.rvz +https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/Super%20Mario%20Galaxy%20%28USA%29%20%28En%2CFr%2CEs%29.rvz +https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/Super%20Mario%20Galaxy%202%20%28USA%29%20%28En%2CFr%2CEs%29.rvz +https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/Super%20Paper%20Mario%20%28USA%29%20%28Rev%202%29.rvz +https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/Super%20Smash%20Bros.%20Brawl%20%28USA%29%20%28Rev%202%29.rvz +https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Wario%20Land%20-%20Shake%20It%21%20%28USA%29%20%28En%2CFr%2CEs%29.rvz +https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/WarioWare%20-%20Smooth%20Moves%20%28USA%29.rvz +https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Wii%20Music%20%28USA%29%20%28En%2CFr%2CEs%29.rvz +https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Wii%20Party%20%28USA%29%20%28En%2CFr%2CEs%29.rvz +https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Wii%20Play%20%28USA%29%20%28Rev%201%29.rvz +https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Wii%20Sports%20%28USA%29%20%28Rev%201%29.rvz +https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Wii%20Sports%20Resort%20%28USA%29%20%28En%2CFr%2CEs%29%20%28Rev%201%29.rvz +https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Wii%20Sports%20%2B%20Wii%20Sports%20Resort%20%28USA%29%20%28En%2CFr%2CEs%29.rvz +https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Xenoblade%20Chronicles%20%28USA%2C%20Asia%29%20%28En%2CFr%2CEs%29.rvz +https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Trauma%20Center%20-%20New%20Blood%20%28USA%29.rvz +https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Trauma%20Center%20-%20Second%20Opinion%20%28USA%29.rvz +https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Trauma%20Team%20%28USA%29.rvz