Add tool to download from archive.org
This commit is contained in:
parent
76fb8c521d
commit
f4316e598d
8
archive-rom/.idea/.gitignore
vendored
Normal file
8
archive-rom/.idea/.gitignore
vendored
Normal file
@ -0,0 +1,8 @@
|
||||
# Default ignored files
|
||||
/shelf/
|
||||
/workspace.xml
|
||||
# Editor-based HTTP Client requests
|
||||
/httpRequests/
|
||||
# Datasource local storage ignored files
|
||||
/dataSources/
|
||||
/dataSources.local.xml
|
||||
8
archive-rom/.idea/archive-rom.iml
Normal file
8
archive-rom/.idea/archive-rom.iml
Normal file
@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="PYTHON_MODULE" version="4">
|
||||
<component name="NewModuleRootManager">
|
||||
<content url="file://$MODULE_DIR$" />
|
||||
<orderEntry type="jdk" jdkName="Python 3.9 (archive-rom)" jdkType="Python SDK" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
</component>
|
||||
</module>
|
||||
12
archive-rom/.idea/inspectionProfiles/Project_Default.xml
Normal file
12
archive-rom/.idea/inspectionProfiles/Project_Default.xml
Normal file
@ -0,0 +1,12 @@
|
||||
<component name="InspectionProjectProfileManager">
|
||||
<profile version="1.0">
|
||||
<option name="myName" value="Project Default" />
|
||||
<inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
|
||||
<option name="ignoredIdentifiers">
|
||||
<list>
|
||||
<option value="list.__getitem__" />
|
||||
</list>
|
||||
</option>
|
||||
</inspection_tool>
|
||||
</profile>
|
||||
</component>
|
||||
@ -0,0 +1,6 @@
|
||||
<component name="InspectionProjectProfileManager">
|
||||
<settings>
|
||||
<option name="USE_PROJECT_PROFILE" value="false" />
|
||||
<version value="1.0" />
|
||||
</settings>
|
||||
</component>
|
||||
11
archive-rom/.idea/misc.xml
Normal file
11
archive-rom/.idea/misc.xml
Normal file
@ -0,0 +1,11 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="Black">
|
||||
<option name="sdkName" value="Python 3.9 (archive-rom)" />
|
||||
</component>
|
||||
<component name="DiscordProjectSettings">
|
||||
<option name="show" value="ASK" />
|
||||
<option name="description" value="" />
|
||||
</component>
|
||||
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (archive-rom)" project-jdk-type="Python SDK" />
|
||||
</project>
|
||||
8
archive-rom/.idea/modules.xml
Normal file
8
archive-rom/.idea/modules.xml
Normal file
@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ProjectModuleManager">
|
||||
<modules>
|
||||
<module fileurl="file://$PROJECT_DIR$/.idea/archive-rom.iml" filepath="$PROJECT_DIR$/.idea/archive-rom.iml" />
|
||||
</modules>
|
||||
</component>
|
||||
</project>
|
||||
60
archive-rom/download-urls.py
Normal file
60
archive-rom/download-urls.py
Normal file
@ -0,0 +1,60 @@
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
import humanize
|
||||
from urllib.parse import unquote
|
||||
|
||||
|
||||
def parse_file(file_name):
|
||||
urls = []
|
||||
with open(file_name, 'r') as f:
|
||||
for line in f:
|
||||
urls.append(unquote(line.strip()))
|
||||
return urls
|
||||
|
||||
|
||||
def get_filename(url):
|
||||
return url.split('/')[-1]
|
||||
|
||||
|
||||
def print_progress(amount_written, file_size, sizes):
|
||||
for size in sizes:
|
||||
if amount_written >= size:
|
||||
sizes.remove(size)
|
||||
percentage = (10 - len(sizes)) * 10
|
||||
print(f"{percentage}% Written {humanize.naturalsize(amount_written)} / {humanize.naturalsize(file_size)}")
|
||||
return sizes
|
||||
|
||||
|
||||
def download_url(url, filename):
|
||||
# Constant
|
||||
temp_file = filename + ".tmp"
|
||||
chunk_size = 1024 * 1024 * 8 # 8MB
|
||||
# Request
|
||||
r = requests.get(url, stream=True)
|
||||
# More constants
|
||||
file_size = int(r.headers.get('Content-Length'))
|
||||
sizes = [file_size * (i / 100) for i in range(0, 101, 10)] # 101, because range is exclusive lmao
|
||||
# Statistics
|
||||
amount_written = 0
|
||||
with open(temp_file, 'wb') as f:
|
||||
for chunk in r.iter_content(chunk_size=chunk_size):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
amount_written += chunk_size
|
||||
sizes = print_progress(amount_written, file_size, sizes)
|
||||
# Rename
|
||||
os.rename(temp_file, filename)
|
||||
|
||||
|
||||
def main():
|
||||
file = sys.argv[1]
|
||||
urls = parse_file(file)
|
||||
for url in urls:
|
||||
filename = get_filename(url)
|
||||
print(f"Downloading {filename}")
|
||||
download_url(url, filename)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
2
archive-rom/run.ps1
Normal file
2
archive-rom/run.ps1
Normal file
@ -0,0 +1,2 @@
|
||||
.\venv\Scripts\activate.ps1
|
||||
python .\download-urls.py .\urls.txt
|
||||
5
archive-rom/todo.url
Normal file
5
archive-rom/todo.url
Normal file
@ -0,0 +1,5 @@
|
||||
[{000214A0-0000-0000-C000-000000000046}]
|
||||
Prop3=19,11
|
||||
[InternetShortcut]
|
||||
IDList=
|
||||
URL=https://stackoverflow.com/questions/62007674/multi-thread-requests-python3
|
||||
36
archive-rom/urls.txt
Normal file
36
archive-rom/urls.txt
Normal file
@ -0,0 +1,36 @@
|
||||
https://archive.org/download/wii_rvz_usa/wii_rvz_usa/Animal%20Crossing%20-%20City%20Folk%20%28USA%2C%20Asia%29%20%28En%2CFr%2CEs%29%20%28Rev%201%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa/wii_rvz_usa/Donkey%20Kong%20-%20Barrel%20Blast%20%28USA%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa/wii_rvz_usa/Donkey%20Kong%20Country%20Returns%20%28USA%29%20%28En%2CFr%2CEs%29%20%28Rev%201%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p2/wii_rvz_usa/GoldenEye%20007%20%28USA%29%20%28En%2CFr%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p2/wii_rvz_usa/Kirby%27s%20Dream%20Collection%20-%20Special%20Edition%20%28USA%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p2/wii_rvz_usa/Kirby%27s%20Epic%20Yarn%20%28USA%29%20%28En%2CFr%2CEs%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p2/wii_rvz_usa/Kirby%27s%20Return%20to%20Dream%20Land%20%28USA%29%20%28En%2CFr%2CEs%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p2/wii_rvz_usa/Last%20Story%2C%20The%20%28USA%29%20%28En%2CFr%2CEs%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p2/wii_rvz_usa/Legend%20of%20Zelda%2C%20The%20-%20Skyward%20Sword%20%28USA%29%20%28En%2CFr%2CEs%29%20%28Rev%202%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p2/wii_rvz_usa/Legend%20of%20Zelda%2C%20The%20-%20Twilight%20Princess%20%28USA%29%20%28En%2CFr%2CEs%29%20%28Rev%202%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p2/wii_rvz_usa/Mario%20Kart%20Wii%20%28USA%29%20%28En%2CFr%2CEs%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/New%20Super%20Mario%20Bros.%20Wii%20%28USA%29%20%28En%2CFr%2CEs%29%20%28Rev%202%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/No%20More%20Heroes%20%28USA%29%20%28En%2CFr%2CEs%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/No%20More%20Heroes%202%20-%20Desperate%20Struggle%20%28USA%29%20%28En%2CFr%2CEs%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/Ookami%20%28USA%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/Pikmin%202%20%28USA%29%20%28En%2CFr%2CEs%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/Pokemon%20Battle%20Revolution%20%28USA%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/Punch-Out%21%21%20%28USA%29%20%28En%2CFr%2CEs%29%20%28Rev%201%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/Rhythm%20Heaven%20Fever%20%28USA%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/Super%20Mario%20All-Stars%20%28USA%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/Super%20Mario%20Galaxy%20%28USA%29%20%28En%2CFr%2CEs%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/Super%20Mario%20Galaxy%202%20%28USA%29%20%28En%2CFr%2CEs%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/Super%20Paper%20Mario%20%28USA%29%20%28Rev%202%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p3/wii_rvz_usa/Super%20Smash%20Bros.%20Brawl%20%28USA%29%20%28Rev%202%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Wario%20Land%20-%20Shake%20It%21%20%28USA%29%20%28En%2CFr%2CEs%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/WarioWare%20-%20Smooth%20Moves%20%28USA%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Wii%20Music%20%28USA%29%20%28En%2CFr%2CEs%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Wii%20Party%20%28USA%29%20%28En%2CFr%2CEs%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Wii%20Play%20%28USA%29%20%28Rev%201%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Wii%20Sports%20%28USA%29%20%28Rev%201%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Wii%20Sports%20Resort%20%28USA%29%20%28En%2CFr%2CEs%29%20%28Rev%201%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Wii%20Sports%20%2B%20Wii%20Sports%20Resort%20%28USA%29%20%28En%2CFr%2CEs%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Xenoblade%20Chronicles%20%28USA%2C%20Asia%29%20%28En%2CFr%2CEs%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Trauma%20Center%20-%20New%20Blood%20%28USA%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Trauma%20Center%20-%20Second%20Opinion%20%28USA%29.rvz
|
||||
https://archive.org/download/wii_rvz_usa_p4/wii_rvz_usa/Trauma%20Team%20%28USA%29.rvz
|
||||
Loading…
Reference in New Issue
Block a user