mirror of
https://github.com/rwang0417/OctoAI.git
synced 2026-11-16 08:00:43 +00:00
102 lines
3.6 KiB
Python
102 lines
3.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Download the OctoWoW quest database into tools/data (stdlib only, Python 3.8+).
|
|
|
|
Source: https://github.com/FrancisEgan/OctoQuestDatabase (MIT, see data/NOTICE.md)
|
|
|
|
python update_data.py latest version (branch main)
|
|
python update_data.py --ref <sha> a specific commit, branch or tag
|
|
python update_data.py --check only show installed vs latest version
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import urllib.request
|
|
|
|
REPO = "FrancisEgan/OctoQuestDatabase"
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
DATA = os.path.join(HERE, "data")
|
|
UA = {"User-Agent": "OctoAI-update-data", "Accept": "application/vnd.github+json"}
|
|
|
|
|
|
def get(url, timeout=60):
|
|
req = urllib.request.Request(url, headers=UA)
|
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
return r.read()
|
|
|
|
|
|
def resolve(ref):
|
|
info = json.loads(get("https://api.github.com/repos/%s/commits/%s" % (REPO, ref)))
|
|
return info["sha"], info["commit"]["committer"]["date"]
|
|
|
|
|
|
def installed():
|
|
p = os.path.join(DATA, "VERSION.txt")
|
|
if os.path.isfile(p):
|
|
with open(p, encoding="utf-8") as f:
|
|
return f.read().strip()
|
|
return None
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("--ref", default="main", help="commit sha, branch or tag (default: main)")
|
|
ap.add_argument("--check", action="store_true", help="only compare installed and remote versions")
|
|
a = ap.parse_args()
|
|
|
|
sha, date = resolve(a.ref)
|
|
have = installed()
|
|
print("installed: %s" % (have or "none"))
|
|
print("remote: OctoQuestDatabase commit %s (%s)" % (sha, date))
|
|
if a.check:
|
|
return 0
|
|
if have and sha in have:
|
|
print("already up to date")
|
|
return 0
|
|
|
|
listing = json.loads(get("https://api.github.com/repos/%s/contents/source?ref=%s" % (REPO, sha)))
|
|
files = [(x["name"], x["download_url"]) for x in listing if x["type"] == "file" and x["name"].endswith(".json")]
|
|
extra = [("LICENSE.txt", "https://raw.githubusercontent.com/%s/%s/LICENSE" % (REPO, sha)),
|
|
("NOTICE.md", "https://raw.githubusercontent.com/%s/%s/NOTICE.md" % (REPO, sha))]
|
|
if not files:
|
|
print("no database files found in the repository, nothing changed")
|
|
return 1
|
|
|
|
tmp = tempfile.mkdtemp(prefix="octo_data_", dir=HERE)
|
|
try:
|
|
for name, url in files + extra:
|
|
sys.stdout.write(" %-26s" % name)
|
|
sys.stdout.flush()
|
|
body = get(url, timeout=300)
|
|
if name.endswith(".json"):
|
|
json.loads(body.decode("utf-8")) # refuse broken downloads
|
|
with open(os.path.join(tmp, name), "wb") as f:
|
|
f.write(body)
|
|
print("%8.0f KB" % (len(body) / 1024.0))
|
|
with open(os.path.join(tmp, "VERSION.txt"), "w", encoding="utf-8") as f:
|
|
f.write("OctoQuestDatabase commit %s (%s)\n" % (sha, date))
|
|
if os.path.isdir(DATA):
|
|
old = DATA + ".old"
|
|
if os.path.isdir(old):
|
|
shutil.rmtree(old)
|
|
os.rename(DATA, old)
|
|
os.rename(tmp, DATA)
|
|
shutil.rmtree(old, ignore_errors=True)
|
|
else:
|
|
os.rename(tmp, DATA)
|
|
except Exception:
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
raise
|
|
print("done: %d files in %s" % (len(files), DATA))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
sys.exit(main())
|
|
except Exception as e: # network errors etc.
|
|
print("failed: %s (the existing data folder was left unchanged)" % e)
|
|
sys.exit(1)
|