import os, re, sys, time, urllib.request, urllib.error
from concurrent.futures import ThreadPoolExecutor

OUT = "/root/alhang/2022-12-01/assets"
UA = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
      "Chrome/120.0.0.0 Safari/537.36")
os.makedirs(OUT, exist_ok=True)

targets = set()
for h in open("chunks.txt").read().split():
    targets.add(h + ".js")
# css and icon named in the shell, plus any <chunkid>.<hash>.css in the bundles
shell = open("/root/alhang/2022-12-01/index.html", encoding="utf-8", errors="replace").read()
for m in re.findall(r'/assets/([A-Za-z0-9._-]+\.(?:css|ico|png|svg|woff2?))', shell):
    targets.add(m)
for fn in os.listdir(OUT):
    if fn.endswith(".js"):
        s = open(os.path.join(OUT, fn), encoding="utf-8", errors="replace").read()
        for m in re.findall(r'"(\d+\.[0-9a-f]{20}\.css)"', s):
            targets.add(m)

targets = sorted(targets)
print("to fetch: %d" % len(targets), flush=True)
ok = fail = skip = 0
def get(name):
    global ok, fail, skip
    path = os.path.join(OUT, name)
    if os.path.exists(path) and os.path.getsize(path) > 0:
        skip += 1; return
    req = urllib.request.Request("https://discord.com/assets/" + name,
                                 headers={"User-Agent": UA})
    for attempt in range(3):
        try:
            with urllib.request.urlopen(req, timeout=30) as r:
                data = r.read()
            with open(path, "wb") as fh:
                fh.write(data)
            ok += 1
            return
        except urllib.error.HTTPError as e:
            if e.code == 404:
                fail += 1; return
            time.sleep(1 + attempt)
        except Exception:
            time.sleep(1 + attempt)
    fail += 1

with ThreadPoolExecutor(max_workers=10) as pool:
    list(pool.map(get, targets))
print("downloaded=%d already=%d failed=%d" % (ok, skip, fail), flush=True)
total = sum(os.path.getsize(os.path.join(OUT, f)) for f in os.listdir(OUT))
print("assets on disk: %d files, %.1f MB" % (len(os.listdir(OUT)), total/1024/1024), flush=True)