import os, sys, time, urllib.request, urllib.error
from concurrent.futures import ThreadPoolExecutor
OUT = "/root/alhang/2022-12-01/assets"
UA = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
      "Chrome/120.0.0.0 Safari/537.36")
names = [n for n in open("/tmp/media2.txt").read().split() if n]
print("to fetch: %d" % len(names), flush=True)
ok = fail = 0
def get(name):
    global ok, fail
    path = os.path.join(OUT, name)
    if os.path.exists(path) and os.path.getsize(path) > 0:
        return
    req = urllib.request.Request("https://discord.com/assets/" + name,
                                 headers={"User-Agent": UA})
    for attempt in range(2):
        try:
            with urllib.request.urlopen(req, timeout=25) as r:
                data = r.read()
            with open(path, "wb") as fh:
                fh.write(data)
            ok += 1
            return
        except urllib.error.HTTPError as e:
            if e.code == 404:
                fail += 1; return
            time.sleep(0.5)
        except Exception:
            time.sleep(0.5)
    fail += 1
with ThreadPoolExecutor(max_workers=12) as pool:
    list(pool.map(get, names))
print("downloaded=%d failed=%d" % (ok, fail), flush=True)
files = os.listdir(OUT)
total = sum(os.path.getsize(os.path.join(OUT, f)) for f in files)
print("assets now: %d files, %.1f MB" % (len(files), total/1024/1024), flush=True)