From 0ccef542e0c56aba79521b839a10deb8010fa92c Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:20:16 -0700 Subject: [PATCH] add extract support to fetch (#17641) --- tinygrad/helpers.py | 15 ++++++++++++++- tinygrad/runtime/support/compiler_qcom.py | 7 ++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/tinygrad/helpers.py b/tinygrad/helpers.py index f3dbca7c3c..d291c7eae2 100644 --- a/tinygrad/helpers.py +++ b/tinygrad/helpers.py @@ -460,14 +460,16 @@ def _ensure_downloads_dir() -> pathlib.Path: return pathlib.Path(cache_dir) / "downloads" def fetch(url:str, name:pathlib.Path|str|None=None, subdir:str|None=None, gunzip:bool=False, allow_caching=not getenv("DISABLE_HTTP_CACHE"), - headers:dict[str, str]={}, sha256:str|None=None) -> pathlib.Path: + headers:dict[str, str]={}, sha256:str|None=None, extract:bool=False) -> pathlib.Path: import urllib.request if url.startswith(("/", ".")): return pathlib.Path(url) if name is not None and (isinstance(name, pathlib.Path) or '/' in name): fp = pathlib.Path(name) else: hh = "_"+hashlib.md5(("\n".join(f"{k.strip()}:{v.strip()}" for k,v in sorted(headers.items()))).encode("utf-8")).hexdigest() if headers else "" fp = _ensure_downloads_dir() / (subdir or "") / ((name or hashlib.md5(url.encode('utf-8')).hexdigest()) + hh + (".gunzip" if gunzip else "")) + extract_dir = fp.parent / f"{fp.name}.extract" if not fp.is_file() or not allow_caching or (sha256 and hashlib.sha256(fp.read_bytes()).hexdigest() != sha256): + if extract: shutil.rmtree(extract_dir, ignore_errors=True) (_dir := fp.parent).mkdir(parents=True, exist_ok=True) with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "tinygrad 0.13.0", **headers}), timeout=10) as r: assert r.status in {200, 206}, r.status @@ -484,6 +486,17 @@ def fetch(url:str, name:pathlib.Path|str|None=None, subdir:str|None=None, gunzip pathlib.Path(f.name).rename(fp) progress_bar.update(close=True) if length and (file_size:=os.stat(fp).st_size) < length: raise RuntimeError(f"fetch size incomplete, {file_size} < {length}") + if extract: + if not extract_dir.is_dir(): + import tarfile + tmpdir = tempfile.mkdtemp(dir=fp.parent) + try: + with tarfile.open(fp) as t: t.extractall(tmpdir, filter="data") + try: os.rename(tmpdir, extract_dir) # rename is atomic, so concurrent fetches can't see a partial extraction + except OSError: + if not extract_dir.is_dir(): raise + finally: shutil.rmtree(tmpdir, ignore_errors=True) + return extract_dir return fp def fetch_fw(path:str, name:str, sha256:str) -> bytes: diff --git a/tinygrad/runtime/support/compiler_qcom.py b/tinygrad/runtime/support/compiler_qcom.py index 64a7cba8d6..62693ea5ed 100644 --- a/tinygrad/runtime/support/compiler_qcom.py +++ b/tinygrad/runtime/support/compiler_qcom.py @@ -1,4 +1,4 @@ -import ctypes, struct, platform, pathlib, shutil, tarfile, tempfile +import ctypes, struct, platform, pathlib, shutil from tinygrad.device import Compiler from tinygrad.helpers import DEBUG, system, fetch from tinygrad.runtime.support.compiler_mesa import disas_adreno @@ -12,8 +12,9 @@ class QCOMCompiler(Compiler): assert arch.split(',')[0] == "a630", "only a630 supported" if platform.machine() == "aarch64": self.arch, self.chip_id, self.llvm_inst = arch, 0x6030001, llvm_qcom.cl_compiler_create_llvm_instance() else: - self.arch, self.chip_id, self.fs, root = arch, 0x6030001, tempfile.TemporaryDirectory(), pathlib.Path(__file__).parents[3] - with tarfile.open(fetch('https://git.tinygrad.win/sirhcm/images/releases/download/v2/qcomcl.tar.gz')) as t: t.extractall(fs:=self.fs.name) + # extract once into the download cache, all processes share the rootfs (extract=True) + self.arch, self.chip_id = arch, 0x6030001 + fs, root = fetch('https://git.tinygrad.win/sirhcm/images/releases/download/v2/qcomcl.tar.gz', extract=True), pathlib.Path(__file__).parents[3] self.compiler_process = self.server(f"{qemu} -cpu max,pauth=off -L {fs} {fs}/usr/bin/python3" if (qemu:=shutil.which("qemu-aarch64-static")) else (f"docker run --rm -i --platform linux/aarch64 -v {fs}/usr:/usr -v {root}:{root} " f"-e PYTHONPATH={root} -e QEMU_CPU=max,pauth=off gcr.io/distroless/static python3"), arch)