diff --git a/test/mockgpu/mockgpu.py b/test/mockgpu/mockgpu.py index 703b1932d3..a4e7797e60 100644 --- a/test/mockgpu/mockgpu.py +++ b/test/mockgpu/mockgpu.py @@ -1,4 +1,4 @@ -import ctypes, time, os, builtins, fcntl +import ctypes, time, os, builtins, fcntl, typing from tinygrad.helpers import DEV from tinygrad.runtime.support.hcq import FileIOInterface from tinygrad.runtime.autogen import libc @@ -9,7 +9,7 @@ start = time.perf_counter() drivers = [cls() for t in DEV.value if (cls:={"MOCKPCI+AMD": AMDriver, "MOCKKFD+AMD": AMDDriver, "MOCK+AMD": AMDDriver, "MOCKUSB+AMD": AMUSBDriver, "MOCK+NV": NVDriver}.get(f"{t.interface}+{t.device}"))] -tracked_fds = {} +tracked_fds: dict[int, typing.Any] = {} original_memoryview = builtins.memoryview class TrackedMemoryView: diff --git a/test/null/test_llm_tokenizer.py b/test/null/test_llm_tokenizer.py index c98fb95e4d..b9927811d1 100644 --- a/test/null/test_llm_tokenizer.py +++ b/test/null/test_llm_tokenizer.py @@ -1,4 +1,4 @@ -import unittest, base64, functools, sys +import unittest, base64, functools, re, sys, time, unicodedata from tinygrad.llm.cli import SimpleTokenizer, FallbackTemplate from tinygrad.helpers import fetch @@ -46,6 +46,29 @@ class TestLLMTokenizer(unittest.TestCase): def test_llama_repeat(self): self._test_coding(self.llama_tok, "00000000000000000", [ 931, 931, 931, 931, 931, 410 ]) def test_llama_pat(self): self._test_coding(self.llama_tok, "today\n \n", [ 31213, 14211 ]) + def test_split_regex_matches_naive_listing(self): + # the compacted codepoint ranges must match the same text as listing every codepoint + def naive(pre): return "".join(re.escape(chr(cp)) for cp in range(0x323b0) if unicodedata.category(chr(cp)).startswith(pre)) + r_ws, r_p_N, r_p_L = r"\t\n\x0b\x0c\r\x85" + naive("Z"), naive("N"), naive("L") + naive_re = re.compile("(?i:'s|'t|'re|'ve|'m|'ll|'d)|" + + f"[^\\r\\n{r_p_N}{r_p_L}]?[{r_p_L}]+|[{r_p_N}]{{1,3}}| ?[^{r_ws}{r_p_N}{r_p_L}]+[\\r\\n]*|[{r_ws}]*[\\r\\n]+|[{r_ws}]+(?![^{r_ws}])|[{r_ws}]+") + sample = "hello world 한국어 中文 текст ١٢٣ 123 😊\n \ttoday\n'équivalent ²³№ " + self.assertEqual(SimpleTokenizer({}, {})._split_to_word.findall(sample), naive_re.findall(sample)) + + def test_split_regex_speed(self): + # the naive listing compiles a 429KB pattern that takes 10+s to match a 225KB prompt; ranges keep it small and fast + tok = SimpleTokenizer({}, {}) + self.assertLess(len(tok._split_to_word.pattern), 100_000) + text = "The quick brown fox jumps over the lazy dog. " * 5000 + tok._split_to_word.findall(text) # warmup + tms = [] + for _ in range(5): + st = time.perf_counter() + words = tok._split_to_word.findall(text) + tms.append(time.perf_counter() - st) + self.assertLess(min(tms), 4) # best-of-5 is robust to CI scheduling pauses; new code takes ~60ms + self.assertEqual(len(words), 50001) + def test_tekken_from_gguf_kv(self): kv = { "tokenizer.ggml.tokens": ["", "", "", "[INST]", "[/INST]", "hello"], diff --git a/tinygrad/helpers.py b/tinygrad/helpers.py index 19ee44c2ee..fde2fb2534 100644 --- a/tinygrad/helpers.py +++ b/tinygrad/helpers.py @@ -412,7 +412,7 @@ def diskcache_get(table:str, key:dict|str|int) -> Any: if (val:=res.fetchone()) is not None: return pickle.loads(val[0]) return None -_db_tables = set() +_db_tables: set[str] = set() def diskcache_put(table:str, key:dict|str|int, val:Any, prepickled=False): if CACHELEVEL < 1: return val if isinstance(key, (str,int)): key = {"key": key} diff --git a/tinygrad/llm/cli.py b/tinygrad/llm/cli.py index 93b6022c43..77cfac8ce2 100644 --- a/tinygrad/llm/cli.py +++ b/tinygrad/llm/cli.py @@ -1,5 +1,5 @@ from __future__ import annotations -import sys, argparse, codecs, typing, re, unicodedata, json, time +import sys, argparse, codecs, itertools, typing, re, unicodedata, json, time from typing import TYPE_CHECKING from tinygrad import nn from tinygrad.uop.ops import UOp, Ops @@ -20,7 +20,11 @@ class SimpleTokenizer: # https://github.com/ggml-org/llama.cpp/blob/94933c8c2eeaa9a7983e3f6c08af76bd86724094/src/llama-vocab.cpp#L286 # 0x323b0 is one past the max codepoint in unicode categories L/N/Z (0x323af is max L) - def ucat_range(pre: str): return "".join(re.escape(chr(cp)) for cp in range(0x323b0) if unicodedata.category(chr(cp)).startswith(pre)) + # compact adjacent codepoints into ranges: listing them all makes re spend seconds on large prompts + def ucat_range(pre:str) -> str: + cps = enumerate(cp for cp in range(0x323b0) if unicodedata.category(chr(cp)).startswith(pre)) + runs = [list(g) for _, g in itertools.groupby(cps, lambda e: e[1]-e[0])] + return "".join(re.escape(chr(g[0][1])) + (f"-{re.escape(chr(g[-1][1]))}" if len(g) > 1 else "") for g in runs) r_ws, r_p_N, r_p_L = r"\t\n\x0b\x0c\r\x85" + ucat_range("Z"), ucat_range("N"), ucat_range("L") self._split_to_word = re.compile("(?i:'s|'t|'re|'ve|'m|'ll|'d)|" + \ f"[^\\r\\n{r_p_N}{r_p_L}]?[{r_p_L}]+|[{r_p_N}]{{1,3}}| ?[^{r_ws}{r_p_N}{r_p_L}]+[\\r\\n]*|[{r_ws}]*[\\r\\n]+|[{r_ws}]+(?![^{r_ws}])|[{r_ws}]+") diff --git a/tinygrad/runtime/ops_qcom.py b/tinygrad/runtime/ops_qcom.py index b2a1b28aa3..8fed9fdb9b 100644 --- a/tinygrad/runtime/ops_qcom.py +++ b/tinygrad/runtime/ops_qcom.py @@ -278,7 +278,7 @@ class QCOMProgram(HCQProgram): def _parse_lib(self, lib): # Extract image binary self.image_size = _read_lib(lib, 0x100) - self.image = bytearray(lib[(image_offset:=_read_lib(lib, 0xc0)):image_offset+self.image_size]) + self.image = lib[(image_offset:=_read_lib(lib, 0xc0)):image_offset+self.image_size] # Parse image descriptors image_desc_off = _read_lib(lib, 0x110)