From f06832bf6f2bcfdfe52964c8cbbb33cbd88ccae3 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Thu, 27 Aug 2026 16:01:55 -0700 Subject: [PATCH] test llm with --no_chat_template (#17785) --- .github/workflows/test.yml | 8 ++++---- tinygrad/llm/cli.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 38b87c21cd..3b2e2bd586 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -390,10 +390,10 @@ jobs: run: | parallel --link --tagstring '[{1}]' '{2}' \ ::: llama 'llama q4' qwen3.5 qwen \ - ::: $'echo "What\'s a male chicken called? Answer with only one word." | python3 -m tinygrad.llm --model llama3.2:1b | tee /dev/stderr | grep -i rooster' \ - $'echo "What\'s a male chicken called? Answer with only one word." | python3 -m tinygrad.llm --model llama3.2:1b-q4 | tee /dev/stderr | grep -i rooster' \ - $'echo "What\'s a male chicken called? Answer with only one word." | python3 -m tinygrad.llm --model qwen3.5:0.8b | tee /dev/stderr | grep -i rooster' \ - $'echo "What\'s a female chicken called? Answer with only one word." | python3 -m tinygrad.llm --model qwen3:0.6b | tee /dev/stderr | grep -i hen' + ::: $'echo "What\'s a male chicken called? Answer with only one word." | python3 -m tinygrad.llm --no_chat_template --model llama3.2:1b | tee /dev/stderr | grep -i rooster' \ + $'echo "What\'s a male chicken called? Answer with only one word." | python3 -m tinygrad.llm --no_chat_template --model llama3.2:1b-q4 | tee /dev/stderr | grep -i rooster' \ + $'echo "What\'s a male chicken called? Answer with only one word." | python3 -m tinygrad.llm --no_chat_template --model qwen3.5:0.8b | tee /dev/stderr | grep -i rooster' \ + $'echo "What\'s a female chicken called? Answer with only one word." | python3 -m tinygrad.llm --no_chat_template --model qwen3:0.6b | tee /dev/stderr | grep -i hen' # NOTE: qwen is dumb and only knows about female chickens # ****** Models Tests ****** diff --git a/tinygrad/llm/cli.py b/tinygrad/llm/cli.py index e0e2336e2c..417ffa7518 100644 --- a/tinygrad/llm/cli.py +++ b/tinygrad/llm/cli.py @@ -145,6 +145,7 @@ def main(): parser.add_argument("--serve", nargs='?', type=int, const=8000, metavar="PORT", help="Run OpenAI compatible API (optional port, default 8000)") parser.add_argument("--warmup", action="store_true", help="warmup the JIT") parser.add_argument("--benchmark", nargs='?', type=int, const=20, metavar="COUNT", help="Benchmark tok/s (optional count, default 20)") + parser.add_argument("--no_chat_template", action="store_true", help="Don't use the model's chat template, always use the fallback template") args = parser.parse_args() # load the model @@ -160,7 +161,7 @@ def main(): # use the model's chat template if jinja2 is available (enables model-specific formatting) template: jinja2.Template|FallbackTemplate = FallbackTemplate(tok) - if (ct := kv.get('tokenizer.chat_template')) is not None: + if not args.no_chat_template and (ct := kv.get('tokenizer.chat_template')) is not None: try: import jinja2 env = jinja2.Environment()