On this tutorial, we work with Dory 1.0 4B, a small reasoning mannequin from NVIDIA’s Nemotron Labs whose center layers kind a shared block that may be utilized as soon as, twice, three, or 4 instances for each token. Every further cross, or loop, spends extra computation per token with out including a single parameter, so one checkpoint can run at 4 totally different compute budgets with no retraining. We construct the llama.cpp department that NVIDIA’s mannequin card pins, obtain the three.7 GB GGUF, and skim the structure straight from the file to foretell what every loop ought to price in parameters and cache reminiscence. We then examine these predictions towards the working server, measure decode velocity at each loop rely, take a look at whether or not the cheaper last-loop cache adjustments the mannequin’s predictions, and run a paired GSM8K analysis to see how accuracy, reply size, and time per solved query transfer because the loops improve.
import os
import re
import sys
import json
import time
import math
import random
import sign
import shutil
import hashlib
import platform
import statistics
import traceback
import subprocess
import collections
import urllib.request
from pathlib import Path
RESULTS = {} # step -> one-line outcome, for the abstract desk
CLAIMS = [] # (step, declare, held)
CURRENT = ["1"]
def banner(title):
print("n" + "=" * 78)
print(title)
print("=" * 78)
def part(title):
def wrap(fn):
def run(*a, **kw):
CURRENT[0] = title.cut up(".")[0]
banner(title)
t0 = time.time()
attempt:
out = fn(*a, **kw)
RESULTS[name] = out if isinstance(out, str) else "ok"
return out
besides Exception as e:
RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}"
print(f"n[!] {name} did not complete: {type(e).__name__}: {e}")
traceback.print_exc(restrict=3)
return None
lastly:
print(f"n[{time.time() - t0:.1f} s]")
return run
return wrap
def declare(textual content, held):
CLAIMS.append((CURRENT[0], textual content, bool(held)))
return held
banner("1. Build the pinned Dory llama.cpp fork and record the environment")
FORK = "https://github.com/brianchmielNV/llama.cpp.git"
FORK_COMMIT = "98a74c97ea09ac72f6ceda1559808fc0a2f447ce" # the commit NVIDIA's mannequin card pins; ggml-org PR #30306 remains to be a draft
GGUF_REPO = "nvidia/NVIDIA-Nemotron-Labs-Dory1.0-4B-GGUF"
GGUF_REV = "df53755cb8741843af8e019e088819fbef17e29b"
GGUF_FILE = "NVIDIA-Nemotron-Labs-Dory1.0-4B-Q5_K_M.gguf" # the cardboard's instructions name it dory-q5_k_m.gguf
GGUF_SIZE = 3_673_288_864
GGUF_SHA256 = "96080be49ffd67b916018b7bdef8c484762576fb52780e10ab98ac4978a660e0"
GSM8K_REV = "740312add88f781978c0658806c59bc2815b9866"
WORK = Path(os.environ.get("DORY_WORK") or ("/content/dory" if os.path.isdir("/content") else "dory_work")).resolve()
WORK.mkdir(dad and mom=True, exist_ok=True)
(WORK / "logs").mkdir(exist_ok=True)
subprocess.run([sys.executable, "-m", "pip", "install", "-q", "gguf==0.19.0", "huggingface_hub", "pandas", "pyarrow"], examine=True)
SYSTEM = platform.system()
GPU = ""
if shutil.which("nvidia-smi"):
q = subprocess.run(["nvidia-smi", "--query-gpu=name,memory.total,compute_cap", "--format=csv,noheader"],
capture_output=True, textual content=True)
GPU = q.stdout.strip().splitlines()[0] if q.returncode == 0 and q.stdout.strip() else ""
BACKEND = "metal" if SYSTEM == "Darwin" else ("cuda" if GPU else "cpu")
SRC = WORK / "llama.cpp-dory"
SERVER_BIN = SRC / "build" / "bin" / "llama-server"
free_gb = shutil.disk_usage(WORK).free / 1e9
print(f"{SYSTEM} {platform.machine()}, Python {platform.python_version()}, {os.cpu_count()} CPU cores, "
f"backend {BACKEND}{' (' + GPU + ')' if GPU else ''}, {free_gb:.1f} GB free disk")
if not SERVER_BIN.exists():
t0 = time.time()
if not (SRC / ".git").exists():
subprocess.run(["git", "init", "-q", str(SRC)], examine=True)
subprocess.run(["git", "-C", str(SRC), "remote", "add", "origin", FORK], examine=True)
subprocess.run(["git", "-C", str(SRC), "fetch", "-q", "--depth", "1", "origin", FORK_COMMIT], examine=True)
subprocess.run(["git", "-C", str(SRC), "checkout", "-q", "--detach", "FETCH_HEAD"], examine=True)
flags = ["-DCMAKE_BUILD_TYPE=Release", "-DLLAMA_BUILD_SERVER=ON", "-DLLAMA_BUILD_TESTS=OFF",
"-DLLAMA_BUILD_EXAMPLES=OFF", "-DLLAMA_BUILD_APP=OFF", "-DLLAMA_BUILD_UI=OFF",
"-DLLAMA_USE_PREBUILT_UI=ON", "-DLLAMA_OPENSSL=OFF"]
if BACKEND == "metal": # the cardboard's Apple-silicon flags
flags += ["-DCMAKE_OSX_ARCHITECTURES=arm64", "-DGGML_METAL=ON", "-DGGML_METAL_EMBED_LIBRARY=ON",
"-DGGML_BLAS=ON", "-DGGML_BLAS_VENDOR=Apple"]
elif BACKEND == "cuda": # kernels for this GPU solely (a T4 is 7.5); flash-attention kernels are unused with --flash-attn off
cap = GPU.cut up(",")[-1].strip().exchange(".", "")
flags += ["-DGGML_CUDA=ON", f"-DCMAKE_CUDA_ARCHITECTURES={cap}", "-DGGML_CUDA_FA=OFF", "-DGGML_NATIVE=OFF"]
env = dict(os.environ, HF_UI_VERSION="b11551")
with open(WORK / "build.log", "w") as log:
subprocess.run(["cmake", "-S", str(SRC), "-B", str(SRC / "build")] + flags, examine=True, env=env, stdout=log, stderr=log)
subprocess.run(["cmake", "--build", str(SRC / "build"), "--parallel", str(os.cpu_count()), "--target", "llama-server"],
examine=True, env=env, stdout=log, stderr=log)
print(f"built llama-server in {time.time() - t0:.0f} s")
head = subprocess.run(["git", "-C", str(SRC), "rev-parse", "HEAD"], capture_output=True, textual content=True).stdout.strip()
assert head == FORK_COMMIT, f"checked out {head}, expected {FORK_COMMIT}"
model = subprocess.run([str(SERVER_BIN), "--version"], capture_output=True, textual content=True)
print(f"fork commit {head[:12]}; {(version.stdout + version.stderr).strip().splitlines()[-1]}")
RESULTS["1. Build the pinned Dory llama.cpp fork and record the environment"] = f"fork {head[:12]}, backend {BACKEND}"
We construct llama-server from the precise commit of the Dory department that NVIDIA’s mannequin card pins, as a result of Dory assist in llama.cpp remains to be an open draft pull request and no launched binary can load the mannequin. The cell detects the platform and passes the cardboard’s Metallic flags on a Mac, or compiles CUDA kernels for the connected GPU alone on Colab, skipping the flash-attention kernels that the cardboard’s –flash-attn off by no means makes use of, which retains a T4 construct manageable. It then checks that the checked-out commit is the pinned one, so each later quantity traces to at least one recognized model of the code.
@part("2. Download the Q5_K_M GGUF at a pinned revision and check its SHA-256")
def download_step():
world GGUF
from huggingface_hub import hf_hub_download
t0 = time.time()
GGUF = Path(hf_hub_download(GGUF_REPO, GGUF_FILE, revision=GGUF_REV, local_dir=str(WORK)))
h = hashlib.sha256()
with open(GGUF, "rb") as f:
for block in iter(lambda: f.learn(1 << 24), b""):
h.replace(block)
dimension, digest = GGUF.stat().st_size, h.hexdigest()
print(f"{GGUF.name}: {size:,} bytes, sha256 {digest[:16]}... ({time.time() - t0:.0f} s)")
assert dimension == GGUF_SIZE and digest == GGUF_SHA256, "the file does not match the card's size and checksum"
declare("the GGUF matches the size and SHA-256 published on the card", True)
return f"{size / 1e9:.2f} GB, checksum matches the card"
download_step()
The Q5_K_M GGUF comes from Hugging Face at a set repository revision, and we hash it earlier than anything touches it. The cardboard’s instructions confer with the file as dory-q5_k_m.gguf, however the repository names it NVIDIA-Nemotron-Labs-Dory1.0-4B-Q5_K_M.gguf, so copying the cardboard’s command verbatim fails; pinning the actual title, the scale, and the SHA-256 collectively catches each a fallacious file and a corrupted obtain. The three.67 GB file matches the cardboard’s revealed checksum precisely, so each measurement that follows makes use of the identical bytes NVIDIA revealed.
@part("3. Read the architecture from the GGUF: three layer groups and a default of four loops")
def architecture_step():
world META, GROUPS, PATTERN
import gguf
reader = gguf.GGUFReader(str(GGUF))
META = {ok: f.contents() for ok, f in reader.fields.objects() if ok.startswith("dory.")}
for ok in ("dory.block_count", "dory.input_layer_count", "dory.recurrent_layer_count", "dory.output_layer_count",
"dory.num_loops", "dory.recurrent_kv_cache_mode", "dory.attention.sliding_window", "dory.context_length"):
print(f"{k:34s} {META[k]}")
PATTERN = META["dory.layer_pattern"].cut up("-")[:-1] # one image per consideration layer: S sliding window, * full
n_in, n_rec = META["dory.input_layer_count"], META["dory.recurrent_layer_count"]
print(f"nlayer entries alternate attention and MLP: {len(PATTERN)} attention layers "
f"({PATTERN.count('S')} sliding-window, {PATTERN.count('*')} full attention)")
GROUPS = collections.defaultdict(lambda: {"params": 0, "bytes": 0})
dtypes = collections.Counter()
for t in reader.tensors:
m = re.match(r"blk.(d+).", t.title)
if m:
i = int(m.group(1))
g = "input block" if i < n_in else "recurrent block" if i < n_in + n_rec else "output block"
else:
g = "embedding" if t.title.startswith("token_embd") else "LM head"
GROUPS[g]["params"] += int(math.prod(int(x) for x in t.form))
GROUPS[g]["bytes"] += int(t.n_bytes)
dtypes[t.tensor_type.name] += 1
print(f"n{'group':16s} {'params':>9s} {'bytes':>9s}")
for g in ("embedding", "input block", "recurrent block", "output block", "LM head"):
print(f"{g:16s} {GROUPS[g]['params'] / 1e9:7.3f} B {GROUPS[g]['bytes'] / 1e9:6.3f} GB")
print(f"tensor types: {dict(dtypes)}")
declare("the GGUF stores 16 input, 40 recurrent and 16 output layer entries (8, 20 and 8 blocks)",
(n_in, n_rec, META["dory.output_layer_count"]) == (16, 40, 16))
declare("the GGUF's built-in loop count is 4, not the 2 the card recommends", META["dory.num_loops"] == 4)
complete = sum(v["params"] for v in GROUPS.values())
return (f"{total / 1e9:.2f} B parameters, {GROUPS['recurrent block']['params'] / 1e9:.2f} B of them in the shared "
f"recurrent block; built-in num_loops = {META['dory.num_loops']}")
architecture_step()
The GGUF header tells us how the mannequin is wired earlier than we load it. Dory’s 72 layer entries cut up into an enter group of 16, a shared recurrent group of 40 and an output group of 16, alternating consideration and MLP, with one full-attention layer for each three sliding-window layers. Summing tensor sizes by group exhibits the recurrent block accounts for 1.84 of the mannequin’s 3.98 billion parameters. The header additionally carries a entice: its built-in loop rely is 4, whereas the cardboard recommends 2, so a server began with out an override silently runs the costliest setting.
LOOPS = (1, 2, 3, 4)
CTX, UBATCH = 4096, 512
def predicted_kv(loops, mode, ctx=CTX, ubatch=UBATCH):
"""KV cache below the fork's allocation rule: per_loop retains one cache per loop for each recurrent consideration layer,
last_loop retains one amongst. Full-attention layers maintain ctx cells; sliding-window layers maintain window + ubatch, padded to 256."""
n_in, n_rec = META["dory.input_layer_count"] // 2, META["dory.recurrent_layer_count"] // 2
rec, mounted = PATTERN[n_in:n_in + n_rec], PATTERN[:n_in] + PATTERN[n_in + n_rec:]
copies = loops if mode == "per_loop" else 1
full = mounted.rely("*") + copies * rec.rely("*")
swa = mounted.rely("S") + copies * rec.rely("S")
swa_cells = min(ctx, -(-(META["dory.attention.sliding_window"] + ubatch) // 256) * 256)
per_cell = 2 * META["dory.attention.head_count_kv"] * META["dory.attention.key_length"] * 2 # Okay and V, 2 bytes every
return full, swa, (full * ctx + swa * swa_cells) * per_cell / 2**20
@part("4. Predict what each extra loop costs: parameters per token and KV-cache size")
def cost_model_step():
world PER_TOKEN
as soon as = ("input block", "output block", "LM head") # the embedding is a one-row lookup, so it prices no matrix work
PER_TOKEN = {}
print(f"{'loops':>5s} {'params per token':>17s} {'vs 1 loop':>10s} {'weight bytes read':>18s} {'vs 1 loop':>10s}")
for n in LOOPS:
p = sum(GROUPS[g]["params"] for g in as soon as) + n * GROUPS["recurrent block"]["params"]
b = sum(GROUPS[g]["bytes"] for g in as soon as) + n * GROUPS["recurrent block"]["bytes"]
PER_TOKEN[n] = (p, b)
print(f"{n:5d} {p / 1e9:15.2f} B {p / PER_TOKEN[1][0]:9.2f}x {b / 1e9:16.2f} GB {b / PER_TOKEN[1][1]:9.2f}x")
print(f"npredicted KV cache at --ctx-size {CTX}, f16 keys and values, micro-batch {UBATCH}:")
print(f"{'loops':>5s} {'per_loop: full + SWA layers':>28s} {'MiB':>7s} {'last_loop: full + SWA layers':>29s} {'MiB':>7s}")
for n in LOOPS:
a, b = predicted_kv(n, "per_loop"), predicted_kv(n, "last_loop")
print(f"{n:5d} {a[0]:>21d} + {a[1]:<4d} {a[2]:7.1f} {b[0]:>22d} + {b[1]:<4d} {b[2]:7.1f}")
full_ctx = META["dory.context_length"]
print(f"nat the full {full_ctx:,}-token context: per_loop {predicted_kv(2, 'per_loop', full_ctx)[2] / 1024:.1f} GiB at 2 loops "
f"and {predicted_kv(4, 'per_loop', full_ctx)[2] / 1024:.1f} GiB at 4; last_loop "
f"{predicted_kv(4, 'last_loop', full_ctx)[2] / 1024:.1f} GiB at any loop count")
return (f"4 loops apply {PER_TOKEN[4][0] / PER_TOKEN[1][0]:.2f}x the parameters per token of 1 loop; predicted KV at 4 loops "
f"{predicted_kv(4, 'per_loop')[2]:.0f} MiB per_loop vs {predicted_kv(4, 'last_loop')[2]:.0f} MiB last_loop")
cost_model_step()
From these group sizes, we will predict what a loop prices earlier than working something. A token passes by the enter block, the recurrent block as soon as per loop, the output block, and the LM head, so 4 loops apply 2.51 instances the parameters of 1 loop and skim 2.28 instances the load bytes, the higher information for a memory-bound decoder. The KV cache follows the fork’s allocation rule: per_loop retains a separate cache for each loop of each recurrent consideration layer, rising from 306 MiB at 1 loop to 816 MiB at 4 for a 4,096-token context, whereas last_loop stays at 306 MiB.
PORT = 8090
KV_LINE = re.compile(r"llama_kv_cache: size =s*([d.]+) MiB (s*(d+) cells,s*(d+) layers")
class DoryServer:
"""llama-server with one loop count and KV mode. Both are read when the model loads, so each setting is a fresh start."""
def __init__(self, loops=None, mode="per_loop", ngl=999, ubatch=UBATCH):
self.loops, self.mode, self.ngl, self.ubatch = loops, mode, ngl, ubatch
self.title = f"{'default' if loops is None else loops}-{mode}-ngl{ngl}-ub{ubatch}"
def __enter__(self):
cmd = [str(SERVER_BIN), "-m", str(GGUF), "--host", "127.0.0.1", "--port", str(PORT), "-ngl", str(self.ngl),
"--ctx-size", str(CTX), "--parallel", "1", "--batch-size", str(self.ubatch), "--ubatch-size", str(self.ubatch),
"--cache-type-k", "f16", "--cache-type-v", "f16", "--flash-attn", "off", "--no-context-shift",
"--jinja", "--reasoning", "on", "--temp", "1.0", "--top-p", "0.95", "--top-k", "0", "--min-p", "0",
"--repeat-penalty", "1"]
if self.loops is just not None:
cmd += ["--override-kv", f"dory.num_loops=int:{self.loops},dory.recurrent_kv_cache_mode=str:{self.mode}"]
self.log = WORK / "logs" / f"server-{self.name}.log"
with open(self.log, "w") as f:
self.proc = subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT, start_new_session=True)
t0 = time.time()
whereas True:
attempt:
with urllib.request.urlopen(f"http://127.0.0.1:{PORT}/health", timeout=2) as r:
if r.standing == 200:
break
besides Exception:
cross
if self.proc.ballot() is just not None or time.time() - t0 > 600:
self.cease()
increase RuntimeError(f"llama-server did not start; see {self.log}")
time.sleep(0.5)
self.load_s = time.time() - t0
textual content = self.log.read_text(errors="ignore")
sizes = KV_LINE.findall(textual content) # the full-attention cache first, then the sliding-window cache
self.kv_mib = sum(float(s[0]) for s in sizes)
self.kv_layers = tuple(int(s[2]) for s in sizes)
m = re.search(r"offloaded (d+)/(d+) layers", textual content)
self.offloaded = f"{m.group(1)}/{m.group(2)}" if m else "n/a"
return self
def __exit__(self, *exc):
self.cease()
def cease(self):
if self.proc.ballot() is None:
os.killpg(self.proc.pid, sign.SIGTERM)
attempt:
self.proc.wait(60)
besides subprocess.TimeoutExpired:
os.killpg(self.proc.pid, sign.SIGKILL)
self.proc.wait()
def publish(self, path, payload, timeout=1800):
req = urllib.request.Request(f"http://127.0.0.1:{PORT}{path}", information=json.dumps(payload).encode(),
headers={"Content-Type": "application/json"})
with urllib.request.urlopen(req, timeout=timeout) as r:
return json.hundreds(r.learn())
def split_reasoning(message):
"""The server returns the thinking in reasoning_content when it can parse it; otherwise it stays inside <think> tags."""
content material, reasoning = message.get("content") or "", message.get("reasoning_content") or ""
m = re.search(r"<think>(.*?)</think>", content material, re.S)
if m:
reasoning, content material = reasoning + m.group(1), content material.exchange(m.group(0), "")
return reasoning.strip(), content material.strip()
@part("5. Start the server, prove the loop override took effect, and catch the silent default")
def override_step():
rows = []
for loops, mode in ((2, "per_loop"), (2, "last_loop"), (None, "per_loop")):
with DoryServer(loops, mode) as s:
need = predicted_kv(META["dory.num_loops"] if loops is None else loops, mode)
rows.append((s.title, s.kv_layers, s.kv_mib, need, s.offloaded, s.load_s))
if loops == 2 and mode == "per_loop":
r = s.publish("/v1/chat/completions", {"messages": [{"role": "user", "content": "What is 17 multiplied by 23?"}],
"max_tokens": 1024, "seed": 1, "cache_prompt": False})
reasoning, reply = split_reasoning(r["choices"][0]["message"])
print(f"2 loops, a first request: {r['usage']['completion_tokens']} tokens, "
f"{len(reasoning.split())} words of reasoning, answer: {answer[:120]!r}")
print(f" fields in the message: {sorted(r['choices'][0]['message'])}; "
f"timings: {sorted(r.get('timings', {}))[:6]}...n")
print(f"{'server':28s} {'KV layers (full, SWA)':>22s} {'KV MiB':>8s} {'predicted':>16s} {'offloaded':>10s} {'load s':>7s}")
for title, layers, mib, need, off, load in rows:
print(f"{name:28s} {str(layers):>22s} {mib:8.1f} {str(want[:2]) + f' {want[2]:.1f}':>16s} {off:>10s} {load:7.1f}")
okay = all(layers == need[:2] for _, layers, _, need, _, _ in rows)
declare("the --override-kv loop count and cache mode are applied (KV layer counts match the prediction)", okay)
default = rows[-1]
declare("without an override the server runs the GGUF's 4 loops (the most expensive setting)",
default[1] == predicted_kv(4, "per_loop")[:2])
return f"override applied: {ok}; no override -> {default[1]} KV layers, {default[2]:.0f} MiB, i.e. 4 loops"
override_step()
Loop rely and cache mode are learn when the mannequin hundreds, so every setting wants its personal server course of; the DoryServer helper begins one per configuration, waits for the well being examine, and parses the KV-cache strains from the server’s log. We evaluate the parsed layer counts with the prediction to show an override truly took impact, slightly than trusting the flag. We additionally begin one server with none override as a destructive management, and ship one chat request to see how the server separates the mannequin’s reasoning from its reply.
CONFIGS = [(1, "per_loop"), (2, "per_loop"), (2, "last_loop"), (3, "per_loop"), (3, "last_loop"), (4, "per_loop"), (4, "last_loop")]
SPEED_PROMPT = ("Explain, step by step, how a recurrent block that is applied several times per token changes the amount of "
"computation a language model spends on each token, and what that means for memory and latency.")
@part("6. Measure KV memory and decode speed for every loop count and cache mode")
def sweep_step():
world SPEED
SPEED = {}
print(f"{'loops':>5s} {'mode':10s} {'KV MiB':>8s} {'predicted':>9s} {'prefill tok/s':>14s} {'decode tok/s':>13s} "
f"{'decode vs 1 loop':>17s} {'params ratio':>13s}")
for loops, mode in CONFIGS:
with DoryServer(loops, mode) as s:
runs = [s.post("/completion", {"prompt": SPEED_PROMPT, "n_predict": 192, "temperature": 0, "ignore_eos": True,
"cache_prompt": False, "seed": 1})["timings"] for _ in vary(3)]
dec = statistics.median(t["predicted_per_second"] for t in runs)
pre = statistics.median(t["prompt_per_second"] for t in runs)
SPEED[(loops, mode)] = {"kv_mib": s.kv_mib, "kv_layers": s.kv_layers, "decode": dec, "prefill": pre,
"decode_runs": [round(t["predicted_per_second"], 2) for t in runs]}
base = SPEED[(1, "per_loop")]["decode"]
print(f"{loops:5d} {mode:10s} {s.kv_mib:8.1f} {predicted_kv(loops, mode)[2]:9.1f} {pre:14.1f} {dec:13.1f} "
f"{base / dec:16.2f}x {PER_TOKEN[loops][0] / PER_TOKEN[1][0]:12.2f}x")
flat = len({spherical(SPEED[(n, 'last_loop')]['kv_mib'], 1) for n in (2, 3, 4)} | {spherical(SPEED[(1, 'per_loop')]['kv_mib'], 1)}) == 1
grows = SPEED[(1, "per_loop")]["kv_mib"] < SPEED[(2, "per_loop")]["kv_mib"] < SPEED[(3, "per_loop")]["kv_mib"] < SPEED[(4, "per_loop")]["kv_mib"]
declare("last_loop keeps the KV cache at the one-loop size for every loop count", flat)
declare("per_loop KV memory grows with every added loop", grows)
gradual = SPEED[(1, "per_loop")]["decode"] / SPEED[(4, "per_loop")]["decode"]
json.dump({f"{k[0]}-{k[1]}": v for ok, v in SPEED.objects()}, open(WORK / "speed.json", "w"), indent=1)
return (f"KV {SPEED[(1, 'per_loop')]['kv_mib']:.0f} MiB at 1 loop; 4 loops: {SPEED[(4, 'per_loop')]['kv_mib']:.0f} MiB per_loop, "
f"{SPEED[(4, 'last_loop')]['kv_mib']:.0f} MiB last_loop; decode {slow:.2f}x slower at 4 loops than at 1")
sweep_step()
With the helper in place, we begin a server for every of seven settings, one to 4 loops in each cache modes, and file the measured KV reminiscence subsequent to the prediction. Decode velocity comes from the server’s personal timings on a set immediate, producing 192 tokens greedily thrice and holding the median, so the one factor that adjustments between rows is the loop setting. Dividing the one-loop velocity by every row’s velocity offers the measured slowdown, which we set beside the expected ratios to see whether or not decoding behaves as compute-bound or memory-bound.
def top_probs(response):
"""Next-token candidates as {token: logprob}, from either of the server's two probability formats."""
first = response["completion_probabilities"][0]
if "top_logprobs" in first:
return {c["token"]: c["logprob"] for c in first["top_logprobs"]}
return {c["tok_str"]: math.log(max(c["prob"], 1e-30)) for c in first["probs"]}
GATE_PROMPTS = ["The capital of France is", "def fibonacci(n):n ", "Water boils at a temperature of",
"In 1969, the first humans to walk on the Moon were", "The derivative of x squared is"]
@part("7. Correctness gates: the same seed gives the same answer, and the GPU agrees with the CPU")
def gates_step():
payload = {"messages": [{"role": "user", "content": "Name three prime numbers greater than 50."}],
"max_tokens": 512, "seed": 7, "cache_prompt": False}
probe = lambda s, p: top_probs(s.publish("/completion", {"prompt": p, "n_predict": 1, "n_probs": 10, "temperature": 0,
"cache_prompt": False}))
with DoryServer(2, "per_loop") as s:
a, b = (s.publish("/v1/chat/completions", payload) for _ in vary(2))
identical = a["choices"][0]["message"] == b["choices"][0]["message"]
accel = [probe(s, p) for p in GATE_PROMPTS]
print(f"two sampled chat answers with seed 7 at temperature 1.0: identical = {same}")
with DoryServer(2, "per_loop", ngl=0) as s:
cpu = [probe(s, p) for p in GATE_PROMPTS]
agree, gaps = 0, []
for p, x, y in zip(GATE_PROMPTS, accel, cpu):
tx, ty = max(x, key=x.get), max(y, key=y.get)
agree += tx == ty
gaps += [abs(x[t] - y[t]) for t in set(x) & set(y)]
print(f" {p[:40]!r:44s} {BACKEND}: {tx!r:12s} CPU: {ty!r:12s}")
print(f"{BACKEND} vs CPU: top-1 token agrees on {agree} of {len(GATE_PROMPTS)} prompts; largest log-probability gap "
f"over shared top-10 tokens {max(gaps):.3f}")
declare("a fixed seed reproduces a sampled answer exactly", identical)
declare(f"the {BACKEND} build agrees with the CPU build on the top-1 token", agree >= len(GATE_PROMPTS) - 1)
return f"seeded answers identical: {same}; {BACKEND} vs CPU top-1 agreement {agree}/{len(GATE_PROMPTS)}"
gates_step()
Comparisons between settings are solely significant if the measurements repeat and the accelerated construct computes the identical mannequin because the reference CPU path. Two chat requests with the identical seed on the card’s sampling temperature should return an identical messages, in any other case per-question comparisons would combine mannequin behaviour with sampling noise. We then begin a CPU-only server, with no layers offloaded, and evaluate the more than likely subsequent token on 5 prompts towards the GPU or Metallic server, as a result of Dory assist has not been reviewed upstream on each backend.
def load_gsm8k():
import pandas as pd
from huggingface_hub import hf_hub_download
path = hf_hub_download("openai/gsm8k", "main/test-00000-of-00001.parquet", repo_type="dataset", revision=GSM8K_REV,
local_dir=str(WORK / "gsm8k"))
df = pd.read_parquet(path)
df["gold"] = df.reply.str.extract(r"####s*([-d,.]+)")[0].str.exchange(",", "").astype(float)
return df
def tv_distance(p, q):
"""Total-variation distance between two top-k next-token distributions, each renormalised over its own top k."""
zp, zq = sum(math.exp(v) for v in p.values()), sum(math.exp(v) for v in q.values())
return 0.5 * sum(abs(math.exp(p.get(t, -1e9)) / zp - math.exp(q.get(t, -1e9)) / zq) for t in set(p) | set(q))
@part("8. What an extra loop changes: next-token distributions on 50 prompts")
def mechanism_step():
world GSM8K
GSM8K = load_gsm8k()
prompts = listing(GSM8K.pattern(50, random_state=11).query)
configs = [(n, m, ub) for ub in (UBATCH, 16) for n in (1, 2, 4) for m in ("per_loop", "last_loop")]
dist, lengths = {}, []
for n, m, ub in configs:
with DoryServer(n, m, ubatch=ub) as s:
bought = [s.post("/completion", {"prompt": p, "n_predict": 1, "n_probs": 20, "temperature": 0, "cache_prompt": False})
for p in prompts]
dist[(n, m, ub)] = [top_probs(r) for r in got]
lengths = lengths or [r["timings"]["prompt_n"] for r in bought]
print(f"50 GSM8K questions as raw prompts, {min(lengths)} to {max(lengths)} tokens: one chunk at a {UBATCH}-token "
f"micro-batch, {math.ceil(min(lengths) / 16)} to {math.ceil(max(lengths) / 16)} chunks at 16n")
print(f"{'comparison':52s} {'same top-1':>10s} {'mean TV':>8s} ':>12s")
def evaluate(label, a, b):
x, y = dist[a], dist[b]
identical = sum(max(p, key=p.get) == max(q, key=q.get) for p, q in zip(x, y))
television = statistics.imply(tv_distance(p, q) for p, q in zip(x, y))
mx = max(abs(p[t] - q[t]) for p, q in zip(x, y) for t in set(p) & set(q))
print(f"{label:52s} {same:>7d}/50 {tv:8.4f} {mx:12.5f}")
return identical, television, mx
out = {}
out["modes_1"] = evaluate("1 loop: per_loop vs last_loop", (1, "per_loop", UBATCH), (1, "last_loop", UBATCH))
for n in (2, 4):
out[f"modes_{n}"] = evaluate(f"{n} loops: per_loop vs last_loop, one {UBATCH}-token chunk",
(n, "per_loop", UBATCH), (n, "last_loop", UBATCH))
out[f"chunks_{n}"] = evaluate(f"{n} loops: per_loop vs last_loop, 16-token chunks", (n, "per_loop", 16), (n, "last_loop", 16))
print()
out["1v2"] = evaluate("1 loop vs 2 loops (per_loop)", (1, "per_loop", UBATCH), (2, "per_loop", UBATCH))
out["2v4"] = evaluate("2 loops vs 4 loops (per_loop)", (2, "per_loop", UBATCH), (4, "per_loop", UBATCH))
out["1v4"] = evaluate("1 loop vs 4 loops (per_loop)", (1, "per_loop", UBATCH), (4, "per_loop", UBATCH))
declare("with the prompt in one chunk, last_loop gives the same next-token distribution as per_loop",
all(out[f"modes_{n}"][2] == 0 for n in (1, 2, 4)))
declare("split into small chunks, last_loop and per_loop disagree at 2 and 4 loops",
all(out[f"chunks_{n}"][1] > 0 for n in (2, 4)))
return (f"top-1 agreement 1 vs 4 loops {out['1v4'][0]}/50 (mean TV {out['1v4'][1]:.3f}); last_loop = per_loop in one chunk: "
f"{all(out[f'modes_{n}'][2] == 0 for n in (1, 2, 4))}")
mechanism_step()
Subsequent-token distributions present what a loop adjustments contained in the mannequin at nearly no price. For 50 GSM8K questions used as uncooked prompts, we request the highest 20 next-token candidates below every setting and evaluate them by top-1 settlement and total-variation distance. The fork’s code implies a pointy prediction: last_loop overwrites a recurrent layer’s cache on each loop, so it might probably differ from per_loop solely when earlier chunks of the immediate hold simply their final-loop keys, which by no means occurs when the entire immediate suits in a single micro-batch. Repeating the probe with 16-token micro-batches exams precisely that.
EVAL_CONFIGS = [(1, "per_loop"), (2, "per_loop"), (2, "last_loop"), (4, "per_loop"), (4, "last_loop")]
N_QUESTIONS = 30
MAX_TOKENS = 2048
INSTRUCTION = "nnSolve the problem. End your reply with a last line of the form 'Final answer: <number>'."
def grade(textual content, gold):
m = re.findall(r"Final answer:s*$?s*(-?[d,]*.?d+)", textual content)
nums = m or re.findall(r"-?[d,]*.?d+", textual content)
if not nums:
return False
attempt:
return abs(float(nums[-1].exchange(",", "")) - gold) < 1e-6
besides ValueError:
return False
@part("9. Paired GSM8K evaluation across loop counts and cache modes")
def eval_step():
world EVAL, SAMPLE
SAMPLE = GSM8K.pattern(N_QUESTIONS, random_state=20261106).reset_index()
EVAL = {}
for loops, mode in EVAL_CONFIGS:
path = WORK / f"gsm8k-{loops}-{mode}.jsonl"
carried out = {json.hundreds(l)["i"]: json.hundreds(l) for l in open(path)} if path.exists() else {}
todo = [i for i in range(len(SAMPLE)) if i not in done]
if todo:
with DoryServer(loops, mode) as s, open(path, "a") as f:
for i in todo:
row = SAMPLE.iloc[i]
t0 = time.time()
r = s.publish("/v1/chat/completions", {"messages": [{"role": "user", "content": row.question + INSTRUCTION}],
"max_tokens": MAX_TOKENS, "seed": 1000 + i, "cache_prompt": False})
reasoning, reply = split_reasoning(r["choices"][0]["message"])
rec = {"i": i, "correct": grade(reply, row.gold), "tokens": r["usage"]["completion_tokens"],
"finish": r["choices"][0]["finish_reason"], "wall_s": spherical(time.time() - t0, 2),
"decode_tps": spherical(r["timings"]["predicted_per_second"], 2), "answer": reply[-200:]}
f.write(json.dumps(rec) + "n")
f.flush()
carried out[i] = rec
EVAL[(loops, mode)] = [done[i] for i in vary(len(SAMPLE))]
rows = EVAL[(loops, mode)]
acc = sum(r["correct"] for r in rows)
print(f"{loops} loops {mode:9s}: {acc:2d}/{len(rows)} correct, median {statistics.median(r['tokens'] for r in rows):6.0f} "
f"tokens, {sum(r['finish'] == 'length' for r in rows)} cut off at {MAX_TOKENS}, "
f"median {statistics.median(r['wall_s'] for r in rows):5.1f} s per question")
return "; ".be part of(f"{n}/{m[:4]} {sum(r['correct'] for r in EVAL[(n, m)])}/{N_QUESTIONS}" for n, m in EVAL_CONFIGS)
eval_step()
The paired GSM8K analysis asks the identical seeded pattern of questions below 5 settings, with reasoning on, the cardboard’s sampling settings and a set seed per query, so every query meets the identical situations at each loop rely. For each reply, we file correctness towards the dataset’s closing reply, the variety of generated tokens (together with reasoning), whether or not the reply hit the two,048-token restrict, and the wall time. Outcomes go to at least one JSONL file per setting as they arrive, so an interrupted Colab session resumes the place it stopped.
def mcnemar(a, b):
"""Exact two-sided McNemar p-value from paired booleans."""
only_a = sum(x and never y for x, y in zip(a, b))
only_b = sum(y and never x for x, y in zip(a, b))
n, ok = only_a + only_b, min(only_a, only_b)
p = 1.0 if n == 0 else min(1.0, 2 * sum(math.comb(n, i) for i in vary(ok + 1)) / 2**n)
return only_a, only_b, p
def wilson(ok, n, z=1.96):
c = (ok + z * z / 2) / (n + z * z)
h = z * math.sqrt(ok * (n - ok) / n + z * z / 4) / (n + z * z)
return 100 * (c - h), 100 * (c + h)
@part("10. Statistics: accuracy, tokens and time per solved question")
def stats_step():
world STATS
rng = random.Random(7)
ref = EVAL[(1, "per_loop")]
print(f"{'setting':18s} {'accuracy':>9s} {'95% CI':>13s} {'median tokens':>14s} {'tokens vs 1 loop [95% CI]':>27s} "
f"{'s per solved':>13s}")
STATS = {}
for key in EVAL_CONFIGS:
rows = EVAL[key]
ok, n = sum(r["correct"] for r in rows), len(rows)
ratios = [r["tokens"] / q["tokens"] for r, q in zip(rows, ref)]
boots = sorted(statistics.median(rng.decisions(ratios, ok=len(ratios))) for _ in vary(2000))
per_solved = sum(r["wall_s"] for r in rows) / max(ok, 1)
STATS[key] = {"acc": ok / n, "ratio": statistics.median(ratios), "ratio_ci": (boots[50], boots[1949]), "per_solved": per_solved,
"tokens": statistics.median(r["tokens"] for r in rows)}
lo, hello = wilson(ok, n)
print(f"{key[0]} loops {key[1]:9s} {100 * k / n:8.1f}% {lo:5.1f}-{hi:5.1f}% {STATS[key]['tokens']:14.0f} "
f"{STATS[key]['ratio']:12.2f} [{boots[50]:.2f}, {boots[1949]:.2f}] {per_solved:12.1f}")
print("npaired accuracy contrasts (McNemar exact; 'only A' = questions only the first setting solved)")
pairs = [((2, "per_loop"), (1, "per_loop")), ((4, "per_loop"), (1, "per_loop")), ((4, "per_loop"), (2, "per_loop")),
((2, "last_loop"), (2, "per_loop")), ((4, "last_loop"), (4, "per_loop"))]
for a, b in pairs:
oa, ob, p = mcnemar([r["correct"] for r in EVAL[a]], [r["correct"] for r in EVAL[b]])
print(f" {a[0]} {a[1]:9s} vs {b[0]} {b[1]:9s}: only A {oa:2d}, only B {ob:2d}, p = {p:.3f}")
STATS[(a, b)] = (oa, ob, p)
fewer = STATS[(4, "per_loop")]["ratio_ci"][1] < 1
declare("on GSM8K, 4 loops need fewer generated tokens than 1 loop (95% CI of the median ratio below 1)", fewer)
return (f"accuracy {', '.join(f'{k[0]}/{k[1][:4]} {100 * STATS[k]['acc']:.0f}%' for k in EVAL_CONFIGS)}; "
f"tokens at 4 loops {STATS[(4, 'per_loop')]['ratio']:.2f}x those at 1 loop")
stats_step()
Thirty paired questions can’t settle small accuracy variations, so the statistics cell is constructed to say what the pattern can and can’t present. Accuracy comes with Wilson intervals, and every distinction makes use of an actual McNemar take a look at on the questions that just one setting solved. Token counts are in contrast because the median of per-question ratios towards one loop, with a bootstrap interval, which exams the cardboard’s declare that extra loops want fewer generated tokens. Time per solved query combines velocity, size, and accuracy into the quantity a consumer truly pays for.
@part("11. The card's claims against this run")
def claims_step():
s, sp = STATS, SPEED
rows = [
("The GGUF runs 2 loops by default", f"built-in num_loops = {META['dory.num_loops']}", META["dory.num_loops"] == 2),
("Last-loop caching keeps the KV footprint at the one-loop size",
f"{sp[(1, 'per_loop')]['kv_mib']:.0f} MiB at 1 loop, {sp[(4, 'last_loop')]['kv_mib']:.0f} at 4 last_loop",
spherical(sp[(1, "per_loop")]["kv_mib"]) == spherical(sp[(4, "last_loop")]["kv_mib"])),
("Per-loop KV memory grows with the loop count",
" / ".be part of(f"{sp[(n, 'per_loop')]['kv_mib']:.0f}" for n in LOOPS) + " MiB",
sp[(4, "per_loop")]["kv_mib"] > sp[(1, "per_loop")]["kv_mib"]),
("Loop count trades compute per token without a model swap",
f"decode {sp[(1, 'per_loop')]['decode']:.1f} -> {sp[(4, 'per_loop')]['decode']:.1f} tok/s, same file",
sp[(4, "per_loop")]["decode"] < sp[(1, "per_loop")]["decode"]),
("More loops solve tasks with fewer generated tokens",
f"GSM8K median ratio at 4 loops {s[(4, 'per_loop')]['ratio']:.2f} [{s[(4, 'per_loop')]['ratio_ci'][0]:.2f}, "
f"{s[(4, 'per_loop')]['ratio_ci'][1]:.2f}]", s[(4, "per_loop")]["ratio_ci"][1] < 1),
("Last-loop accuracy stays close to per-loop",
f"4 loops: {100 * s[(4, 'last_loop')]['acc']:.0f}% vs {100 * s[(4, 'per_loop')]['acc']:.0f}%, "
f"McNemar p = {s[((4, 'last_loop'), (4, 'per_loop'))][2]:.2f}", s[((4, "last_loop"), (4, "per_loop"))][2] > 0.05),
]
print(f"{'claim (model card)':60s} {'this run':48s} verdict")
for textual content, discovered, okay in rows:
print(f"{text:60s} {found:48s} {'holds' if ok else 'does not hold'}")
declare(textual content, okay)
print(f"nnot testable here: the benchmark table (SWE-bench, Terminal-Bench, GPQA ...), the 256K context, NVFP4 pre-training;"
f"nwith {N_QUESTIONS} paired questions only large accuracy differences are resolvable")
return f"{sum(bool(ok) for *_, ok in rows)} of {len(rows)} card claims hold in this run"
claims_step()
Reasonably than restate the mannequin card, we examine its claims towards this run. Every row pairs a sentence from the cardboard with the measurement that exams it: cache-size claims and the loop defaults to token effectivity and the accuracy of last-loop caching, and the decision comes from the saved outcomes slightly than being typed by hand. Claims the pocket book can’t attain, such because the benchmark desk, the 256K context and the NVFP4 pre-training, are listed as untested slightly than quietly dropped.
banner("SUMMARY")
for title, res in RESULTS.objects():
print(f" {name:<100s} {res}")
print("nchecks decided in this run:")
for step, textual content, held in CLAIMS:
print(f" [{'held' if held else 'did not hold'}] step {step}: {text}")
failed = [k for k, v in RESULTS.items() if str(v).startswith("SKIPPED")]
print(f"n{sum(h for *_, h in CLAIMS)} of {len(CLAIMS)} checks held; steps failed or skipped: {failed or 'none'}")
print(f"llama.cpp fork {FORK_COMMIT[:12]}, GGUF @ {GGUF_REV[:7]}, backend {BACKEND}{' ' + GPU if GPU else ''}, "
f"Python {platform.python_version()}")
The abstract prints every step’s one-line outcome and each examine the run determined, marking which handed and which failed, together with the fork commit, the GGUF revision, and the backend, so the pocket book doubles as a file of precisely what was verified and on which {hardware}.
In conclusion, Dory turns test-time compute right into a load-time setting: one 3.7 GB file runs as 4 totally different fashions, from a single cross by its 20-block recurrent core to 4, and the GGUF header alone predicts what every setting prices in parameters and cache reminiscence. The pocket book checks these predictions towards the server, measures velocity, exams the last-loop cache towards a exact prediction from the code, and evaluates accuracy and reply size on a paired GSM8K pattern. Two sensible classes maintain regardless of the measurements present: at all times set the loop rely explicitly, as a result of the GGUF defaults to 4, and pin the fork commit, as a result of Dory assist in llama.cpp remains to be an open draft.
Take a look at the FULL CODES here. Additionally, be at liberty to comply with us on Twitter and don’t neglect to affix our 150k+ML SubReddit and Subscribe to our Newsletter. Wait! are you on telegram? now you can join us on telegram as well.
Must associate with us for selling your GitHub Repo OR Hugging Face Web page OR Product Launch OR Webinar and many others.? Connect with us
The publish How to Run NVIDIA Dory 4B with llama.cpp: Loop by Loop Guide appeared first on MarkTechPost.

