Qwen3-Coder 30B on a Surface Laptop Ultra: run it on RTX Spark with Ollama

tokens per second ≈ memory bandwidth ÷ bytes read per token

# Ollama version of the speed benchmark from Microsoft's "Qwen3.8-27B on a Lenovo laptop" post:
#  - code benchmark: three code-generation prompts, 512-token replies, temperature 0, thinking off, one at a time
#  - long-prompt benchmark: one question after a long prompt whose random first line defeats the prefix cache, 384-token reply
# GPU power, temperature, clock and memory are sampled every second with nvidia-smi during the whole run.
# Writes speed-bench-<COMPUTER>-<stamp>.json and .gpu.csv in the current directory.
param(
    [string[]]$Models = @('qwen3:8b', 'qwen3-coder:30b'),
    [string]$Api = 'http://localhost:11434'
)
$ErrorActionPreference = 'Stop'

# Context per model: big enough for the long prompt plus reply, within the model's maximum and the GPU's memory
$plan = @{
    'qwen3:8b'        = @{ num_ctx = 40960;  long_tokens = 36000 }   # qwen3:8b's trained maximum is 40,960
    'qwen3-coder:30b' = @{ num_ctx = 102400; long_tokens = 97880 }   # the article's long prompt was 97,880 tokens
}
$codePrompts = @(
    'Write a thread-safe LRU cache in Python with unit tests.',
    'Write a token-bucket rate limiter in TypeScript.',
    "Implement Dijkstra's shortest-path algorithm in Go."
)

function Post-Ollama($path, $body) {
    Invoke-RestMethod -Method Post -Uri ($Api + $path) -ContentType 'application/json' -Body ($body | ConvertTo-Json -Depth 8) -TimeoutSec 1800
}
function Unload-All {
    foreach ($m in @((Invoke-RestMethod ($Api + '/api/ps')).models)) { Post-Ollama '/api/generate' @{ model = $m.name; keep_alive = 0 } | Out-Null }
    Start-Sleep 3
}
function Measure-Reply($r, $wall) {
    [ordered]@{
        prompt_tokens       = $r.prompt_eval_count
        prefill_tok_s       = if ($r.prompt_eval_duration) { [math]::Round($r.prompt_eval_count / ($r.prompt_eval_duration / 1e9), 1) } else { $null }
        output_tokens       = $r.eval_count
        decode_tok_s        = if ($r.eval_duration) { [math]::Round($r.eval_count / ($r.eval_duration / 1e9), 2) } else { $null }
        first_token_s       = [math]::Round(($r.load_duration + $r.prompt_eval_duration) / 1e9, 2)
        wall_s              = [math]::Round($wall, 2)
        done_reason         = $r.done_reason
    }
}
function Invoke-Timed($body) {
    $sw = [Diagnostics.Stopwatch]::StartNew(); $r = Post-Ollama '/api/generate' $body; $sw.Stop()
    return $r, $sw.Elapsed.TotalSeconds
}

# Text for the long prompt: PowerShell's own format/type files and module sources, present on every Windows machine
$corpus = (Get-ChildItem "$env:windir\System32\WindowsPowerShell\v1.0" -Recurse -File -Include *.ps1xml, *.psm1, *.psd1 -ErrorAction SilentlyContinue |
    Sort-Object FullName | ForEach-Object { "`n### FILE: $($_.Name)`n" + [IO.File]::ReadAllText($_.FullName) }) -join ''

$stamp = Get-Date -Format 'yyyyMMdd-HHmmss'
$base = Join-Path (Get-Location) "speed-bench-$env:COMPUTERNAME-$stamp"
$gpuCsv = "$base.gpu.csv"
$gpuInfo = (& nvidia-smi -i 0 --query-gpu=name,driver_version,memory.total --format=csv,noheader).Trim()
$sampler = Start-Process nvidia-smi -ArgumentList '-i 0 --query-gpu=timestamp,power.draw,temperature.gpu,clocks.gr,memory.used --format=csv,noheader,nounits -l 1 -f', "`"$gpuCsv`"" -WindowStyle Hidden -PassThru
$phases = @()
$results = @()

try {
    foreach ($model in $Models) {
        $p = $plan[$model]
        if (-not $p) { $p = @{ num_ctx = 32768; long_tokens = 30000 } }
        $caps = @((Post-Ollama '/api/show' @{ model = $model }).capabilities)
        $opts = { param($n) @{ num_ctx = $p.num_ctx; num_predict = $n; temperature = 0; seed = 42 } }
        function New-Body($prompt, $n) {
            $b = @{ model = $model; prompt = $prompt; stream = $false; keep_alive = '15m'; options = (& $opts $n) }
            if ($caps -contains 'thinking') { $b.think = $false }   # thinking off, as in the article; non-thinking models reject the flag
            $b
        }

        Write-Host "=== $model (num_ctx $($p.num_ctx)) ==="
        Unload-All
        $t0 = Get-Date
        $r, $w = Invoke-Timed (New-Body 'Reply with exactly: warm' 8)
        $loaded = @((Invoke-RestMethod ($Api + '/api/ps')).models | Where-Object name -eq $model)[0]
        $phases += [pscustomobject]@{ model = $model; phase = 'load'; start = $t0; end = Get-Date }
        $modelInfo = [ordered]@{
            model = $model; num_ctx = $p.num_ctx; loaded_context = $loaded.context_length
            load_s = [math]::Round($r.load_duration / 1e9, 1)
            size_gib = [math]::Round($loaded.size / 1GB, 1)
            gpu_percent = if ($loaded.size) { [math]::Round(100 * $loaded.size_vram / $loaded.size) } else { $null }
        }

        # Code benchmark
        $t0 = Get-Date
        $code = foreach ($cp in $codePrompts) {
            $r, $w = Invoke-Timed (New-Body $cp 512)
            $m = Measure-Reply $r $w; $m.prompt = $cp
            Write-Host ("  code: {0} tok/s decode, {1} tokens" -f $m.decode_tok_s, $m.output_tokens)
            [pscustomobject]$m
        }
        $phases += [pscustomobject]@{ model = $model; phase = 'code'; start = $t0; end = Get-Date }

        # Long-prompt benchmark: calibrate characters per token, then build the prompt just under the target
        # A large slice: token density varies a lot between files, and Ollama silently cuts prompts longer than num_ctx
        $sample = $corpus.Substring(0, [math]::Min($p.num_ctx * 3, $corpus.Length))
        $r, $w = Invoke-Timed (New-Body ("Calibration $(Get-Random)`n" + $sample) 1)
        $charsPerToken = $sample.Length / $r.prompt_eval_count
        $chars = [int]($p.long_tokens * $charsPerToken * 0.97)
        $text = $corpus
        while ($text.Length -lt $chars) { $text += $corpus }
        $question = "`n`nQuestion: Describe what the files above define, as a numbered list covering every file."
        $longPrompt = "Session $([guid]::NewGuid())`n" + $text.Substring(0, $chars) + $question
        Start-Sleep 5
        $t0 = Get-Date
        $r, $w = Invoke-Timed (New-Body $longPrompt 384)
        $long = Measure-Reply $r $w
        $long.chars_per_token = [math]::Round($charsPerToken, 2)
        # Ollama keeps only the end of an over-long prompt (about half the context); a short count means that happened
        $long.truncated = $r.prompt_eval_count -lt 0.8 * $p.long_tokens
        if ($long.truncated) { Write-Warning "$model long prompt was truncated to $($r.prompt_eval_count) tokens" }
        $phases += [pscustomobject]@{ model = $model; phase = 'long'; start = $t0; end = Get-Date }
        Write-Host ("  long: {0} prompt tokens, {1} tok/s prefill, {2} tok/s decode, {3} s to first token" -f $long.prompt_tokens, $long.prefill_tok_s, $long.decode_tok_s, $long.first_token_s)

        $modelInfo.code = $code
        $modelInfo.code_mean_decode_tok_s = [math]::Round(($code | Measure-Object decode_tok_s -Average).Average, 2)
        $modelInfo.long = [pscustomobject]$long
        $results += [pscustomobject]$modelInfo
        Start-Sleep 5
    }
}
finally {
    Stop-Process -Id $sampler.Id -ErrorAction SilentlyContinue
    Unload-All
}

# GPU summary per phase from the nvidia-smi samples
$samples = Get-Content $gpuCsv | Where-Object { $_ -match '^\d{4}/' } | ForEach-Object {
    $f = $_.Split(',') | ForEach-Object { $_.Trim() }
    [pscustomobject]@{
        time = [datetime]::ParseExact($f[0], 'yyyy/MM/dd HH:mm:ss.fff', $null)
        power_w = [double]$f[1]; temp_c = [int]$f[2]; clock_mhz = [int]$f[3]; mem_mib = [int]$f[4]
    }
}
$gpuPhases = foreach ($ph in $phases) {
    $s = @($samples | Where-Object { $_.time -ge $ph.start -and $_.time -le $ph.end })
    if (-not $s.Count) { continue }
    [pscustomobject][ordered]@{
        model = $ph.model; phase = $ph.phase; seconds = [math]::Round(($ph.end - $ph.start).TotalSeconds, 1)
        avg_power_w = [math]::Round(($s | Measure-Object power_w -Average).Average, 1)
        max_power_w = ($s | Measure-Object power_w -Maximum).Maximum
        max_temp_c = ($s | Measure-Object temp_c -Maximum).Maximum
        avg_clock_mhz = [math]::Round(($s | Measure-Object clock_mhz -Average).Average)
        max_mem_mib = ($s | Measure-Object mem_mib -Maximum).Maximum
    }
}

$battery = Get-CimInstance Win32_Battery -ErrorAction SilentlyContinue | Select-Object -First 1
$out = [ordered]@{
    computer = $env:COMPUTERNAME; date = (Get-Date).ToString('s')
    ollama_version = (Invoke-RestMethod ($Api + '/api/version')).version
    gpu = $gpuInfo
    power_scheme = (powercfg /getactivescheme) -replace '^.*\((.*)\)\s*$', '$1'
    on_ac_power = if ($battery) { $battery.BatteryStatus -eq 2 } else { $null }
    models = $results
    gpu_phases = $gpuPhases
}
$json = $out | ConvertTo-Json -Depth 8
[IO.File]::WriteAllText("$base.json", $json, (New-Object Text.UTF8Encoding $false))
$gpuPhases | Format-Table -AutoSize | Out-String | Write-Host
Write-Host "Results: $base.json"
Write-Host "GPU samples: $gpuCsv"