{ "schema_version": "gb10-benchmark-public-v1", "release_id": "gb10-llm-benchmark-v1-20260906", "generated_at": "2026-09-06", "title": "GB10 LLM Benchmark v1 통합 재검증 결과", "status": "published-candidate", "scope": { "hardware": "NVIDIA DGX Spark GB10", "runtime": "llama.cpp", "reasoning_mode": "off", "raw_runs_public": false, "model_variant_count": 18, "suite_count": 7, "revalidated_evaluator_runs": 72, "reused_source_runs": 54, "source_run_references": 126 }, "suite_versions": { "performance": "existing-v1", "server_performance": "existing-v1", "knowledge": "1.2", "coding": "existing-v1", "tool_call": "1.1", "agent_single": "1.1", "agent_multi": "1.1" }, "methodology": { "evaluator_versions": { "performance": "existing-v1", "server_performance": "existing-v1", "knowledge": "1.2", "coding": "existing-v1", "tool_call": "1.1", "agent_single": "1.1", "agent_multi": "1.1" }, "datasets": [ { "id": "knowledge-v1", "version": "legacy v1", "public_path": "datasets/knowledge-v1/knowledge-v1.jsonl", "sha256": "44883816f2b11f8560c69a6c28933c2496ea137a5b98860dd102d397349f650c", "scoring_type": "deterministic normalized answer matching", "reasoning": "off" }, { "id": "knowledge-v1.2", "version": "canonical Knowledge evaluator", "public_path": "datasets/knowledge-v1.2/knowledge-v1.2.jsonl", "sha256": "66e691da99a416fdf59887930d40c0dd7d12538b9f81b818e15e5bd0f6584079", "scoring_type": "deterministic normalized answer matching", "reasoning": "off" }, { "id": "coding-v1", "version": "existing Coding v1", "public_path": "datasets/coding-v1/coding-v1.jsonl", "sha256": "53a283316be1ab63ee04c051d5194decbc3c7e27b76e5c177293bcab87dccbbd", "scoring_type": "pytest execution", "reasoning": "off" }, { "id": "tool-call-v1.1", "version": "canonical Tool-call evaluator", "public_path": "datasets/tool-call-v1/tool-call-v1.jsonl", "sha256": "8c0c1abefa2825c97341afaee620adbbf60cf43add20e0535c599872733065d5", "scoring_type": "deterministic tool/argument/execution", "reasoning": "off" }, { "id": "agent-single-v1.1", "version": "canonical Agent-single evaluator", "public_path": "datasets/agent-single-v1/agent-single-v1.jsonl", "sha256": "188a63368a0631cc6cc6a80a8eed165f559f8131b5196ba28a951c428479b5d5", "scoring_type": "deterministic completion/step/tool", "reasoning": "off" }, { "id": "agent-multi-v1.1", "version": "canonical Agent-multi evaluator", "public_path": "datasets/agent-multi-v1/agent-multi-v1.jsonl", "sha256": "ee9bc5035f9ca17be36a3749e62322d767232570960ae80d3041b2b1741a010a", "scoring_type": "deterministic role/handoff/completion", "reasoning": "off" } ], "limitations": [ "Performance and server-performance are reused historical runs, while Knowledge/Tool-call/Agent suites use the 2026-09-06 revalidation lane.", "Scores are fixed-harness observations, not universal product quality or safety scores.", "Agent-multi is a role/tool handoff evaluator in this release, not an independent-agent deployment benchmark.", "Raw runs, raw prompts/responses, and local model artifacts are not public." ] }, "source_policy": { "revalidated_lane": "Knowledge 1.2, Tool-call 1.1, Agent-single 1.1, Agent-multi 1.1", "reused_lane": "Existing Performance, Server-performance, and Coding v1 runs", "source_run_id_semantics": "Each suite points to the exact local run used for this public projection." }, "models": [ { "model_id": "ornith-1-5-q5-k-m", "model": "Ornith 1.5", "variant": "Q5_K_M", "quantization": "Q5_K_M", "benchmark_versions": { "performance": "existing-v1", "server_performance": "existing-v1", "knowledge": "1.2", "coding": "existing-v1", "tool_call": "1.1", "agent_single": "1.1", "agent_multi": "1.1" }, "suites": { "performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-131958-3cfdfd", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_type_k": "f16", "cache_type_v": "f16", "flash_attn": true, "gen_tokens": [ 512 ], "n_gpu_layers": 999, "prompt_tokens": [ 512, 2048, 8192, 32768 ], "repetitions": 5, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "metrics": { "pp": { "token_count": 32768, "mean_tps": 1634.94996, "stddev_tps": 28.21344, "repetitions": 5, "unit": "tokens/s" }, "tg": { "token_count": 512, "mean_tps": 64.545895, "stddev_tps": 1.149455, "repetitions": 5, "unit": "tokens/s" } }, "repetitions": 5 }, "server_performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-132350-d86bcf", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_prompt": false, "cache_type_k": "f16", "cache_type_v": "f16", "concurrencies": [ 1, 2, 4, 8 ], "ctx_size": 8192, "flash_attn": true, "gen_tokens": 512, "n_gpu_layers": 999, "parallel": 8, "repetitions": 3, "request_timeout_s": 300, "requests_per_concurrency": 4, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "formal": true, "concurrencies": [ 1, 2, 4, 8 ], "requests_per_concurrency": 4, "repetitions": 3, "gen_tokens": 512, "samples_count": 180, "conditions": [ { "concurrency": 1, "repetitions": 3, "requests_per_repetition": 4, "successful_requests": 12, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 68.33767884023186, "stddev": 1.2555456652906163, "min": 67.00320269313808, "max": 69.49560518270145 }, "per_request_generation_throughput_tps": { "mean": 68.34538901178355, "stddev": 1.2480926992219137, "min": 65.97346155791323, "max": 70.19380292511086 }, "latency_p50_s": { "mean": 7.487469965331305, "stddev": 0.12327183528434356, "min": 7.373318388999905, "max": 7.618186832998617 }, "latency_p95_s": { "mean": 7.574939027749254, "stddev": 0.15785404917718465, "min": 7.426039278853932, "max": 7.740435864945903 }, "latency_p99_s": { "mean": 7.5824458175495, "stddev": 0.16537134131278589, "min": 7.427605439774416, "max": 7.756644224985648 } }, { "concurrency": 2, "repetitions": 3, "requests_per_repetition": 8, "successful_requests": 24, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 92.76442346945097, "stddev": 1.435845550580574, "min": 91.89731618822645, "max": 94.42180151219681 }, "per_request_generation_throughput_tps": { "mean": 46.760522703476866, "stddev": 1.3563699373368017, "min": 44.86374358175412, "max": 49.80848185877438 }, "latency_p50_s": { "mean": 11.032101824666219, "stddev": 0.2042295358862432, "min": 10.797707159999845, "max": 11.171750419998716 }, "latency_p95_s": { "mean": 11.251250208982794, "stddev": 0.15734755175391385, "min": 11.079705378347716, "max": 11.38886402044991 }, "latency_p99_s": { "mean": 11.270934069262706, "stddev": 0.15595665866608074, "min": 11.101061924468668, "max": 11.40763953608919 } }, { "concurrency": 4, "repetitions": 3, "requests_per_repetition": 16, "successful_requests": 48, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 129.08793060216533, "stddev": 1.2199357370394341, "min": 127.86248274304725, "max": 130.30227846820225 }, "per_request_generation_throughput_tps": { "mean": 32.623662563935184, "stddev": 1.9523819515588845, "min": 28.048499080265252, "max": 36.7360155722831 }, "latency_p50_s": { "mean": 15.90993307283255, "stddev": 0.2622586886411756, "min": 15.608482866995473, "max": 16.08566717050053 }, "latency_p95_s": { "mean": 16.95501932458319, "stddev": 0.31353457616305724, "min": 16.674913328750335, "max": 17.29371623449697 }, "latency_p99_s": { "mean": 17.230368366785477, "stddev": 0.7328102643301873, "min": 16.67934884015085, "max": 18.06202019330085 } }, { "concurrency": 8, "repetitions": 3, "requests_per_repetition": 32, "successful_requests": 96, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 164.05800885558978, "stddev": 2.9027068066576223, "min": 161.4146801155931, "max": 167.16441877738174 }, "per_request_generation_throughput_tps": { "mean": 21.110228230955702, "stddev": 1.8601325838510172, "min": 18.005247091919983, "max": 25.383993097051235 }, "latency_p50_s": { "mean": 24.919508402498952, "stddev": 0.41720352773995595, "min": 24.448869509498763, "max": 25.243895563999104 }, "latency_p95_s": { "mean": 27.05375105841655, "stddev": 1.0025382930838813, "min": 25.952491096498854, "max": 27.913416476351994 }, "latency_p99_s": { "mean": 27.311439787290436, "stddev": 1.1515448134751984, "min": 26.06019449536856, "max": 28.326734567261663 } } ] }, "knowledge": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-012545-d5c2ab", "evaluator_version": "1.2", "condition": { "dataset_version": 1.2, "max_tokens": 128, "request_timeout_s": 300, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.2, "total": 100, "correct": 92, "accuracy": 0.92, "pass_rate": 0.92, "categories": { "general": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "korea": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "math": { "total": 20, "correct": 15, "accuracy": 0.75, "pass_rate": 0.75 }, "science": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "logic": { "total": 20, "correct": 17, "accuracy": 0.85, "pass_rate": 0.85 } }, "difficulties": { "easy": { "total": 25, "correct": 25, "accuracy": 1.0, "pass_rate": 1.0 }, "medium": { "total": 40, "correct": 37, "accuracy": 0.925, "pass_rate": 0.925 }, "hard": { "total": 35, "correct": 30, "accuracy": 0.8571428571428571, "pass_rate": 0.8571428571428571 } } }, "coding": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-133625-26bbb8", "evaluator_version": "existing-v1", "condition": { "dataset_version": 1, "request_timeout_s": 300.0, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": -1, "thinking_mode": "auto" }, "version": 1, "total": 12, "passed": 9, "pass_rate": 0.75, "categories": { "basic": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "string": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "collection": { "total": 3, "passed": 1, "pass_rate": 0.3333333333333333 }, "edge_case": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 } }, "failure_types": { "runtime_error": 1, "syntax_error": 2 } }, "tool_call": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-005207-3d30c2", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 15, "passed": 11, "pass_rate": 0.7333333333333333, "tool_selection_accuracy": 0.9333333333333333, "argument_accuracy": 0.8, "execution_success_rate": 0.8666666666666667, "tool_rate": 0.9333333333333333, "args_rate": 0.8, "exec_rate": 0.8666666666666667, "failure_types": { "wrong_arguments": 2, "execution_error": 1, "wrong_tool": 1 } }, "agent_single": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-005233-6b715b", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 12, "passed": 8, "correct": 8, "pass_rate": 0.6666666666666666, "avg_steps": 3.3333333333333335, "avg_tool_calls": 2.3333333333333335, "tool_calls_total": 28, "failed_tool_calls": 4, "repeated_tool_calls": 4, "failure_types": { "tool_error": 2, "missing_required_step": 2 } }, "agent_multi": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-005319-5297e7", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 8, "max_tokens": 256, "request_timeout_s": 300, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 10, "passed": 8, "correct": 8, "pass_rate": 0.8, "handoff_success_rate": 1.0, "role_participation_success_rate": 1.0, "avg_steps": 3.5, "avg_tool_calls": 2.5, "tool_calls_total": 25, "failed_tool_calls": 3, "invalid_tool_calls": 0, "handoffs": 11, "failure_types": { "missing_required_step": 2 } } }, "provenance": { "hardware": "NVIDIA DGX Spark GB10", "reasoning_mode": "off", "raw_artifacts_public": false } }, { "model_id": "ornith-1-5-q6-k", "model": "Ornith 1.5", "variant": "Q6_K", "quantization": "Q6_K", "benchmark_versions": { "performance": "existing-v1", "server_performance": "existing-v1", "knowledge": "1.2", "coding": "existing-v1", "tool_call": "1.1", "agent_single": "1.1", "agent_multi": "1.1" }, "suites": { "performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-134111-76e244", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_type_k": "f16", "cache_type_v": "f16", "flash_attn": true, "gen_tokens": [ 512 ], "n_gpu_layers": 999, "prompt_tokens": [ 512, 2048, 8192, 32768 ], "repetitions": 5, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "metrics": { "pp": { "token_count": 32768, "mean_tps": 1432.075212, "stddev_tps": 17.219698, "repetitions": 5, "unit": "tokens/s" }, "tg": { "token_count": 512, "mean_tps": 57.901217, "stddev_tps": 0.894029, "repetitions": 5, "unit": "tokens/s" } }, "repetitions": 5 }, "server_performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-134533-c5b64a", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_prompt": false, "cache_type_k": "f16", "cache_type_v": "f16", "concurrencies": [ 1, 2, 4, 8 ], "ctx_size": 8192, "flash_attn": true, "gen_tokens": 512, "n_gpu_layers": 999, "parallel": 8, "repetitions": 3, "request_timeout_s": 300, "requests_per_concurrency": 4, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "formal": true, "concurrencies": [ 1, 2, 4, 8 ], "requests_per_concurrency": 4, "repetitions": 3, "gen_tokens": 512, "samples_count": 180, "conditions": [ { "concurrency": 1, "repetitions": 3, "requests_per_repetition": 4, "successful_requests": 12, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 62.40177855629509, "stddev": 0.2494938612528866, "min": 62.14159322181844, "max": 62.638992092626836 }, "per_request_generation_throughput_tps": { "mean": 62.40929693442324, "stddev": 0.6537628629761182, "min": 61.261633639149416, "max": 63.91123478047478 }, "latency_p50_s": { "mean": 8.193539806498544, "stddev": 0.04025836800561985, "min": 8.162664861996745, "max": 8.239073616499809 }, "latency_p95_s": { "mean": 8.284897822481678, "stddev": 0.041046658549896836, "min": 8.250620475499819, "max": 8.330384790295648 }, "latency_p99_s": { "mean": 8.296653103161468, "stddev": 0.05088434916633981, "min": 8.252200641499584, "max": 8.352153963655729 } }, { "concurrency": 2, "repetitions": 3, "requests_per_repetition": 8, "successful_requests": 24, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 85.52228155296592, "stddev": 0.41061530391798867, "min": 85.16379756103989, "max": 85.97026477848117 }, "per_request_generation_throughput_tps": { "mean": 42.94414470141386, "stddev": 1.4327036838570895, "min": 39.73522891206207, "max": 45.9414481021232 }, "latency_p50_s": { "mean": 11.925163934833108, "stddev": 0.08351746906964408, "min": 11.870592042498174, "max": 12.021309151001333 }, "latency_p95_s": { "mean": 12.457034810881062, "stddev": 0.2904132099270711, "min": 12.122815656248713, "max": 12.647872009396451 }, "latency_p99_s": { "mean": 12.551362327239525, "stddev": 0.34226221487129144, "min": 12.168721915247152, "max": 12.828316405396981 } }, { "concurrency": 4, "repetitions": 3, "requests_per_repetition": 16, "successful_requests": 48, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 116.2352243146355, "stddev": 0.9509553636040772, "min": 115.25305602977048, "max": 117.15154391974879 }, "per_request_generation_throughput_tps": { "mean": 29.539560182061145, "stddev": 1.8626301212096306, "min": 26.616641093132888, "max": 34.36680009785428 }, "latency_p50_s": { "mean": 17.52692246149915, "stddev": 0.027222226557863917, "min": 17.502883964498324, "max": 17.556482023999706 }, "latency_p95_s": { "mean": 18.798018955833566, "stddev": 0.152143120168426, "min": 18.632613298501383, "max": 18.93198829799985 }, "latency_p99_s": { "mean": 19.01587590103433, "stddev": 0.2050070345698494, "min": 18.784609305297636, "max": 19.175266546804416 } }, { "concurrency": 8, "repetitions": 3, "requests_per_repetition": 32, "successful_requests": 96, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 154.51043736214484, "stddev": 1.5874163593056727, "min": 152.82593269413883, "max": 155.97856805125429 }, "per_request_generation_throughput_tps": { "mean": 19.718604172206213, "stddev": 1.7442991183143972, "min": 16.018219916737678, "max": 23.438923649951278 }, "latency_p50_s": { "mean": 26.48932683450039, "stddev": 0.4897341370199569, "min": 25.923906131502008, "max": 26.78004321049957 }, "latency_p95_s": { "mean": 29.107577623650286, "stddev": 0.2562806950115407, "min": 28.948108750900793, "max": 29.403198728449933 }, "latency_p99_s": { "mean": 30.147006194449812, "stddev": 0.8961297426863651, "min": 29.245616580479183, "max": 31.037781776516862 } } ] }, "knowledge": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-012641-3397c7", "evaluator_version": "1.2", "condition": { "dataset_version": 1.2, "max_tokens": 128, "request_timeout_s": 300, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.2, "total": 100, "correct": 94, "accuracy": 0.94, "pass_rate": 0.94, "categories": { "general": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "korea": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "math": { "total": 20, "correct": 16, "accuracy": 0.8, "pass_rate": 0.8 }, "science": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "logic": { "total": 20, "correct": 18, "accuracy": 0.9, "pass_rate": 0.9 } }, "difficulties": { "easy": { "total": 25, "correct": 25, "accuracy": 1.0, "pass_rate": 1.0 }, "medium": { "total": 40, "correct": 39, "accuracy": 0.975, "pass_rate": 0.975 }, "hard": { "total": 35, "correct": 30, "accuracy": 0.8571428571428571, "pass_rate": 0.8571428571428571 } } }, "coding": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-135918-37492a", "evaluator_version": "existing-v1", "condition": { "dataset_version": 1, "request_timeout_s": 300.0, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": -1, "thinking_mode": "auto" }, "version": 1, "total": 12, "passed": 9, "pass_rate": 0.75, "categories": { "basic": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "string": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "collection": { "total": 3, "passed": 1, "pass_rate": 0.3333333333333333 }, "edge_case": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 } }, "failure_types": { "syntax_error": 2, "test_failure": 1 } }, "tool_call": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-005517-5ceb4f", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 15, "passed": 11, "pass_rate": 0.7333333333333333, "tool_selection_accuracy": 1.0, "argument_accuracy": 0.8666666666666667, "execution_success_rate": 0.8666666666666667, "tool_rate": 1.0, "args_rate": 0.8666666666666667, "exec_rate": 0.8666666666666667, "failure_types": { "wrong_arguments": 2, "execution_error": 2 } }, "agent_single": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-005548-c7c4b3", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 12, "passed": 10, "correct": 10, "pass_rate": 0.8333333333333334, "avg_steps": 3.3333333333333335, "avg_tool_calls": 2.3333333333333335, "tool_calls_total": 28, "failed_tool_calls": 2, "repeated_tool_calls": 4, "failure_types": { "missing_required_step": 2 } }, "agent_multi": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-005634-c44293", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 8, "max_tokens": 256, "request_timeout_s": 300, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 10, "passed": 8, "correct": 8, "pass_rate": 0.8, "handoff_success_rate": 1.0, "role_participation_success_rate": 1.0, "avg_steps": 3.6, "avg_tool_calls": 2.6, "tool_calls_total": 26, "failed_tool_calls": 3, "invalid_tool_calls": 0, "handoffs": 11, "failure_types": { "final_answer_error": 1, "missing_required_step": 1 } } }, "provenance": { "hardware": "NVIDIA DGX Spark GB10", "reasoning_mode": "off", "raw_artifacts_public": false } }, { "model_id": "ornith-1-5-q8-0", "model": "Ornith 1.5", "variant": "Q8_0", "quantization": "Q8_0", "benchmark_versions": { "performance": "existing-v1", "server_performance": "existing-v1", "knowledge": "1.2", "coding": "existing-v1", "tool_call": "1.1", "agent_single": "1.1", "agent_multi": "1.1" }, "suites": { "performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-140444-106393", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_type_k": "f16", "cache_type_v": "f16", "flash_attn": true, "gen_tokens": [ 512 ], "n_gpu_layers": 999, "prompt_tokens": [ 512, 2048, 8192, 32768 ], "repetitions": 5, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "metrics": { "pp": { "token_count": 32768, "mean_tps": 1726.666138, "stddev_tps": 7.450603, "repetitions": 5, "unit": "tokens/s" }, "tg": { "token_count": 512, "mean_tps": 57.79458, "stddev_tps": 0.276795, "repetitions": 5, "unit": "tokens/s" } }, "repetitions": 5 }, "server_performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-140846-2c88a0", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_prompt": false, "cache_type_k": "f16", "cache_type_v": "f16", "concurrencies": [ 1, 2, 4, 8 ], "ctx_size": 8192, "flash_attn": true, "gen_tokens": 512, "n_gpu_layers": 999, "parallel": 8, "repetitions": 3, "request_timeout_s": 300, "requests_per_concurrency": 4, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "formal": true, "concurrencies": [ 1, 2, 4, 8 ], "requests_per_concurrency": 4, "repetitions": 3, "gen_tokens": 512, "samples_count": 180, "conditions": [ { "concurrency": 1, "repetitions": 3, "requests_per_repetition": 4, "successful_requests": 12, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 60.34894015597344, "stddev": 0.22133010494925728, "min": 60.15648753367686, "max": 60.59079787336786 }, "per_request_generation_throughput_tps": { "mean": 60.35445516958323, "stddev": 0.5414998846339495, "min": 59.61046213459626, "max": 61.099706674496716 }, "latency_p50_s": { "mean": 8.489768419331085, "stddev": 0.041193456040039586, "min": 8.44544011149992, "max": 8.52687083499768 }, "latency_p95_s": { "mean": 8.54572135723441, "stddev": 0.0629419944723729, "min": 8.473528978253308, "max": 8.589090096549262 }, "latency_p99_s": { "mean": 8.548377413314714, "stddev": 0.0619649833341334, "min": 8.477065532453416, "max": 8.58909506250995 } }, { "concurrency": 2, "repetitions": 3, "requests_per_repetition": 8, "successful_requests": 24, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 84.16253475127034, "stddev": 0.3286403543336648, "min": 83.86923494816179, "max": 84.51771738795011 }, "per_request_generation_throughput_tps": { "mean": 42.13017001057737, "stddev": 0.7021682550348329, "min": 40.50835646741657, "max": 43.11831328751064 }, "latency_p50_s": { "mean": 12.074959968000863, "stddev": 0.03241727604967999, "min": 12.041590904500481, "max": 12.106333236501087 }, "latency_p95_s": { "mean": 12.488610776266917, "stddev": 0.06611899733124374, "min": 12.428917882801397, "max": 12.55967967635188 }, "latency_p99_s": { "mean": 12.552821523521464, "stddev": 0.06279962414049438, "min": 12.503212089361696, "max": 12.623429852074114 } }, { "concurrency": 4, "repetitions": 3, "requests_per_repetition": 16, "successful_requests": 48, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 117.31759093774491, "stddev": 1.2151611792853778, "min": 116.5017320889232, "max": 118.71415449354195 }, "per_request_generation_throughput_tps": { "mean": 29.823398688483, "stddev": 1.9638801039865945, "min": 26.839729055092626, "max": 35.18345804224202 }, "latency_p50_s": { "mean": 17.32417635483095, "stddev": 0.16000964714110016, "min": 17.229391523494996, "max": 17.5089185575016 }, "latency_p95_s": { "mean": 18.535950775664485, "stddev": 0.1894096222402098, "min": 18.335092756498852, "max": 18.711333418246795 }, "latency_p99_s": { "mean": 18.865028194066205, "stddev": 0.19420564145217634, "min": 18.64082374170139, "max": 18.98101678444873 } }, { "concurrency": 8, "repetitions": 3, "requests_per_repetition": 32, "successful_requests": 96, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 158.08451366055377, "stddev": 3.9258547215499657, "min": 153.55216866404473, "max": 160.4263295271946 }, "per_request_generation_throughput_tps": { "mean": 20.2610006724516, "stddev": 2.1514228003243714, "min": 16.003605216667147, "max": 24.85829750358861 }, "latency_p50_s": { "mean": 26.224168289832352, "stddev": 0.7314971271064821, "min": 25.717430322998553, "max": 27.062772714496532 }, "latency_p95_s": { "mean": 28.475788485634013, "stddev": 0.45599538049132715, "min": 28.171156288299606, "max": 29.000033616899962 }, "latency_p99_s": { "mean": 29.649243244157116, "stddev": 1.3690458411359139, "min": 28.848386137351337, "max": 31.23003506197929 } } ] }, "knowledge": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-012743-4222c1", "evaluator_version": "1.2", "condition": { "dataset_version": 1.2, "max_tokens": 128, "request_timeout_s": 300, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.2, "total": 100, "correct": 93, "accuracy": 0.93, "pass_rate": 0.93, "categories": { "general": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "korea": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "math": { "total": 20, "correct": 16, "accuracy": 0.8, "pass_rate": 0.8 }, "science": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "logic": { "total": 20, "correct": 17, "accuracy": 0.85, "pass_rate": 0.85 } }, "difficulties": { "easy": { "total": 25, "correct": 25, "accuracy": 1.0, "pass_rate": 1.0 }, "medium": { "total": 40, "correct": 38, "accuracy": 0.95, "pass_rate": 0.95 }, "hard": { "total": 35, "correct": 30, "accuracy": 0.8571428571428571, "pass_rate": 0.8571428571428571 } } }, "coding": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-142222-7d90b3", "evaluator_version": "existing-v1", "condition": { "dataset_version": 1, "request_timeout_s": 300.0, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": -1, "thinking_mode": "auto" }, "version": 1, "total": 12, "passed": 10, "pass_rate": 0.8333333333333334, "categories": { "basic": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "string": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "collection": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 }, "edge_case": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 } }, "failure_types": { "syntax_error": 2 } }, "tool_call": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-005852-8e3526", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 15, "passed": 11, "pass_rate": 0.7333333333333333, "tool_selection_accuracy": 0.9333333333333333, "argument_accuracy": 0.8, "execution_success_rate": 0.8666666666666667, "tool_rate": 0.9333333333333333, "args_rate": 0.8, "exec_rate": 0.8666666666666667, "failure_types": { "wrong_arguments": 2, "wrong_tool": 1, "execution_error": 1 } }, "agent_single": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-005928-9c1a52", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 12, "passed": 9, "correct": 9, "pass_rate": 0.75, "avg_steps": 3.3333333333333335, "avg_tool_calls": 2.3333333333333335, "tool_calls_total": 28, "failed_tool_calls": 3, "repeated_tool_calls": 4, "failure_types": { "tool_error": 1, "missing_required_step": 2 } }, "agent_multi": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-010019-112caa", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 8, "max_tokens": 256, "request_timeout_s": 300, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 10, "passed": 9, "correct": 9, "pass_rate": 0.9, "handoff_success_rate": 1.0, "role_participation_success_rate": 1.0, "avg_steps": 3.7, "avg_tool_calls": 2.7, "tool_calls_total": 27, "failed_tool_calls": 2, "invalid_tool_calls": 0, "handoffs": 11, "failure_types": { "repeated_loop": 1 } } }, "provenance": { "hardware": "NVIDIA DGX Spark GB10", "reasoning_mode": "off", "raw_artifacts_public": false } }, { "model_id": "north-mini-mxfp4", "model": "North Mini", "variant": "MXFP4", "quantization": "MXFP4", "benchmark_versions": { "performance": "existing-v1", "server_performance": "existing-v1", "knowledge": "1.2", "coding": "existing-v1", "tool_call": "1.1", "agent_single": "1.1", "agent_multi": "1.1" }, "suites": { "performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-183924-10553d", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_type_k": "f16", "cache_type_v": "f16", "flash_attn": true, "gen_tokens": [ 512 ], "n_gpu_layers": 999, "prompt_tokens": [ 512, 2048, 8192, 32768 ], "repetitions": 5, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "metrics": { "pp": { "token_count": 32768, "mean_tps": 2454.923638, "stddev_tps": 41.135493, "repetitions": 5, "unit": "tokens/s" }, "tg": { "token_count": 512, "mean_tps": 68.848626, "stddev_tps": 1.018831, "repetitions": 5, "unit": "tokens/s" } }, "repetitions": 5 }, "server_performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-184209-003ab1", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_prompt": false, "cache_type_k": "f16", "cache_type_v": "f16", "concurrencies": [ 1, 2, 4, 8 ], "ctx_size": 8192, "flash_attn": true, "gen_tokens": 512, "n_gpu_layers": 999, "parallel": 8, "repetitions": 3, "request_timeout_s": 300, "requests_per_concurrency": 4, "spec": "off", "spec_draft_n_max": 3, "spec_type": "none", "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "formal": true, "concurrencies": [ 1, 2, 4, 8 ], "requests_per_concurrency": 4, "repetitions": 3, "gen_tokens": 512, "samples_count": 180, "conditions": [ { "concurrency": 1, "repetitions": 3, "requests_per_repetition": 4, "successful_requests": 12, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 69.16984014874247, "stddev": 0.6318633614978502, "min": 68.46822525855062, "max": 69.69400767286228 }, "per_request_generation_throughput_tps": { "mean": 69.20447963189375, "stddev": 1.610716936112924, "min": 64.47938372966513, "max": 70.90214797550976 }, "latency_p50_s": { "mean": 7.381369592832925, "stddev": 0.029335595054220256, "min": 7.356169774000591, "max": 7.413573174999328 }, "latency_p95_s": { "mean": 7.556105684684978, "stddev": 0.26607198138437677, "min": 7.378378808849812, "max": 7.862004067904491 }, "latency_p99_s": { "mean": 7.577185834804938, "stddev": 0.3020687471062212, "min": 7.378691514570237, "max": 7.924818847184943 } }, { "concurrency": 2, "repetitions": 3, "requests_per_repetition": 8, "successful_requests": 24, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 114.51654256015364, "stddev": 7.293179228489843, "min": 106.7459526650169, "max": 121.21322733662778 }, "per_request_generation_throughput_tps": { "mean": 57.53811744569568, "stddev": 5.136138748700808, "min": 52.97254073541199, "max": 63.91313433294083 }, "latency_p50_s": { "mean": 8.854362665833227, "stddev": 0.753968121191296, "min": 8.099807831502403, "max": 9.607742700995004 }, "latency_p95_s": { "mean": 9.62497908701213, "stddev": 0.03503762339234448, "min": 9.589980657499108, "max": 9.660055773193744 }, "latency_p99_s": { "mean": 9.629514840333771, "stddev": 0.034804197166656725, "min": 9.594710750702362, "max": 9.664319145034678 } }, { "concurrency": 4, "repetitions": 3, "requests_per_repetition": 16, "successful_requests": 48, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 192.8578721428912, "stddev": 19.872659831547423, "min": 174.0219157699332, "max": 213.62607805472513 }, "per_request_generation_throughput_tps": { "mean": 48.608152808418474, "stddev": 4.360674990388913, "min": 43.12266411604361, "max": 55.45479573214989 }, "latency_p50_s": { "mean": 10.558632274000653, "stddev": 1.227584348556305, "min": 9.304650113503158, "max": 11.757995513500646 }, "latency_p95_s": { "mean": 11.038839286581302, "stddev": 0.7296575691009592, "min": 10.519123034500808, "max": 11.8729982320001 }, "latency_p99_s": { "mean": 11.039109774380146, "stddev": 0.72952357375034, "min": 10.519319125301262, "max": 11.873085150397674 } }, { "concurrency": 8, "repetitions": 3, "requests_per_repetition": 32, "successful_requests": 96, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 269.8952835361626, "stddev": 4.55814531743193, "min": 266.91503710690944, "max": 275.1424384448955 }, "per_request_generation_throughput_tps": { "mean": 33.90465220703734, "stddev": 1.034664137899748, "min": 32.373957812602164, "max": 36.27508962861949 }, "latency_p50_s": { "mean": 15.157407225000497, "stddev": 0.4583230049593848, "min": 14.628196302997821, "max": 15.425451711998903 }, "latency_p95_s": { "mean": 15.652485687905088, "stddev": 0.142809413433524, "min": 15.544161161800002, "max": 15.8143228150082 }, "latency_p99_s": { "mean": 15.653920218339335, "stddev": 0.14207624711191835, "min": 15.546276994383225, "max": 15.814958257443358 } } ] }, "knowledge": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-012854-b6a868", "evaluator_version": "1.2", "condition": { "dataset_version": 1.2, "max_tokens": 128, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.2, "total": 100, "correct": 88, "accuracy": 0.88, "pass_rate": 0.88, "categories": { "general": { "total": 20, "correct": 18, "accuracy": 0.9, "pass_rate": 0.9 }, "korea": { "total": 20, "correct": 19, "accuracy": 0.95, "pass_rate": 0.95 }, "math": { "total": 20, "correct": 15, "accuracy": 0.75, "pass_rate": 0.75 }, "science": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "logic": { "total": 20, "correct": 16, "accuracy": 0.8, "pass_rate": 0.8 } }, "difficulties": { "easy": { "total": 25, "correct": 24, "accuracy": 0.96, "pass_rate": 0.96 }, "medium": { "total": 40, "correct": 38, "accuracy": 0.95, "pass_rate": 0.95 }, "hard": { "total": 35, "correct": 26, "accuracy": 0.7428571428571429, "pass_rate": 0.7428571428571429 } } }, "coding": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-185126-78fe5a", "evaluator_version": "existing-v1", "condition": { "dataset_version": 1, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto" }, "version": 1, "total": 12, "passed": 11, "pass_rate": 0.9166666666666666, "categories": { "basic": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "string": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "collection": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 }, "edge_case": { "total": 3, "passed": 3, "pass_rate": 1.0 } }, "failure_types": { "syntax_error": 1 } }, "tool_call": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-010242-4406ed", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 15, "passed": 12, "pass_rate": 0.8, "tool_selection_accuracy": 0.9333333333333333, "argument_accuracy": 0.8666666666666667, "execution_success_rate": 0.9333333333333333, "tool_rate": 0.9333333333333333, "args_rate": 0.8666666666666667, "exec_rate": 0.9333333333333333, "failure_types": { "wrong_arguments": 1, "wrong_tool": 1, "final_answer_error": 1 } }, "agent_single": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-010308-34728b", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 12, "passed": 10, "correct": 10, "pass_rate": 0.8333333333333334, "avg_steps": 3.5833333333333335, "avg_tool_calls": 2.5833333333333335, "tool_calls_total": 31, "failed_tool_calls": 4, "repeated_tool_calls": 4, "failure_types": { "tool_error": 1, "missing_required_step": 1 } }, "agent_multi": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-010349-d84a1c", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 8, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 10, "passed": 7, "correct": 7, "pass_rate": 0.7, "handoff_success_rate": 0.8, "role_participation_success_rate": 0.8, "avg_steps": 3.8, "avg_tool_calls": 2.8, "tool_calls_total": 28, "failed_tool_calls": 6, "invalid_tool_calls": 0, "handoffs": 11, "failure_types": { "tool_error": 3 } } }, "provenance": { "hardware": "NVIDIA DGX Spark GB10", "reasoning_mode": "off", "raw_artifacts_public": false } }, { "model_id": "north-mini-ud-q4", "model": "North Mini", "variant": "UD-Q4", "quantization": "UD-Q4", "benchmark_versions": { "performance": "existing-v1", "server_performance": "existing-v1", "knowledge": "1.2", "coding": "existing-v1", "tool_call": "1.1", "agent_single": "1.1", "agent_multi": "1.1" }, "suites": { "performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-185701-b2d23b", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_type_k": "f16", "cache_type_v": "f16", "flash_attn": true, "gen_tokens": [ 512 ], "n_gpu_layers": 999, "prompt_tokens": [ 512, 2048, 8192, 32768 ], "repetitions": 5, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "metrics": { "pp": { "token_count": 32768, "mean_tps": 2299.059191, "stddev_tps": 29.340568, "repetitions": 5, "unit": "tokens/s" }, "tg": { "token_count": 512, "mean_tps": 71.687116, "stddev_tps": 0.050564, "repetitions": 5, "unit": "tokens/s" } }, "repetitions": 5 }, "server_performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-185954-c24ba2", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_prompt": false, "cache_type_k": "f16", "cache_type_v": "f16", "concurrencies": [ 1, 2, 4, 8 ], "ctx_size": 8192, "flash_attn": true, "gen_tokens": 512, "n_gpu_layers": 999, "parallel": 8, "repetitions": 3, "request_timeout_s": 300, "requests_per_concurrency": 4, "spec": "off", "spec_draft_n_max": 3, "spec_type": "none", "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "formal": true, "concurrencies": [ 1, 2, 4, 8 ], "requests_per_concurrency": 4, "repetitions": 3, "gen_tokens": 512, "samples_count": 180, "conditions": [ { "concurrency": 1, "repetitions": 3, "requests_per_repetition": 4, "successful_requests": 12, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 68.54736875171422, "stddev": 0.290906954171383, "min": 68.2115776278115, "max": 68.72301652730962 }, "per_request_generation_throughput_tps": { "mean": 68.55885155853463, "stddev": 0.8598596798984735, "min": 66.1284021896894, "max": 69.43287042535452 }, "latency_p50_s": { "mean": 7.451276738504627, "stddev": 0.006495791436584868, "min": 7.446622551004111, "max": 7.458697869005846 }, "latency_p95_s": { "mean": 7.564411193162232, "stddev": 0.11977339508245989, "min": 7.468167936145619, "max": 7.698547894644435 }, "latency_p99_s": { "mean": 7.57959533836125, "stddev": 0.1377947034550985, "min": 7.468304241624719, "max": 7.73371981412347 } }, { "concurrency": 2, "repetitions": 3, "requests_per_repetition": 8, "successful_requests": 24, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 115.7404080952416, "stddev": 7.919555732616752, "min": 107.06353337724778, "max": 122.57949473956474 }, "per_request_generation_throughput_tps": { "mean": 58.14294585240137, "stddev": 5.0737609834634005, "min": 50.79135119512842, "max": 62.55472917222953 }, "latency_p50_s": { "mean": 8.887361708332415, "stddev": 0.9592411817214268, "min": 8.321388042495528, "max": 9.994908425003814 }, "latency_p95_s": { "mean": 9.493344149730062, "stddev": 0.9305402288384782, "min": 8.42044076535385, "max": 10.080430447992693 }, "latency_p99_s": { "mean": 9.494855282475715, "stddev": 0.9315700116332307, "min": 8.420628661876544, "max": 10.080451190389285 } }, { "concurrency": 4, "repetitions": 3, "requests_per_repetition": 16, "successful_requests": 48, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 194.82528344627522, "stddev": 5.346102032926122, "min": 188.75210351407804, "max": 198.82008585219333 }, "per_request_generation_throughput_tps": { "mean": 48.96176942124976, "stddev": 3.8001211529824523, "min": 43.19496794879131, "max": 55.34689278517983 }, "latency_p50_s": { "mean": 10.545515477497247, "stddev": 0.42670014190310285, "min": 10.05453267149278, "max": 10.826712961999874 }, "latency_p95_s": { "mean": 11.182899268832747, "stddev": 0.5855898470932233, "min": 10.770126174247707, "max": 11.853106091995869 }, "latency_p99_s": { "mean": 11.186600755629238, "stddev": 0.5818788729069654, "min": 10.780428772440064, "max": 11.853207256791212 } }, { "concurrency": 8, "repetitions": 3, "requests_per_repetition": 32, "successful_requests": 96, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 299.76716564814257, "stddev": 4.410270004983431, "min": 295.072734741252, "max": 303.82383141153537 }, "per_request_generation_throughput_tps": { "mean": 37.508107361984635, "stddev": 1.1613396747068365, "min": 35.20034352827536, "max": 38.671703398113635 }, "latency_p50_s": { "mean": 13.58630177466936, "stddev": 0.20511457316333664, "min": 13.440456525502668, "max": 13.820837705003214 }, "latency_p95_s": { "mean": 14.160821862016261, "stddev": 0.5598996868992929, "min": 13.517899421648327, "max": 14.541238244344276 }, "latency_p99_s": { "mean": 14.162787322966711, "stddev": 0.5604767548280138, "min": 13.519272360251634, "max": 14.54413139907425 } } ] }, "knowledge": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-012950-ab0899", "evaluator_version": "1.2", "condition": { "dataset_version": 1.2, "max_tokens": 128, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.2, "total": 100, "correct": 90, "accuracy": 0.9, "pass_rate": 0.9, "categories": { "general": { "total": 20, "correct": 19, "accuracy": 0.95, "pass_rate": 0.95 }, "korea": { "total": 20, "correct": 18, "accuracy": 0.9, "pass_rate": 0.9 }, "math": { "total": 20, "correct": 16, "accuracy": 0.8, "pass_rate": 0.8 }, "science": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "logic": { "total": 20, "correct": 17, "accuracy": 0.85, "pass_rate": 0.85 } }, "difficulties": { "easy": { "total": 25, "correct": 24, "accuracy": 0.96, "pass_rate": 0.96 }, "medium": { "total": 40, "correct": 38, "accuracy": 0.95, "pass_rate": 0.95 }, "hard": { "total": 35, "correct": 28, "accuracy": 0.8, "pass_rate": 0.8 } } }, "coding": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-190857-ccec3d", "evaluator_version": "existing-v1", "condition": { "dataset_version": 1, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto" }, "version": 1, "total": 12, "passed": 12, "pass_rate": 1.0, "categories": { "basic": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "string": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "collection": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "edge_case": { "total": 3, "passed": 3, "pass_rate": 1.0 } }, "failure_types": {} }, "tool_call": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-010527-039740", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 15, "passed": 9, "pass_rate": 0.6, "tool_selection_accuracy": 0.9333333333333333, "argument_accuracy": 0.8, "execution_success_rate": 0.9333333333333333, "tool_rate": 0.9333333333333333, "args_rate": 0.8, "exec_rate": 0.9333333333333333, "failure_types": { "wrong_arguments": 2, "final_answer_error": 3, "wrong_tool": 1 } }, "agent_single": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-010558-04371f", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 12, "passed": 11, "correct": 11, "pass_rate": 0.9166666666666666, "avg_steps": 3.4166666666666665, "avg_tool_calls": 2.4166666666666665, "tool_calls_total": 29, "failed_tool_calls": 2, "repeated_tool_calls": 4, "failure_types": { "missing_required_step": 1 } }, "agent_multi": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-010639-645e21", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 8, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 10, "passed": 7, "correct": 7, "pass_rate": 0.7, "handoff_success_rate": 0.9, "role_participation_success_rate": 0.9, "avg_steps": 4.0, "avg_tool_calls": 3.0, "tool_calls_total": 30, "failed_tool_calls": 7, "invalid_tool_calls": 0, "handoffs": 12, "failure_types": { "tool_error": 3 } } }, "provenance": { "hardware": "NVIDIA DGX Spark GB10", "reasoning_mode": "off", "raw_artifacts_public": false } }, { "model_id": "north-mini-ud-q5", "model": "North Mini", "variant": "UD-Q5", "quantization": "UD-Q5", "benchmark_versions": { "performance": "existing-v1", "server_performance": "existing-v1", "knowledge": "1.2", "coding": "existing-v1", "tool_call": "1.1", "agent_single": "1.1", "agent_multi": "1.1" }, "suites": { "performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-191515-5b2b06", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_type_k": "f16", "cache_type_v": "f16", "flash_attn": true, "gen_tokens": [ 512 ], "n_gpu_layers": 999, "prompt_tokens": [ 512, 2048, 8192, 32768 ], "repetitions": 5, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "metrics": { "pp": { "token_count": 32768, "mean_tps": 2127.254922, "stddev_tps": 27.846825, "repetitions": 5, "unit": "tokens/s" }, "tg": { "token_count": 512, "mean_tps": 66.020624, "stddev_tps": 0.057222, "repetitions": 5, "unit": "tokens/s" } }, "repetitions": 5 }, "server_performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-191832-4bd7c6", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_prompt": false, "cache_type_k": "f16", "cache_type_v": "f16", "concurrencies": [ 1, 2, 4, 8 ], "ctx_size": 8192, "flash_attn": true, "gen_tokens": 512, "n_gpu_layers": 999, "parallel": 8, "repetitions": 3, "request_timeout_s": 300, "requests_per_concurrency": 4, "spec": "off", "spec_draft_n_max": 3, "spec_type": "none", "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "formal": true, "concurrencies": [ 1, 2, 4, 8 ], "requests_per_concurrency": 4, "repetitions": 3, "gen_tokens": 512, "samples_count": 180, "conditions": [ { "concurrency": 1, "repetitions": 3, "requests_per_repetition": 4, "successful_requests": 12, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 63.37936551069864, "stddev": 0.23962360986125325, "min": 63.119563437359005, "max": 63.591708262654215 }, "per_request_generation_throughput_tps": { "mean": 63.388619069232675, "stddev": 0.7216249221621092, "min": 61.34660299435411, "max": 64.08652396109413 }, "latency_p50_s": { "mean": 8.058172603166895, "stddev": 0.020146753751108568, "min": 8.039821262500482, "max": 8.079730232493603 }, "latency_p95_s": { "mean": 8.174009274499259, "stddev": 0.1135473489445481, "min": 8.105092746000446, "max": 8.305064042448794 }, "latency_p99_s": { "mean": 8.187345977298683, "stddev": 0.130329673719987, "min": 8.110711585200333, "max": 8.33782912768671 } }, { "concurrency": 2, "repetitions": 3, "requests_per_repetition": 8, "successful_requests": 24, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 105.91669914195563, "stddev": 8.78197970398569, "min": 98.42381106267608, "max": 115.58054452802776 }, "per_request_generation_throughput_tps": { "mean": 53.25462347053762, "stddev": 5.301394860949299, "min": 46.88711278649156, "max": 58.061663172296534 }, "latency_p50_s": { "mean": 9.866912361501212, "stddev": 1.0047604597766648, "min": 8.864724735496566, "max": 10.874226041007205 }, "latency_p95_s": { "mean": 10.232710156249716, "stddev": 1.1677748027789292, "min": 8.884306577604729, "max": 10.91424896325043 }, "latency_p99_s": { "mean": 10.235262350184978, "stddev": 1.1697703041286207, "min": 8.884560432329309, "max": 10.91872572225111 } }, { "concurrency": 4, "repetitions": 3, "requests_per_repetition": 16, "successful_requests": 48, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 177.7288383883828, "stddev": 26.96030744926229, "min": 147.11656957229664, "max": 197.93649564218586 }, "per_request_generation_throughput_tps": { "mean": 44.98208164209404, "stddev": 6.179432865072815, "min": 35.056549590747146, "max": 51.99028903612417 }, "latency_p50_s": { "mean": 11.469240284670377, "stddev": 1.9252372431060563, "min": 9.92000349000591, "max": 13.62459669400414 }, "latency_p95_s": { "mean": 12.602739030498924, "stddev": 1.3959599856249223, "min": 11.685606539998844, "max": 14.209284055003081 }, "latency_p99_s": { "mean": 12.708508174098824, "stddev": 1.5779920241996526, "min": 11.685615544799656, "max": 14.525836118197185 } }, { "concurrency": 8, "repetitions": 3, "requests_per_repetition": 32, "successful_requests": 96, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 272.3283549029115, "stddev": 3.3682348150181927, "min": 268.5710727514945, "max": 275.0771275175744 }, "per_request_generation_throughput_tps": { "mean": 34.40050014741642, "stddev": 0.8000348668348557, "min": 32.14548449580878, "max": 35.959482122866916 }, "latency_p50_s": { "mean": 14.855871520000315, "stddev": 0.10515705066675796, "min": 14.767952370501007, "max": 14.972361589498178 }, "latency_p95_s": { "mean": 15.365086735981944, "stddev": 0.5254703479862691, "min": 14.882178700402438, "max": 15.924690024543088 }, "latency_p99_s": { "mean": 15.366203367176931, "stddev": 0.5264551157279946, "min": 14.882604011969525, "max": 15.926991659951163 } } ] }, "knowledge": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-013046-0c6ff8", "evaluator_version": "1.2", "condition": { "dataset_version": 1.2, "max_tokens": 128, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.2, "total": 100, "correct": 93, "accuracy": 0.93, "pass_rate": 0.93, "categories": { "general": { "total": 20, "correct": 19, "accuracy": 0.95, "pass_rate": 0.95 }, "korea": { "total": 20, "correct": 19, "accuracy": 0.95, "pass_rate": 0.95 }, "math": { "total": 20, "correct": 17, "accuracy": 0.85, "pass_rate": 0.85 }, "science": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "logic": { "total": 20, "correct": 18, "accuracy": 0.9, "pass_rate": 0.9 } }, "difficulties": { "easy": { "total": 25, "correct": 25, "accuracy": 1.0, "pass_rate": 1.0 }, "medium": { "total": 40, "correct": 39, "accuracy": 0.975, "pass_rate": 0.975 }, "hard": { "total": 35, "correct": 29, "accuracy": 0.8285714285714286, "pass_rate": 0.8285714285714286 } } }, "coding": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-192841-b07e97", "evaluator_version": "existing-v1", "condition": { "dataset_version": 1, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto" }, "version": 1, "total": 12, "passed": 11, "pass_rate": 0.9166666666666666, "categories": { "basic": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "string": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "collection": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 }, "edge_case": { "total": 3, "passed": 3, "pass_rate": 1.0 } }, "failure_types": { "syntax_error": 1 } }, "tool_call": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-010817-3fddf8", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 15, "passed": 12, "pass_rate": 0.8, "tool_selection_accuracy": 0.9333333333333333, "argument_accuracy": 0.8666666666666667, "execution_success_rate": 0.9333333333333333, "tool_rate": 0.9333333333333333, "args_rate": 0.8666666666666667, "exec_rate": 0.9333333333333333, "failure_types": { "wrong_arguments": 1, "wrong_tool": 1, "final_answer_error": 1 } }, "agent_single": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-010848-ae5c5a", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 12, "passed": 11, "correct": 11, "pass_rate": 0.9166666666666666, "avg_steps": 3.5, "avg_tool_calls": 2.5, "tool_calls_total": 30, "failed_tool_calls": 3, "repeated_tool_calls": 4, "failure_types": { "missing_required_step": 1 } }, "agent_multi": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-010934-955895", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 8, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 10, "passed": 7, "correct": 7, "pass_rate": 0.7, "handoff_success_rate": 1.0, "role_participation_success_rate": 1.0, "avg_steps": 4.0, "avg_tool_calls": 3.0, "tool_calls_total": 30, "failed_tool_calls": 6, "invalid_tool_calls": 0, "handoffs": 12, "failure_types": { "tool_error": 3 } } }, "provenance": { "hardware": "NVIDIA DGX Spark GB10", "reasoning_mode": "off", "raw_artifacts_public": false } }, { "model_id": "north-mini-ud-q6", "model": "North Mini", "variant": "UD-Q6", "quantization": "UD-Q6", "benchmark_versions": { "performance": "existing-v1", "server_performance": "existing-v1", "knowledge": "1.2", "coding": "existing-v1", "tool_call": "1.1", "agent_single": "1.1", "agent_multi": "1.1" }, "suites": { "performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-193540-0c3ce7", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_type_k": "f16", "cache_type_v": "f16", "flash_attn": true, "gen_tokens": [ 512 ], "n_gpu_layers": 999, "prompt_tokens": [ 512, 2048, 8192, 32768 ], "repetitions": 5, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "metrics": { "pp": { "token_count": 32768, "mean_tps": 1969.278577, "stddev_tps": 23.27233, "repetitions": 5, "unit": "tokens/s" }, "tg": { "token_count": 512, "mean_tps": 60.782286, "stddev_tps": 0.065456, "repetitions": 5, "unit": "tokens/s" } }, "repetitions": 5 }, "server_performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-193918-79dcd0", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_prompt": false, "cache_type_k": "f16", "cache_type_v": "f16", "concurrencies": [ 1, 2, 4, 8 ], "ctx_size": 8192, "flash_attn": true, "gen_tokens": 512, "n_gpu_layers": 999, "parallel": 8, "repetitions": 3, "request_timeout_s": 300, "requests_per_concurrency": 4, "spec": "off", "spec_draft_n_max": 3, "spec_type": "none", "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "formal": true, "concurrencies": [ 1, 2, 4, 8 ], "requests_per_concurrency": 4, "repetitions": 3, "gen_tokens": 512, "samples_count": 180, "conditions": [ { "concurrency": 1, "repetitions": 3, "requests_per_repetition": 4, "successful_requests": 12, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 58.336643058874486, "stddev": 0.2181741470164372, "min": 58.12208331730135, "max": 58.55826035309889 }, "per_request_generation_throughput_tps": { "mean": 58.342788190282604, "stddev": 0.5534245341423862, "min": 56.86630168854621, "max": 58.987286145936665 }, "latency_p50_s": { "mean": 8.688242021336919, "stddev": 0.027706489949210367, "min": 8.670328324005823, "max": 8.720154788003128 }, "latency_p95_s": { "mean": 8.765216146599657, "stddev": 0.10182278043662746, "min": 8.686039725397132, "max": 8.880078518504888 }, "latency_p99_s": { "mean": 8.775607513318924, "stddev": 0.11714004777234242, "min": 8.68747617147572, "max": 8.908535103704343 } }, { "concurrency": 2, "repetitions": 3, "requests_per_repetition": 8, "successful_requests": 24, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 98.08370199978043, "stddev": 7.870650939283807, "min": 89.64186064066428, "max": 105.2198640173217 }, "per_request_generation_throughput_tps": { "mean": 49.29106440343423, "stddev": 4.660335334390135, "min": 42.735661155078525, "max": 52.90475906090699 }, "latency_p50_s": { "mean": 10.477309070833144, "stddev": 1.2679748691931467, "min": 9.731795709500147, "max": 11.941357481497107 }, "latency_p95_s": { "mean": 11.23868175373597, "stddev": 1.2594913744288072, "min": 9.784381900953303, "max": 11.974633720106794 }, "latency_p99_s": { "mean": 11.24212705554819, "stddev": 1.2622888217877903, "min": 9.78459479139041, "max": 11.97942804642662 } }, { "concurrency": 4, "repetitions": 3, "requests_per_repetition": 16, "successful_requests": 48, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 154.22181268351548, "stddev": 3.36610513446124, "min": 150.33522097061524, "max": 156.20341544740668 }, "per_request_generation_throughput_tps": { "mean": 38.71353585344832, "stddev": 1.2249101074351902, "min": 34.63054428182204, "max": 40.034497044261975 }, "latency_p50_s": { "mean": 13.12913312233286, "stddev": 0.05601780238140063, "min": 13.07023012550053, "max": 13.181732448007097 }, "latency_p95_s": { "mean": 13.722948933420412, "stddev": 0.8948062359253967, "min": 13.186338352254097, "max": 14.755920826508373 }, "latency_p99_s": { "mean": 13.733846335216366, "stddev": 0.9051723000320652, "min": 13.195678516846238, "max": 14.778893157302809 } }, { "concurrency": 8, "repetitions": 3, "requests_per_repetition": 32, "successful_requests": 96, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 250.67322843912655, "stddev": 2.485499251871519, "min": 248.1074641986577, "max": 253.0698191962146 }, "per_request_generation_throughput_tps": { "mean": 31.355597477621195, "stddev": 0.7624859346284066, "min": 29.619237332821584, "max": 32.4514042764086 }, "latency_p50_s": { "mean": 16.36029032566633, "stddev": 0.10803639712550668, "min": 16.274290368004586, "max": 16.481551844000933 }, "latency_p95_s": { "mean": 16.64753806104903, "stddev": 0.5521465991576371, "min": 16.308536333245865, "max": 17.284665098695406 }, "latency_p99_s": { "mean": 16.64921474730756, "stddev": 0.5516808656319107, "min": 16.308967228737746, "max": 17.28573479418992 } } ] }, "knowledge": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-013143-e503c0", "evaluator_version": "1.2", "condition": { "dataset_version": 1.2, "max_tokens": 128, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.2, "total": 100, "correct": 92, "accuracy": 0.92, "pass_rate": 0.92, "categories": { "general": { "total": 20, "correct": 19, "accuracy": 0.95, "pass_rate": 0.95 }, "korea": { "total": 20, "correct": 19, "accuracy": 0.95, "pass_rate": 0.95 }, "math": { "total": 20, "correct": 17, "accuracy": 0.85, "pass_rate": 0.85 }, "science": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "logic": { "total": 20, "correct": 17, "accuracy": 0.85, "pass_rate": 0.85 } }, "difficulties": { "easy": { "total": 25, "correct": 25, "accuracy": 1.0, "pass_rate": 1.0 }, "medium": { "total": 40, "correct": 39, "accuracy": 0.975, "pass_rate": 0.975 }, "hard": { "total": 35, "correct": 28, "accuracy": 0.8, "pass_rate": 0.8 } } }, "coding": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-195043-f17e76", "evaluator_version": "existing-v1", "condition": { "dataset_version": 1, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto" }, "version": 1, "total": 12, "passed": 11, "pass_rate": 0.9166666666666666, "categories": { "basic": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "string": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "collection": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 }, "edge_case": { "total": 3, "passed": 3, "pass_rate": 1.0 } }, "failure_types": { "syntax_error": 1 } }, "tool_call": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-011127-af54cb", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 15, "passed": 11, "pass_rate": 0.7333333333333333, "tool_selection_accuracy": 0.9333333333333333, "argument_accuracy": 0.8666666666666667, "execution_success_rate": 0.9333333333333333, "tool_rate": 0.9333333333333333, "args_rate": 0.8666666666666667, "exec_rate": 0.9333333333333333, "failure_types": { "wrong_arguments": 1, "wrong_tool": 1, "final_answer_error": 2 } }, "agent_single": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-011203-9766e8", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 12, "passed": 11, "correct": 11, "pass_rate": 0.9166666666666666, "avg_steps": 3.4166666666666665, "avg_tool_calls": 2.4166666666666665, "tool_calls_total": 29, "failed_tool_calls": 2, "repeated_tool_calls": 4, "failure_types": { "missing_required_step": 1 } }, "agent_multi": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-011249-c4feed", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 8, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 10, "passed": 8, "correct": 8, "pass_rate": 0.8, "handoff_success_rate": 1.0, "role_participation_success_rate": 1.0, "avg_steps": 3.9, "avg_tool_calls": 2.9, "tool_calls_total": 29, "failed_tool_calls": 5, "invalid_tool_calls": 0, "handoffs": 11, "failure_types": { "tool_error": 1, "wrong_tool": 1 } } }, "provenance": { "hardware": "NVIDIA DGX Spark GB10", "reasoning_mode": "off", "raw_artifacts_public": false } }, { "model_id": "n2-mini-ud-q4", "model": "N2 Mini", "variant": "UD-Q4", "quantization": "UD-Q4", "benchmark_versions": { "performance": "existing-v1", "server_performance": "existing-v1", "knowledge": "1.2", "coding": "existing-v1", "tool_call": "1.1", "agent_single": "1.1", "agent_multi": "1.1" }, "suites": { "performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-153010-ec5db8", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_type_k": "f16", "cache_type_v": "f16", "flash_attn": true, "gen_tokens": [ 512 ], "n_gpu_layers": 999, "prompt_tokens": [ 512, 2048, 8192, 32768 ], "repetitions": 5, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "metrics": { "pp": { "token_count": 32768, "mean_tps": 1808.512053, "stddev_tps": 2.956163, "repetitions": 5, "unit": "tokens/s" }, "tg": { "token_count": 512, "mean_tps": 63.764987, "stddev_tps": 0.093344, "repetitions": 5, "unit": "tokens/s" } }, "repetitions": 5 }, "server_performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-161120-b37a54", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_prompt": false, "cache_type_k": "f16", "cache_type_v": "f16", "concurrencies": [ 1, 2, 4, 8 ], "ctx_size": 8192, "flash_attn": true, "gen_tokens": 512, "n_gpu_layers": 999, "parallel": 8, "repetitions": 3, "request_timeout_s": 300, "requests_per_concurrency": 4, "spec": "off", "spec_draft_n_max": 3, "spec_type": "none", "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "formal": true, "concurrencies": [ 1, 2, 4, 8 ], "requests_per_concurrency": 4, "repetitions": 3, "gen_tokens": 512, "samples_count": 180, "conditions": [ { "concurrency": 1, "repetitions": 3, "requests_per_repetition": 4, "successful_requests": 12, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 60.03673766864971, "stddev": 0.3472690054520575, "min": 59.652725220654595, "max": 60.32872685112651 }, "per_request_generation_throughput_tps": { "mean": 60.04038101034668, "stddev": 0.4562290616434823, "min": 59.180848388738355, "max": 60.763866842540644 }, "latency_p50_s": { "mean": 8.52179124583563, "stddev": 0.047728058188202374, "min": 8.482053193503816, "max": 8.57473039350225 }, "latency_p95_s": { "mean": 8.582615719382254, "stddev": 0.07030108594212667, "min": 8.504838528000983, "max": 8.641635734801094 }, "latency_p99_s": { "mean": 8.590004128941752, "stddev": 0.07380156702619216, "min": 8.507413084800355, "max": 8.649485108560912 } }, { "concurrency": 2, "repetitions": 3, "requests_per_repetition": 8, "successful_requests": 24, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 91.63138442986437, "stddev": 0.25973003628960756, "min": 91.34105683007652, "max": 91.84167967648007 }, "per_request_generation_throughput_tps": { "mean": 45.82466115743034, "stddev": 0.23369479412366126, "min": 45.38772016804005, "max": 46.217626730107426 }, "latency_p50_s": { "mean": 11.17682969199935, "stddev": 0.02247766900844874, "min": 11.163461280499178, "max": 11.202780702999007 }, "latency_p95_s": { "mean": 11.231807709150235, "stddev": 0.04228081963667353, "min": 11.189810499801752, "max": 11.274366543747965 }, "latency_p99_s": { "mean": 11.233601080496761, "stddev": 0.04470565684247471, "min": 11.19000660396232, "max": 11.279340686347059 } }, { "concurrency": 4, "repetitions": 3, "requests_per_repetition": 16, "successful_requests": 48, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 141.42532355089068, "stddev": 4.569420046129663, "min": 136.43007257657706, "max": 145.3944110102662 }, "per_request_generation_throughput_tps": { "mean": 35.433262716692816, "stddev": 1.865268454617353, "min": 31.612361574158598, "max": 37.43211134264674 }, "latency_p50_s": { "mean": 14.409296264999284, "stddev": 0.6093055878540758, "min": 13.803392039997561, "max": 15.021947185996396 }, "latency_p95_s": { "mean": 15.38563132008494, "stddev": 0.7016496026874472, "min": 14.956762506251835, "max": 16.19535206175351 }, "latency_p99_s": { "mean": 15.385950893085829, "stddev": 0.7019575638090882, "min": 14.956895822054502, "max": 16.196027109152055 } }, { "concurrency": 8, "repetitions": 3, "requests_per_repetition": 32, "successful_requests": 96, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 180.38362485845377, "stddev": 5.568898062000794, "min": 176.95145523920928, "max": 186.80903917486438 }, "per_request_generation_throughput_tps": { "mean": 22.793708199023953, "stddev": 0.9103251995355471, "min": 21.09306404426374, "max": 24.505014809782832 }, "latency_p50_s": { "mean": 22.141261220999393, "stddev": 0.363318928700603, "min": 21.910095212999295, "max": 22.560031215998606 }, "latency_p95_s": { "mean": 23.567930622568383, "stddev": 0.6643146902457262, "min": 22.953775946603127, "max": 24.2730337551533 }, "latency_p99_s": { "mean": 23.578684123595885, "stddev": 0.6622801524424259, "min": 22.954444661749367, "max": 24.273372568640333 } } ] }, "knowledge": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-013249-879db0", "evaluator_version": "1.2", "condition": { "dataset_version": 1.2, "max_tokens": 128, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.2, "total": 100, "correct": 91, "accuracy": 0.91, "pass_rate": 0.91, "categories": { "general": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "korea": { "total": 20, "correct": 19, "accuracy": 0.95, "pass_rate": 0.95 }, "math": { "total": 20, "correct": 16, "accuracy": 0.8, "pass_rate": 0.8 }, "science": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "logic": { "total": 20, "correct": 16, "accuracy": 0.8, "pass_rate": 0.8 } }, "difficulties": { "easy": { "total": 25, "correct": 24, "accuracy": 0.96, "pass_rate": 0.96 }, "medium": { "total": 40, "correct": 38, "accuracy": 0.95, "pass_rate": 0.95 }, "hard": { "total": 35, "correct": 29, "accuracy": 0.8285714285714286, "pass_rate": 0.8285714285714286 } } }, "coding": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-153414-41f495", "evaluator_version": "existing-v1", "condition": { "dataset_version": 1, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto" }, "version": 1, "total": 12, "passed": 10, "pass_rate": 0.8333333333333334, "categories": { "basic": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "string": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "collection": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 }, "edge_case": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 } }, "failure_types": { "syntax_error": 2 } }, "tool_call": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-011407-a27ccc", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 15, "passed": 9, "pass_rate": 0.6, "tool_selection_accuracy": 0.7333333333333333, "argument_accuracy": 0.6, "execution_success_rate": 0.8666666666666667, "tool_rate": 0.7333333333333333, "args_rate": 0.6, "exec_rate": 0.8666666666666667, "failure_types": { "wrong_arguments": 2, "missing_tool_call": 2, "incomplete_sequence": 2 } }, "agent_single": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-011438-5c19a8", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 12, "passed": 7, "correct": 7, "pass_rate": 0.5833333333333334, "avg_steps": 3.0833333333333335, "avg_tool_calls": 2.0833333333333335, "tool_calls_total": 25, "failed_tool_calls": 2, "repeated_tool_calls": 3, "failure_types": { "missing_required_step": 3, "tool_error": 2 } }, "agent_multi": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-011519-289ee9", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 8, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 10, "passed": 5, "correct": 5, "pass_rate": 0.5, "handoff_success_rate": 0.7, "role_participation_success_rate": 0.7, "avg_steps": 3.2, "avg_tool_calls": 2.2, "tool_calls_total": 22, "failed_tool_calls": 3, "invalid_tool_calls": 0, "handoffs": 11, "failure_types": { "tool_error": 3, "missing_required_step": 2 } } }, "provenance": { "hardware": "NVIDIA DGX Spark GB10", "reasoning_mode": "off", "raw_artifacts_public": false } }, { "model_id": "n2-mini-ud-q5xl", "model": "N2 Mini", "variant": "UD-Q5XL", "quantization": "UD-Q5XL", "benchmark_versions": { "performance": "existing-v1", "server_performance": "existing-v1", "knowledge": "1.2", "coding": "existing-v1", "tool_call": "1.1", "agent_single": "1.1", "agent_multi": "1.1" }, "suites": { "performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-153800-d9d315", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_type_k": "f16", "cache_type_v": "f16", "flash_attn": true, "gen_tokens": [ 512 ], "n_gpu_layers": 999, "prompt_tokens": [ 512, 2048, 8192, 32768 ], "repetitions": 5, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "metrics": { "pp": { "token_count": 32768, "mean_tps": 1691.704631, "stddev_tps": 10.258967, "repetitions": 5, "unit": "tokens/s" }, "tg": { "token_count": 512, "mean_tps": 59.354877, "stddev_tps": 0.082034, "repetitions": 5, "unit": "tokens/s" } }, "repetitions": 5 }, "server_performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-162247-e466ce", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_prompt": false, "cache_type_k": "f16", "cache_type_v": "f16", "concurrencies": [ 1, 2, 4, 8 ], "ctx_size": 8192, "flash_attn": true, "gen_tokens": 512, "n_gpu_layers": 999, "parallel": 8, "repetitions": 3, "request_timeout_s": 300, "requests_per_concurrency": 4, "spec": "off", "spec_draft_n_max": 3, "spec_type": "none", "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "formal": true, "concurrencies": [ 1, 2, 4, 8 ], "requests_per_concurrency": 4, "repetitions": 3, "gen_tokens": 512, "samples_count": 180, "conditions": [ { "concurrency": 1, "repetitions": 3, "requests_per_repetition": 4, "successful_requests": 12, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 56.45513501237678, "stddev": 0.2010027976792921, "min": 56.315266903561806, "max": 56.685474206282024 }, "per_request_generation_throughput_tps": { "mean": 56.45979777126875, "stddev": 0.46878728748777143, "min": 55.417514309716125, "max": 57.08806930225007 }, "latency_p50_s": { "mean": 9.067750368666262, "stddev": 0.03667504720405309, "min": 9.02676463050011, "max": 9.097472866000317 }, "latency_p95_s": { "mean": 9.143598081316426, "stddev": 0.06768269521846125, "min": 9.09009649050131, "max": 9.219685800698063 }, "latency_p99_s": { "mean": 9.150731639196431, "stddev": 0.07431280003745748, "min": 9.094996926901512, "max": 9.235102301737934 } }, { "concurrency": 2, "repetitions": 3, "requests_per_repetition": 8, "successful_requests": 24, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 85.00902177714305, "stddev": 0.1252348757424851, "min": 84.86479268950534, "max": 85.0902055994134 }, "per_request_generation_throughput_tps": { "mean": 42.509167793765315, "stddev": 0.13607877354245132, "min": 42.147807348723724, "max": 42.712890563973325 }, "latency_p50_s": { "mean": 12.036808877499425, "stddev": 0.011345909498862381, "min": 12.025004410501424, "max": 12.047632551999413 }, "latency_p95_s": { "mean": 12.093290716952591, "stddev": 0.049990475072435266, "min": 12.041228751101517, "max": 12.140913628503403 }, "latency_p99_s": { "mean": 12.095147382859844, "stddev": 0.05260249140681704, "min": 12.041260059021225, "max": 12.146363237703117 } }, { "concurrency": 4, "repetitions": 3, "requests_per_repetition": 16, "successful_requests": 48, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 135.90896219026627, "stddev": 4.386585723411369, "min": 131.1991407503789, "max": 139.87797353040665 }, "per_request_generation_throughput_tps": { "mean": 34.010729313265244, "stddev": 1.3432368092343172, "min": 31.963332735450127, "max": 35.085630238952696 }, "latency_p50_s": { "mean": 15.064861066166486, "stddev": 0.7098339493503385, "min": 14.631948098998691, "max": 15.884064601003047 }, "latency_p95_s": { "mean": 15.551220846999058, "stddev": 0.7502992734746085, "min": 14.685756239998227, "max": 16.01826437749878 }, "latency_p99_s": { "mean": 15.561466514202039, "stddev": 0.7463069917123463, "min": 14.700238915204318, "max": 16.018336606702725 } }, { "concurrency": 8, "repetitions": 3, "requests_per_repetition": 32, "successful_requests": 96, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 162.9143951993306, "stddev": 7.8554697684250705, "min": 157.25049487521412, "max": 171.88218924153242 }, "per_request_generation_throughput_tps": { "mean": 20.553187867468523, "stddev": 1.0355913452767824, "min": 19.096899615860693, "max": 21.95002023093021 }, "latency_p50_s": { "mean": 24.73374881983303, "stddev": 1.1578908111988377, "min": 23.425235157003044, "max": 25.625818868498754 }, "latency_p95_s": { "mean": 26.05200714786576, "stddev": 0.8041211283682219, "min": 25.1319721318494, "max": 26.620479559248633 }, "latency_p99_s": { "mean": 26.14234234625219, "stddev": 0.8744604801501145, "min": 25.1416815452437, "max": 26.75967532082279 } } ] }, "knowledge": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-013315-178550", "evaluator_version": "1.2", "condition": { "dataset_version": 1.2, "max_tokens": 128, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.2, "total": 100, "correct": 93, "accuracy": 0.93, "pass_rate": 0.93, "categories": { "general": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "korea": { "total": 20, "correct": 19, "accuracy": 0.95, "pass_rate": 0.95 }, "math": { "total": 20, "correct": 17, "accuracy": 0.85, "pass_rate": 0.85 }, "science": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "logic": { "total": 20, "correct": 17, "accuracy": 0.85, "pass_rate": 0.85 } }, "difficulties": { "easy": { "total": 25, "correct": 24, "accuracy": 0.96, "pass_rate": 0.96 }, "medium": { "total": 40, "correct": 39, "accuracy": 0.975, "pass_rate": 0.975 }, "hard": { "total": 35, "correct": 30, "accuracy": 0.8571428571428571, "pass_rate": 0.8571428571428571 } } }, "coding": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-154314-4cdc32", "evaluator_version": "existing-v1", "condition": { "dataset_version": 1, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto" }, "version": 1, "total": 12, "passed": 9, "pass_rate": 0.75, "categories": { "basic": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "string": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "collection": { "total": 3, "passed": 1, "pass_rate": 0.3333333333333333 }, "edge_case": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 } }, "failure_types": { "syntax_error": 2, "test_failure": 1 } }, "tool_call": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-011652-d8b541", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 15, "passed": 9, "pass_rate": 0.6, "tool_selection_accuracy": 0.7333333333333333, "argument_accuracy": 0.6, "execution_success_rate": 0.8666666666666667, "tool_rate": 0.7333333333333333, "args_rate": 0.6, "exec_rate": 0.8666666666666667, "failure_types": { "wrong_arguments": 2, "missing_tool_call": 2, "incomplete_sequence": 2 } }, "agent_single": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-011723-ef8d8d", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 12, "passed": 9, "correct": 9, "pass_rate": 0.75, "avg_steps": 3.1666666666666665, "avg_tool_calls": 2.1666666666666665, "tool_calls_total": 26, "failed_tool_calls": 1, "repeated_tool_calls": 4, "failure_types": { "tool_error": 1, "missing_required_step": 2 } }, "agent_multi": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-011804-db9de7", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 8, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 10, "passed": 7, "correct": 7, "pass_rate": 0.7, "handoff_success_rate": 1.0, "role_participation_success_rate": 1.0, "avg_steps": 3.1, "avg_tool_calls": 2.1, "tool_calls_total": 21, "failed_tool_calls": 0, "invalid_tool_calls": 0, "handoffs": 11, "failure_types": { "missing_required_step": 3 } } }, "provenance": { "hardware": "NVIDIA DGX Spark GB10", "reasoning_mode": "off", "raw_artifacts_public": false } }, { "model_id": "n2-mini-q5-k-m", "model": "N2 Mini", "variant": "Q5_K_M", "quantization": "Q5_K_M", "benchmark_versions": { "performance": "existing-v1", "server_performance": "existing-v1", "knowledge": "1.2", "coding": "existing-v1", "tool_call": "1.1", "agent_single": "1.1", "agent_multi": "1.1" }, "suites": { "performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-154830-2ed4f8", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_type_k": "f16", "cache_type_v": "f16", "flash_attn": true, "gen_tokens": [ 512 ], "n_gpu_layers": 999, "prompt_tokens": [ 512, 2048, 8192, 32768 ], "repetitions": 5, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "metrics": { "pp": { "token_count": 32768, "mean_tps": 1732.157584, "stddev_tps": 2.224269, "repetitions": 5, "unit": "tokens/s" }, "tg": { "token_count": 512, "mean_tps": 64.075384, "stddev_tps": 0.120109, "repetitions": 5, "unit": "tokens/s" } }, "repetitions": 5 }, "server_performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-163531-7e5f86", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_prompt": false, "cache_type_k": "f16", "cache_type_v": "f16", "concurrencies": [ 1, 2, 4, 8 ], "ctx_size": 8192, "flash_attn": true, "gen_tokens": 512, "n_gpu_layers": 999, "parallel": 8, "repetitions": 3, "request_timeout_s": 300, "requests_per_concurrency": 4, "spec": "off", "spec_draft_n_max": 3, "spec_type": "none", "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "formal": true, "concurrencies": [ 1, 2, 4, 8 ], "requests_per_concurrency": 4, "repetitions": 3, "gen_tokens": 512, "samples_count": 180, "conditions": [ { "concurrency": 1, "repetitions": 3, "requests_per_repetition": 4, "successful_requests": 12, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 60.49887066248982, "stddev": 0.193700423938216, "min": 60.27980068548933, "max": 60.64747114876747 }, "per_request_generation_throughput_tps": { "mean": 60.503767916218095, "stddev": 0.46388780382524186, "min": 59.57872354003143, "max": 61.163753761784974 }, "latency_p50_s": { "mean": 8.459056847833077, "stddev": 0.030713559958008584, "min": 8.432969740497356, "max": 8.49290712850052 }, "latency_p95_s": { "mean": 8.530642237849436, "stddev": 0.05252780113006592, "min": 8.479937063649777, "max": 8.584820314946773 }, "latency_p99_s": { "mean": 8.5376453947694, "stddev": 0.053380789599868826, "min": 8.485185305530395, "max": 8.591901551785776 } }, { "concurrency": 2, "repetitions": 3, "requests_per_repetition": 8, "successful_requests": 24, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 89.91938905225406, "stddev": 0.29690214358006123, "min": 89.62640455384347, "max": 90.22005959315685 }, "per_request_generation_throughput_tps": { "mean": 44.966036560059784, "stddev": 0.3148135446201516, "min": 44.35087691434621, "max": 45.5168907002321 }, "latency_p50_s": { "mean": 11.39640763516824, "stddev": 0.0303351673770257, "min": 11.362329484003567, "max": 11.42046326900163 }, "latency_p95_s": { "mean": 11.464752034215294, "stddev": 0.07753326876350458, "min": 11.378999122549066, "max": 11.529904853547123 }, "latency_p99_s": { "mean": 11.4699728265752, "stddev": 0.08292300570450396, "min": 11.379045167707881, "max": 11.541424066706167 } }, { "concurrency": 4, "repetitions": 3, "requests_per_repetition": 16, "successful_requests": 48, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 137.38063379658155, "stddev": 10.34338798410712, "min": 128.48194231015006, "max": 148.7289390527786 }, "per_request_generation_throughput_tps": { "mean": 34.44426123989281, "stddev": 2.722185026311975, "min": 30.25580450735529, "max": 37.271393634084816 }, "latency_p50_s": { "mean": 14.910943151667501, "stddev": 1.0752965687072278, "min": 13.773066933001246, "max": 15.910219492998294 }, "latency_p95_s": { "mean": 15.850268737418446, "stddev": 1.7832690562705036, "min": 13.791699490249812, "max": 16.921601167254266 }, "latency_p99_s": { "mean": 15.850773680019119, "stddev": 1.783267775227006, "min": 13.792209037253997, "max": 16.922218387850442 } }, { "concurrency": 8, "repetitions": 3, "requests_per_repetition": 32, "successful_requests": 96, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 170.9879666778922, "stddev": 6.3077545947804925, "min": 164.11845637586316, "max": 176.51918055392665 }, "per_request_generation_throughput_tps": { "mean": 21.578558019804607, "stddev": 0.9219245695016312, "min": 19.666447163728073, "max": 22.584426381783082 }, "latency_p50_s": { "mean": 23.380708969833602, "stddev": 0.6245899358941382, "min": 22.841312518499763, "max": 24.06501687849959 }, "latency_p95_s": { "mean": 24.86160668820133, "stddev": 1.0155674451260859, "min": 24.21929039080278, "max": 26.032440990603938 }, "latency_p99_s": { "mean": 24.86628889250079, "stddev": 1.0129946870497768, "min": 24.219640211391088, "max": 26.033735468520025 } } ] }, "knowledge": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-013406-4ef35a", "evaluator_version": "1.2", "condition": { "dataset_version": 1.2, "max_tokens": 128, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.2, "total": 100, "correct": 92, "accuracy": 0.92, "pass_rate": 0.92, "categories": { "general": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "korea": { "total": 20, "correct": 19, "accuracy": 0.95, "pass_rate": 0.95 }, "math": { "total": 20, "correct": 17, "accuracy": 0.85, "pass_rate": 0.85 }, "science": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "logic": { "total": 20, "correct": 16, "accuracy": 0.8, "pass_rate": 0.8 } }, "difficulties": { "easy": { "total": 25, "correct": 24, "accuracy": 0.96, "pass_rate": 0.96 }, "medium": { "total": 40, "correct": 38, "accuracy": 0.95, "pass_rate": 0.95 }, "hard": { "total": 35, "correct": 30, "accuracy": 0.8571428571428571, "pass_rate": 0.8571428571428571 } } }, "coding": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-155244-eaa71f", "evaluator_version": "existing-v1", "condition": { "dataset_version": 1, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto" }, "version": 1, "total": 12, "passed": 9, "pass_rate": 0.75, "categories": { "basic": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "string": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 }, "collection": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 }, "edge_case": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 } }, "failure_types": { "syntax_error": 3 } }, "tool_call": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-011932-b32a24", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 15, "passed": 8, "pass_rate": 0.5333333333333333, "tool_selection_accuracy": 0.7333333333333333, "argument_accuracy": 0.5333333333333333, "execution_success_rate": 0.8, "tool_rate": 0.7333333333333333, "args_rate": 0.5333333333333333, "exec_rate": 0.8, "failure_types": { "wrong_arguments": 3, "missing_tool_call": 2, "incomplete_sequence": 2 } }, "agent_single": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-011958-55657b", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 12, "passed": 6, "correct": 6, "pass_rate": 0.5, "avg_steps": 2.9166666666666665, "avg_tool_calls": 1.9166666666666667, "tool_calls_total": 23, "failed_tool_calls": 1, "repeated_tool_calls": 2, "failure_types": { "missing_required_step": 5, "tool_error": 1 } }, "agent_multi": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-012040-fa2694", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 8, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 10, "passed": 6, "correct": 6, "pass_rate": 0.6, "handoff_success_rate": 0.7, "role_participation_success_rate": 0.7, "avg_steps": 3.3, "avg_tool_calls": 2.3, "tool_calls_total": 23, "failed_tool_calls": 3, "invalid_tool_calls": 0, "handoffs": 11, "failure_types": { "tool_error": 3, "missing_required_step": 1 } } }, "provenance": { "hardware": "NVIDIA DGX Spark GB10", "reasoning_mode": "off", "raw_artifacts_public": false } }, { "model_id": "n2-mini-q6-k", "model": "N2 Mini", "variant": "Q6_K", "quantization": "Q6_K", "benchmark_versions": { "performance": "existing-v1", "server_performance": "existing-v1", "knowledge": "1.2", "coding": "existing-v1", "tool_call": "1.1", "agent_single": "1.1", "agent_multi": "1.1" }, "suites": { "performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-155619-88ff7e", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_type_k": "f16", "cache_type_v": "f16", "flash_attn": true, "gen_tokens": [ 512 ], "n_gpu_layers": 999, "prompt_tokens": [ 512, 2048, 8192, 32768 ], "repetitions": 5, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "metrics": { "pp": { "token_count": 32768, "mean_tps": 1565.12997, "stddev_tps": 3.664127, "repetitions": 5, "unit": "tokens/s" }, "tg": { "token_count": 512, "mean_tps": 58.682543, "stddev_tps": 0.119688, "repetitions": 5, "unit": "tokens/s" } }, "repetitions": 5 }, "server_performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-164726-020f2f", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_prompt": false, "cache_type_k": "f16", "cache_type_v": "f16", "concurrencies": [ 1, 2, 4, 8 ], "ctx_size": 8192, "flash_attn": true, "gen_tokens": 512, "n_gpu_layers": 999, "parallel": 8, "repetitions": 3, "request_timeout_s": 300, "requests_per_concurrency": 4, "spec": "off", "spec_draft_n_max": 3, "spec_type": "none", "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "formal": true, "concurrencies": [ 1, 2, 4, 8 ], "requests_per_concurrency": 4, "repetitions": 3, "gen_tokens": 512, "samples_count": 180, "conditions": [ { "concurrency": 1, "repetitions": 3, "requests_per_repetition": 4, "successful_requests": 12, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 55.36139878739795, "stddev": 0.19650010472127868, "min": 55.16739440139576, "max": 55.560302966711426 }, "per_request_generation_throughput_tps": { "mean": 55.364386486526925, "stddev": 0.3490050161973834, "min": 54.623087886472234, "max": 55.71739469671933 }, "latency_p50_s": { "mean": 9.232761977501164, "stddev": 0.030176926914448354, "min": 9.214238948003185, "max": 9.267583616499905 }, "latency_p95_s": { "mean": 9.307608657816187, "stddev": 0.06130107185058697, "min": 9.237537441498716, "max": 9.35132412155399 }, "latency_p99_s": { "mean": 9.316540162495972, "stddev": 0.06952146965611215, "min": 9.237668583498452, "max": 9.368925479514074 } }, { "concurrency": 2, "repetitions": 3, "requests_per_repetition": 8, "successful_requests": 24, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 82.01332243615636, "stddev": 0.3220693564754001, "min": 81.78239289501224, "max": 82.381239837237 }, "per_request_generation_throughput_tps": { "mean": 41.01559228301552, "stddev": 0.47516746386496667, "min": 39.73565899111502, "max": 41.504960826294344 }, "latency_p50_s": { "mean": 12.445088844666316, "stddev": 0.05682113876299758, "min": 12.401788118997501, "max": 12.509429103000002 }, "latency_p95_s": { "mean": 12.666509636830718, "stddev": 0.21145747045301932, "min": 12.4556743736488, "max": 12.878583869997238 }, "latency_p99_s": { "mean": 12.668352569763398, "stddev": 0.21397263081671009, "min": 12.455925791527697, "max": 12.883838254798029 } }, { "concurrency": 4, "repetitions": 3, "requests_per_repetition": 16, "successful_requests": 48, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 130.10327827893147, "stddev": 5.70857458284005, "min": 123.89550988001585, "max": 135.1269365267908 }, "per_request_generation_throughput_tps": { "mean": 32.57626148062587, "stddev": 1.6944768537777686, "min": 30.131325113597978, "max": 33.95679345823964 }, "latency_p50_s": { "mean": 15.771022521666358, "stddev": 1.026112447070977, "min": 15.156200358000206, "max": 16.955589253502694 }, "latency_p95_s": { "mean": 16.36823126024865, "stddev": 1.012175065046637, "min": 15.200307593247999, "max": 16.990456566500143 }, "latency_p99_s": { "mean": 16.369212444848745, "stddev": 1.0122737862538613, "min": 15.201193693048117, "max": 16.991917620499226 } }, { "concurrency": 8, "repetitions": 3, "requests_per_repetition": 32, "successful_requests": 96, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 157.3739105518877, "stddev": 3.1055456657357237, "min": 154.55216504828215, "max": 160.70121961012785 }, "per_request_generation_throughput_tps": { "mean": 19.873997565695422, "stddev": 0.5499123107772382, "min": 18.672144979183248, "max": 20.63321728159487 }, "latency_p50_s": { "mean": 25.600388379001135, "stddev": 0.4354116250732988, "min": 25.238848763499846, "max": 26.083730679001746 }, "latency_p95_s": { "mean": 26.603363418680722, "stddev": 0.704488707542332, "min": 26.180963628445898, "max": 27.416633203397215 }, "latency_p99_s": { "mean": 26.60533593560467, "stddev": 0.705151558131633, "min": 26.18157451844032, "max": 27.41934488295963 } } ] }, "knowledge": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-013458-772406", "evaluator_version": "1.2", "condition": { "dataset_version": 1.2, "max_tokens": 128, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.2, "total": 100, "correct": 93, "accuracy": 0.93, "pass_rate": 0.93, "categories": { "general": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "korea": { "total": 20, "correct": 19, "accuracy": 0.95, "pass_rate": 0.95 }, "math": { "total": 20, "correct": 17, "accuracy": 0.85, "pass_rate": 0.85 }, "science": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "logic": { "total": 20, "correct": 17, "accuracy": 0.85, "pass_rate": 0.85 } }, "difficulties": { "easy": { "total": 25, "correct": 24, "accuracy": 0.96, "pass_rate": 0.96 }, "medium": { "total": 40, "correct": 39, "accuracy": 0.975, "pass_rate": 0.975 }, "hard": { "total": 35, "correct": 30, "accuracy": 0.8571428571428571, "pass_rate": 0.8571428571428571 } } }, "coding": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-160053-23c1f9", "evaluator_version": "existing-v1", "condition": { "dataset_version": 1, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto" }, "version": 1, "total": 12, "passed": 8, "pass_rate": 0.6666666666666666, "categories": { "basic": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "string": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 }, "collection": { "total": 3, "passed": 1, "pass_rate": 0.3333333333333333 }, "edge_case": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 } }, "failure_types": { "syntax_error": 4 } }, "tool_call": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-012217-7896f9", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 15, "passed": 8, "pass_rate": 0.5333333333333333, "tool_selection_accuracy": 0.6666666666666666, "argument_accuracy": 0.5333333333333333, "execution_success_rate": 0.8, "tool_rate": 0.6666666666666666, "args_rate": 0.5333333333333333, "exec_rate": 0.8, "failure_types": { "wrong_arguments": 2, "missing_tool_call": 3, "incomplete_sequence": 2 } }, "agent_single": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-012248-967a21", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 12, "passed": 8, "correct": 8, "pass_rate": 0.6666666666666666, "avg_steps": 3.1666666666666665, "avg_tool_calls": 2.1666666666666665, "tool_calls_total": 26, "failed_tool_calls": 2, "repeated_tool_calls": 4, "failure_types": { "tool_error": 2, "missing_required_step": 2 } }, "agent_multi": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-012334-65f1f4", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 8, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 10, "passed": 9, "correct": 9, "pass_rate": 0.9, "handoff_success_rate": 1.0, "role_participation_success_rate": 1.0, "avg_steps": 3.3, "avg_tool_calls": 2.3, "tool_calls_total": 23, "failed_tool_calls": 0, "invalid_tool_calls": 0, "handoffs": 11, "failure_types": { "missing_required_step": 1 } } }, "provenance": { "hardware": "NVIDIA DGX Spark GB10", "reasoning_mode": "off", "raw_artifacts_public": false } }, { "model_id": "gemma4-nvfp4", "model": "Gemma4", "variant": "NVFP4", "quantization": "NVFP4", "benchmark_versions": { "performance": "existing-v1", "server_performance": "existing-v1", "knowledge": "1.2", "coding": "existing-v1", "tool_call": "1.1", "agent_single": "1.1", "agent_multi": "1.1" }, "suites": { "performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-201154-565aad", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_type_k": "f16", "cache_type_v": "f16", "flash_attn": true, "gen_tokens": [ 512 ], "n_gpu_layers": 999, "prompt_tokens": [ 512, 2048, 8192, 32768 ], "repetitions": 5, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "metrics": { "pp": { "token_count": 32768, "mean_tps": 2609.503148, "stddev_tps": 20.36111, "repetitions": 5, "unit": "tokens/s" }, "tg": { "token_count": 512, "mean_tps": 37.432661, "stddev_tps": 0.020152, "repetitions": 5, "unit": "tokens/s" } }, "repetitions": 5 }, "server_performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-201504-70bab9", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_prompt": false, "cache_type_k": "f16", "cache_type_v": "f16", "concurrencies": [ 1, 2, 4, 8 ], "ctx_size": 8192, "flash_attn": true, "gen_tokens": 512, "n_gpu_layers": 999, "parallel": 8, "repetitions": 3, "request_timeout_s": 300, "requests_per_concurrency": 4, "spec": "off", "spec_draft_n_max": 3, "spec_type": "none", "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "formal": true, "concurrencies": [ 1, 2, 4, 8 ], "requests_per_concurrency": 4, "repetitions": 3, "gen_tokens": 512, "samples_count": 180, "conditions": [ { "concurrency": 1, "repetitions": 3, "requests_per_repetition": 4, "successful_requests": 12, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 36.07108293153662, "stddev": 0.13148318836800194, "min": 35.926786492127384, "max": 36.18411797067951 }, "per_request_generation_throughput_tps": { "mean": 36.07236448606724, "stddev": 0.19946154378843817, "min": 35.59403032445276, "max": 36.332351912076895 }, "latency_p50_s": { "mean": 14.18238037916793, "stddev": 0.04779365916554936, "min": 14.14814421000483, "max": 14.23698375949607 }, "latency_p95_s": { "mean": 14.270056867950673, "stddev": 0.08610094906395552, "min": 14.205443503457355, "max": 14.36780213420061 }, "latency_p99_s": { "mean": 14.278965093590863, "stddev": 0.09042691135278831, "min": 14.20912835269817, "max": 14.381108076441597 } }, { "concurrency": 2, "repetitions": 3, "requests_per_repetition": 8, "successful_requests": 24, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 63.017280282068555, "stddev": 0.25439436735546106, "min": 62.72387008221037, "max": 63.17620565640412 }, "per_request_generation_throughput_tps": { "mean": 31.51132040030469, "stddev": 0.2678261533950897, "min": 30.684350599290916, "max": 31.728662201924983 }, "latency_p50_s": { "mean": 16.224527087499155, "stddev": 0.014714664952588908, "min": 16.209276076500828, "max": 16.23863922250166 }, "latency_p95_s": { "mean": 16.39609732820148, "stddev": 0.2418973485975657, "min": 16.256258413457545, "max": 16.675416249997216 }, "latency_p99_s": { "mean": 16.398998903242173, "stddev": 0.2467381269116514, "min": 16.25628216670011, "max": 16.683907389998204 } }, { "concurrency": 4, "repetitions": 3, "requests_per_repetition": 16, "successful_requests": 48, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 114.81991277922204, "stddev": 4.594007316862012, "min": 110.67220050054719, "max": 119.75772865420156 }, "per_request_generation_throughput_tps": { "mean": 28.91218187959015, "stddev": 2.6133124825459593, "min": 25.54200283798327, "max": 31.66971897696568 }, "latency_p50_s": { "mean": 17.754503211001673, "stddev": 1.4712647272028865, "min": 16.27240473400161, "max": 19.214689415501198 }, "latency_p95_s": { "mean": 19.671873481833853, "stddev": 0.3728375997154407, "min": 19.299319536003168, "max": 20.044994089497777 }, "latency_p99_s": { "mean": 19.67782038623433, "stddev": 0.3729293539101469, "min": 19.29969725040428, "max": 20.04532935149764 } }, { "concurrency": 8, "repetitions": 3, "requests_per_repetition": 32, "successful_requests": 96, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 175.38608757441546, "stddev": 4.460763416608497, "min": 170.70026007422058, "max": 179.5812096653228 }, "per_request_generation_throughput_tps": { "mean": 22.09067619425695, "stddev": 1.612166307383009, "min": 20.32793101370162, "max": 24.469756754406056 }, "latency_p50_s": { "mean": 23.34479263916243, "stddev": 1.3962590144357558, "min": 22.521120770994457, "max": 24.956925505503023 }, "latency_p95_s": { "mean": 24.859499505233302, "stddev": 0.5056746095008999, "min": 24.27705530785097, "max": 25.186441043948435 }, "latency_p99_s": { "mean": 24.861593682676805, "stddev": 0.5055522939906236, "min": 24.27916668122285, "max": 25.186971681676077 } } ], "alternatives": [ { "source_run_id": "20260905-212404-c6e555", "condition": { "batch_size": 2048, "cache_prompt": false, "cache_type_k": "f16", "cache_type_v": "f16", "concurrencies": [ 1, 2, 4, 8 ], "ctx_size": 8192, "flash_attn": true, "gen_tokens": 512, "n_gpu_layers": 999, "parallel": 8, "repetitions": 3, "request_timeout_s": 300, "requests_per_concurrency": 4, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "formal": true, "concurrencies": [ 1, 2, 4, 8 ], "requests_per_concurrency": 4, "repetitions": 3, "gen_tokens": 512, "samples_count": 180, "conditions": [ { "concurrency": 1, "repetitions": 3, "requests_per_repetition": 4, "successful_requests": 12, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 64.04227315588989, "stddev": 2.28725609910979, "min": 61.417468009977505, "max": 65.60832691139578 }, "per_request_generation_throughput_tps": { "mean": 64.05799734379944, "stddev": 2.1531566119625003, "min": 59.2028088944886, "max": 65.70679876997876 }, "latency_p50_s": { "mean": 7.97681136100194, "stddev": 0.2703806288458357, "min": 7.8030348860047525, "max": 8.2883259260052 }, "latency_p95_s": { "mean": 8.125808377535336, "stddev": 0.4228425653176118, "min": 7.814494858955731, "max": 8.607208524949238 }, "latency_p99_s": { "mean": 8.142267014175207, "stddev": 0.437976064639824, "min": 7.815955130195944, "max": 8.64003235378812 } }, { "concurrency": 2, "repetitions": 3, "requests_per_repetition": 8, "successful_requests": 24, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 90.4514163030455, "stddev": 0.1457904280456717, "min": 90.30385103960491, "max": 90.59536458646961 }, "per_request_generation_throughput_tps": { "mean": 46.10516706896656, "stddev": 3.315502870366178, "min": 41.30601020691439, "max": 52.81224738184816 }, "latency_p50_s": { "mean": 11.143525624332446, "stddev": 0.0340214438449715, "min": 11.106775338004809, "max": 11.173922455498541 }, "latency_p95_s": { "mean": 12.247956666369038, "stddev": 0.042185279371849245, "min": 12.199249250351567, "max": 12.272847509702114 }, "latency_p99_s": { "mean": 12.352396044211005, "stddev": 0.023818518214164697, "min": 12.325494546874543, "max": 12.37080198114112 } }, { "concurrency": 4, "repetitions": 3, "requests_per_repetition": 16, "successful_requests": 48, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 149.61582452445987, "stddev": 9.534993005375972, "min": 138.7098223192836, "max": 156.37665915547834 }, "per_request_generation_throughput_tps": { "mean": 40.89705579027077, "stddev": 6.705038766722148, "min": 31.937932877189052, "max": 52.3608965797365 }, "latency_p50_s": { "mean": 12.24418738799674, "stddev": 2.1043958632492266, "min": 10.961326078991988, "max": 14.672845531000348 }, "latency_p95_s": { "mean": 15.551503829084444, "stddev": 0.3669494075866195, "min": 15.13217855874609, "max": 15.813859974005027 }, "latency_p99_s": { "mean": 15.779505033284416, "stddev": 0.2868123104320766, "min": 15.45234441654975, "max": 15.98764709160314 } }, { "concurrency": 8, "repetitions": 3, "requests_per_repetition": 32, "successful_requests": 95, "failed_requests": 1, "failure_rate": 0.010416666666666666, "aggregate_generation_throughput_tps": { "mean": 183.25687242335485, "stddev": 9.844901447257282, "min": 175.37426264600387, "max": 194.29186689007904 }, "per_request_generation_throughput_tps": { "mean": 25.725429158812975, "stddev": 7.09465076960136, "min": 6.774970927411723, "max": 38.496338414064425 }, "latency_p50_s": { "mean": 18.632614055163383, "stddev": 0.7699834902533863, "min": 17.8914596064933, "max": 19.428505443000176 }, "latency_p95_s": { "mean": 27.687419824289343, "stddev": 5.455297131601262, "min": 21.39065286710902, "max": 30.98848339090182 }, "latency_p99_s": { "mean": 28.788620760280658, "stddev": 6.045656658878261, "min": 21.81334222835445, "max": 32.51932686597327 } } ] } ] }, "knowledge": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-015332-2d0b6b", "evaluator_version": "1.2", "condition": { "dataset_version": 1.2, "max_tokens": 128, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.2, "total": 100, "correct": 98, "accuracy": 0.98, "pass_rate": 0.98, "categories": { "general": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "korea": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "math": { "total": 20, "correct": 19, "accuracy": 0.95, "pass_rate": 0.95 }, "science": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "logic": { "total": 20, "correct": 19, "accuracy": 0.95, "pass_rate": 0.95 } }, "difficulties": { "easy": { "total": 25, "correct": 25, "accuracy": 1.0, "pass_rate": 1.0 }, "medium": { "total": 40, "correct": 39, "accuracy": 0.975, "pass_rate": 0.975 }, "hard": { "total": 35, "correct": 34, "accuracy": 0.9714285714285714, "pass_rate": 0.9714285714285714 } } }, "coding": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-203019-49bf2b", "evaluator_version": "existing-v1", "condition": { "dataset_version": 1, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto" }, "version": 1, "total": 12, "passed": 4, "pass_rate": 0.3333333333333333, "categories": { "basic": { "total": 3, "passed": 1, "pass_rate": 0.3333333333333333 }, "string": { "total": 3, "passed": 1, "pass_rate": 0.3333333333333333 }, "collection": { "total": 3, "passed": 1, "pass_rate": 0.3333333333333333 }, "edge_case": { "total": 3, "passed": 1, "pass_rate": 0.3333333333333333 } }, "failure_types": { "syntax_error": 7, "extraction_error": 1 } }, "tool_call": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-015418-4a3ac8", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 15, "passed": 11, "pass_rate": 0.7333333333333333, "tool_selection_accuracy": 0.8666666666666667, "argument_accuracy": 0.7333333333333333, "execution_success_rate": 0.9333333333333333, "tool_rate": 0.8666666666666667, "args_rate": 0.7333333333333333, "exec_rate": 0.9333333333333333, "failure_types": { "wrong_arguments": 2, "wrong_tool": 1, "incomplete_sequence": 1 } }, "agent_single": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-015454-69cbe0", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 12, "passed": 12, "correct": 12, "pass_rate": 1.0, "avg_steps": 3.75, "avg_tool_calls": 2.75, "tool_calls_total": 33, "failed_tool_calls": 5, "repeated_tool_calls": 5, "failure_types": {} }, "agent_multi": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-015541-b2872f", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 8, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 10, "passed": 7, "correct": 7, "pass_rate": 0.7, "handoff_success_rate": 1.0, "role_participation_success_rate": 1.0, "avg_steps": 3.8, "avg_tool_calls": 2.8, "tool_calls_total": 28, "failed_tool_calls": 4, "invalid_tool_calls": 0, "handoffs": 14, "failure_types": { "tool_error": 3 } } }, "provenance": { "hardware": "NVIDIA DGX Spark GB10", "reasoning_mode": "off", "raw_artifacts_public": false } }, { "model_id": "gemma4-q4-0", "model": "Gemma4", "variant": "Q4_0", "quantization": "Q4_0", "benchmark_versions": { "performance": "existing-v1", "server_performance": "existing-v1", "knowledge": "1.2", "coding": "existing-v1", "tool_call": "1.1", "agent_single": "1.1", "agent_multi": "1.1" }, "suites": { "performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-204203-52bb92", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_type_k": "f16", "cache_type_v": "f16", "flash_attn": true, "gen_tokens": [ 512 ], "n_gpu_layers": 999, "prompt_tokens": [ 512, 2048, 8192, 32768 ], "repetitions": 5, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "metrics": { "pp": { "token_count": 32768, "mean_tps": 2516.309271, "stddev_tps": 7.838954, "repetitions": 5, "unit": "tokens/s" }, "tg": { "token_count": 512, "mean_tps": 51.713058, "stddev_tps": 0.036374, "repetitions": 5, "unit": "tokens/s" } }, "repetitions": 5 }, "server_performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-204454-8fee22", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_prompt": false, "cache_type_k": "f16", "cache_type_v": "f16", "concurrencies": [ 1, 2, 4, 8 ], "ctx_size": 8192, "flash_attn": true, "gen_tokens": 512, "n_gpu_layers": 999, "parallel": 8, "repetitions": 3, "request_timeout_s": 300, "requests_per_concurrency": 4, "spec": "off", "spec_draft_n_max": 3, "spec_type": "none", "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "formal": true, "concurrencies": [ 1, 2, 4, 8 ], "requests_per_concurrency": 4, "repetitions": 3, "gen_tokens": 512, "samples_count": 180, "conditions": [ { "concurrency": 1, "repetitions": 3, "requests_per_repetition": 4, "successful_requests": 12, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 49.72395345157007, "stddev": 0.2655856633034683, "min": 49.43162552832987, "max": 49.950392150953036 }, "per_request_generation_throughput_tps": { "mean": 49.72964622661754, "stddev": 0.5383923290283947, "min": 48.35555211262779, "max": 50.16951649566191 }, "latency_p50_s": { "mean": 10.251653278333833, "stddev": 0.03472072463537624, "min": 10.219962822993693, "max": 10.28876620450319 }, "latency_p95_s": { "mean": 10.427923865671133, "stddev": 0.1070914082525208, "min": 10.332963760851271, "max": 10.544000591403892 }, "latency_p99_s": { "mean": 10.451866969671197, "stddev": 0.11768938923903592, "min": 10.347431666572083, "max": 10.57938895668296 } }, { "concurrency": 2, "repetitions": 3, "requests_per_repetition": 8, "successful_requests": 24, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 79.06480118586718, "stddev": 0.5188026348612128, "min": 78.47640096270004, "max": 79.45644388039878 }, "per_request_generation_throughput_tps": { "mean": 39.533628585638745, "stddev": 0.22996966530226476, "min": 39.141271238016124, "max": 39.78040520790833 }, "latency_p50_s": { "mean": 12.951071913836737, "stddev": 0.09361079273608303, "min": 12.885312061509467, "max": 13.058246603497537 }, "latency_p95_s": { "mean": 12.980751573733869, "stddev": 0.08971460308064172, "min": 12.907427884457865, "max": 13.08078878779197 }, "latency_p99_s": { "mean": 12.980899435014969, "stddev": 0.08963853945211143, "min": 12.90753893050016, "max": 13.080815318352542 } }, { "concurrency": 4, "repetitions": 3, "requests_per_repetition": 16, "successful_requests": 48, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 136.98613785098243, "stddev": 14.99057288722432, "min": 121.31788328287004, "max": 151.19181318702755 }, "per_request_generation_throughput_tps": { "mean": 34.667877383299015, "stddev": 5.082577395251032, "min": 30.182384776568437, "max": 41.03069221620755 }, "latency_p50_s": { "mean": 14.775058838165327, "stddev": 2.134425436601118, "min": 12.615679031499894, "max": 16.883622437504528 }, "latency_p95_s": { "mean": 16.792134529496252, "stddev": 0.27876728565238107, "min": 16.470467924245895, "max": 16.963415121244907 }, "latency_p99_s": { "mean": 16.792466514695843, "stddev": 0.27887932928339, "min": 16.470657400039634, "max": 16.96351260024676 } }, { "concurrency": 8, "repetitions": 3, "requests_per_repetition": 32, "successful_requests": 96, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 192.99549965876759, "stddev": 4.1370741234985315, "min": 189.48299026987664, "max": 197.55569633692738 }, "per_request_generation_throughput_tps": { "mean": 24.160762063659053, "stddev": 0.9939301351830663, "min": 22.517201844060295, "max": 25.474226237800398 }, "latency_p50_s": { "mean": 21.339540386165027, "stddev": 0.7039445993771777, "min": 20.52974719749909, "max": 21.805381567501172 }, "latency_p95_s": { "mean": 22.111222409114028, "stddev": 0.5435155591337864, "min": 21.767996721100644, "max": 22.73786960164434 }, "latency_p99_s": { "mean": 22.112778001444045, "stddev": 0.5424313287713087, "min": 21.769296424563798, "max": 22.738112276599423 } } ], "alternatives": [ { "source_run_id": "20260905-213513-340ca6", "condition": { "batch_size": 2048, "cache_prompt": false, "cache_type_k": "f16", "cache_type_v": "f16", "concurrencies": [ 1, 2, 4, 8 ], "ctx_size": 8192, "flash_attn": true, "gen_tokens": 512, "n_gpu_layers": 999, "parallel": 8, "repetitions": 3, "request_timeout_s": 300, "requests_per_concurrency": 4, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "formal": true, "concurrencies": [ 1, 2, 4, 8 ], "requests_per_concurrency": 4, "repetitions": 3, "gen_tokens": 512, "samples_count": 180, "conditions": [ { "concurrency": 1, "repetitions": 3, "requests_per_repetition": 4, "successful_requests": 12, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 90.90668547979037, "stddev": 0.37164301665428034, "min": 90.48449363968429, "max": 91.18437020850355 }, "per_request_generation_throughput_tps": { "mean": 90.92199485420291, "stddev": 1.095291609843269, "min": 87.86500735499162, "max": 92.15285795631029 }, "latency_p50_s": { "mean": 5.62012046383461, "stddev": 0.004050759139974106, "min": 5.617504126006679, "max": 5.624786419502925 }, "latency_p95_s": { "mean": 5.700075364945708, "stddev": 0.08885766631489903, "min": 5.625697408302949, "max": 5.798474020093272 }, "latency_p99_s": { "mean": 5.710482412984614, "stddev": 0.10013307850667592, "min": 5.626725272862386, "max": 5.82139133521152 } }, { "concurrency": 2, "repetitions": 3, "requests_per_repetition": 8, "successful_requests": 24, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 128.72698116547923, "stddev": 0.3850767259500029, "min": 128.4207703048052, "max": 129.15930025320486 }, "per_request_generation_throughput_tps": { "mean": 69.28334921579295, "stddev": 8.783923814211764, "min": 48.88046152355727, "max": 77.88828044205675 }, "latency_p50_s": { "mean": 7.112475856168506, "stddev": 0.02982989835985501, "min": 7.079104616001132, "max": 7.136550189497939 }, "latency_p95_s": { "mean": 9.594465272163022, "stddev": 0.043975806906045065, "min": 9.54377236345099, "max": 9.62236972334722 }, "latency_p99_s": { "mean": 10.252633510163577, "stddev": 0.05795109168895415, "min": 10.18933396629247, "max": 10.303077026729296 } }, { "concurrency": 4, "repetitions": 3, "requests_per_repetition": 16, "successful_requests": 48, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 208.42425549526413, "stddev": 14.57044398399738, "min": 191.71359939534955, "max": 218.47171753034075 }, "per_request_generation_throughput_tps": { "mean": 58.74432718040644, "stddev": 12.384410011812097, "min": 36.481672153186366, "max": 80.1399936748932 }, "latency_p50_s": { "mean": 8.807095317497442, "stddev": 0.34188567910049567, "min": 8.607230611996783, "max": 9.201860491004481 }, "latency_p95_s": { "mean": 12.423348377248962, "stddev": 0.7584084897144301, "min": 11.919285190753726, "max": 13.295560319496872 }, "latency_p99_s": { "mean": 13.236575441850922, "stddev": 0.5712992440135359, "min": 12.814483140547964, "max": 13.886667595896142 } }, { "concurrency": 8, "repetitions": 3, "requests_per_repetition": 32, "successful_requests": 96, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 297.80679578759896, "stddev": 0.47166998665249776, "min": 297.2856118240635, "max": 298.2043112276339 }, "per_request_generation_throughput_tps": { "mean": 41.68563761215117, "stddev": 7.425610944125623, "min": 25.14073428653219, "max": 53.151661441711184 }, "latency_p50_s": { "mean": 12.214456870164819, "stddev": 0.18708252249312754, "min": 11.999155415505811, "max": 12.33739919448999 }, "latency_p95_s": { "mean": 17.675384406965893, "stddev": 0.41780078614240473, "min": 17.418307967150756, "max": 18.157463678195924 }, "latency_p99_s": { "mean": 19.514177483773917, "stddev": 0.05043651259263282, "min": 19.46167817827867, "max": 19.562260803057292 } } ] } ] }, "knowledge": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-015637-2b954c", "evaluator_version": "1.2", "condition": { "dataset_version": 1.2, "max_tokens": 128, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.2, "total": 100, "correct": 98, "accuracy": 0.98, "pass_rate": 0.98, "categories": { "general": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "korea": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "math": { "total": 20, "correct": 19, "accuracy": 0.95, "pass_rate": 0.95 }, "science": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "logic": { "total": 20, "correct": 19, "accuracy": 0.95, "pass_rate": 0.95 } }, "difficulties": { "easy": { "total": 25, "correct": 25, "accuracy": 1.0, "pass_rate": 1.0 }, "medium": { "total": 40, "correct": 40, "accuracy": 1.0, "pass_rate": 1.0 }, "hard": { "total": 35, "correct": 33, "accuracy": 0.9428571428571428, "pass_rate": 0.9428571428571428 } } }, "coding": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-205718-854c2d", "evaluator_version": "existing-v1", "condition": { "dataset_version": 1, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto" }, "version": 1, "total": 12, "passed": 3, "pass_rate": 0.25, "categories": { "basic": { "total": 3, "passed": 1, "pass_rate": 0.3333333333333333 }, "string": { "total": 3, "passed": 0, "pass_rate": 0.0 }, "collection": { "total": 3, "passed": 1, "pass_rate": 0.3333333333333333 }, "edge_case": { "total": 3, "passed": 1, "pass_rate": 0.3333333333333333 } }, "failure_types": { "syntax_error": 7, "test_failure": 2 } }, "tool_call": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-015713-f535c9", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 15, "passed": 12, "pass_rate": 0.8, "tool_selection_accuracy": 0.8666666666666667, "argument_accuracy": 0.8, "execution_success_rate": 0.9333333333333333, "tool_rate": 0.8666666666666667, "args_rate": 0.8, "exec_rate": 0.9333333333333333, "failure_types": { "wrong_arguments": 1, "wrong_tool": 1, "incomplete_sequence": 1 } }, "agent_single": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-015739-e15da7", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 12, "passed": 12, "correct": 12, "pass_rate": 1.0, "avg_steps": 3.75, "avg_tool_calls": 2.75, "tool_calls_total": 33, "failed_tool_calls": 5, "repeated_tool_calls": 5, "failure_types": {} }, "agent_multi": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-015820-d0681f", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 8, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 10, "passed": 7, "correct": 7, "pass_rate": 0.7, "handoff_success_rate": 1.0, "role_participation_success_rate": 1.0, "avg_steps": 3.9, "avg_tool_calls": 2.9, "tool_calls_total": 29, "failed_tool_calls": 5, "invalid_tool_calls": 0, "handoffs": 14, "failure_types": { "tool_error": 3 } } }, "provenance": { "hardware": "NVIDIA DGX Spark GB10", "reasoning_mode": "off", "raw_artifacts_public": false } }, { "model_id": "qwen3-6-35b-a3b-hq", "model": "Qwen3.6 35B-A3B", "variant": "HQ", "quantization": "NVFP4 MTP HQ", "benchmark_versions": { "performance": "existing-v1", "server_performance": "existing-v1", "knowledge": "1.2", "coding": "existing-v1", "tool_call": "1.1", "agent_single": "1.1", "agent_multi": "1.1" }, "suites": { "performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-115655-385b40", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_type_k": "f16", "cache_type_v": "f16", "flash_attn": true, "gen_tokens": [ 512 ], "n_gpu_layers": 999, "prompt_tokens": [ 512, 2048, 8192, 32768 ], "repetitions": 5, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "metrics": { "pp": { "token_count": 32768, "mean_tps": 2091.597452, "stddev_tps": 9.09916, "repetitions": 5, "unit": "tokens/s" }, "tg": { "token_count": 512, "mean_tps": 68.729809, "stddev_tps": 0.58145, "repetitions": 5, "unit": "tokens/s" } }, "repetitions": 5 }, "server_performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-115943-c5d61a", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_prompt": false, "cache_type_k": "f16", "cache_type_v": "f16", "concurrencies": [ 1, 2, 4, 8 ], "ctx_size": 8192, "flash_attn": true, "gen_tokens": 512, "n_gpu_layers": 999, "parallel": 8, "repetitions": 3, "request_timeout_s": 300, "requests_per_concurrency": 4, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "formal": true, "concurrencies": [ 1, 2, 4, 8 ], "requests_per_concurrency": 4, "repetitions": 3, "gen_tokens": 512, "samples_count": 180, "conditions": [ { "concurrency": 1, "repetitions": 3, "requests_per_repetition": 4, "successful_requests": 12, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 87.12046830577675, "stddev": 0.478852060975797, "min": 86.61520791469717, "max": 87.56760621120318 }, "per_request_generation_throughput_tps": { "mean": 87.13586056041669, "stddev": 1.1195550112289752, "min": 84.61019497108262, "max": 88.53791265964232 }, "latency_p50_s": { "mean": 5.86078421966522, "stddev": 0.041894962116211215, "min": 5.828375694996794, "max": 5.908092396501161 }, "latency_p95_s": { "mean": 5.956498799447097, "stddev": 0.07080150382641702, "min": 5.880211759347003, "max": 6.020099510596265 }, "latency_p99_s": { "mean": 5.968976167086755, "stddev": 0.08064137199792148, "min": 5.88443108306732, "max": 6.0450440853159675 } }, { "concurrency": 2, "repetitions": 3, "requests_per_repetition": 8, "successful_requests": 24, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 119.36521413728069, "stddev": 0.6496901148225129, "min": 118.80050242856814, "max": 120.07526469563773 }, "per_request_generation_throughput_tps": { "mean": 59.88519985872432, "stddev": 1.911620478111302, "min": 55.59217794633424, "max": 63.702619244434935 }, "latency_p50_s": { "mean": 8.521887679334517, "stddev": 0.011777071471559472, "min": 8.508998674504255, "max": 8.532087788000354 }, "latency_p95_s": { "mean": 8.959279746102402, "stddev": 0.09013925435526828, "min": 8.889887934850776, "max": 9.061159485900863 }, "latency_p99_s": { "mean": 9.075256351089338, "stddev": 0.09089811446963317, "min": 9.0202403915162, "max": 9.1801749267806 } }, { "concurrency": 4, "repetitions": 3, "requests_per_repetition": 16, "successful_requests": 48, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 153.2592817162828, "stddev": 1.0989467629313387, "min": 152.09540205943733, "max": 154.27908236282383 }, "per_request_generation_throughput_tps": { "mean": 38.792939841149824, "stddev": 1.7089780769061296, "min": 35.53645160830998, "max": 43.917123856616634 }, "latency_p50_s": { "mean": 13.164271127333146, "stddev": 0.12386555732660506, "min": 13.022859912001877, "max": 13.253546264495526 }, "latency_p95_s": { "mean": 13.971693230500629, "stddev": 0.1389393814443958, "min": 13.814042763255202, "max": 14.076284708497042 }, "latency_p99_s": { "mean": 14.135910463701535, "stddev": 0.1740225234966403, "min": 13.997100531855903, "max": 14.331143450350647 } }, { "concurrency": 8, "repetitions": 3, "requests_per_repetition": 32, "successful_requests": 96, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 201.8061599086222, "stddev": 1.560331205924694, "min": 200.01812538273592, "max": 202.8920960941899 }, "per_request_generation_throughput_tps": { "mean": 25.849141588629767, "stddev": 2.06205287609798, "min": 21.95360197330285, "max": 30.530141033491567 }, "latency_p50_s": { "mean": 20.041166892831825, "stddev": 0.11259338871986072, "min": 19.968703073500365, "max": 20.1708815139973 }, "latency_p95_s": { "mean": 22.277365074933915, "stddev": 0.2848567979094447, "min": 22.012794965851572, "max": 22.57889961499968 }, "latency_p99_s": { "mean": 22.744598780645344, "stddev": 0.5613673028182077, "min": 22.09669375952668, "max": 23.08580125874978 } } ] }, "knowledge": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-015907-f823d8", "evaluator_version": "1.2", "condition": { "dataset_version": 1.2, "max_tokens": 128, "request_timeout_s": 300, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.2, "total": 100, "correct": 94, "accuracy": 0.94, "pass_rate": 0.94, "categories": { "general": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "korea": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "math": { "total": 20, "correct": 16, "accuracy": 0.8, "pass_rate": 0.8 }, "science": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "logic": { "total": 20, "correct": 18, "accuracy": 0.9, "pass_rate": 0.9 } }, "difficulties": { "easy": { "total": 25, "correct": 25, "accuracy": 1.0, "pass_rate": 1.0 }, "medium": { "total": 40, "correct": 39, "accuracy": 0.975, "pass_rate": 0.975 }, "hard": { "total": 35, "correct": 30, "accuracy": 0.8571428571428571, "pass_rate": 0.8571428571428571 } } }, "coding": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-120944-4cc44e", "evaluator_version": "existing-v1", "condition": { "dataset_version": 1, "request_timeout_s": 300.0, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": -1, "thinking_mode": "auto" }, "version": 1, "total": 12, "passed": 10, "pass_rate": 0.8333333333333334, "categories": { "basic": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "string": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "collection": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "edge_case": { "total": 3, "passed": 1, "pass_rate": 0.3333333333333333 } }, "failure_types": { "syntax_error": 2 } }, "tool_call": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-015958-979df7", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 15, "passed": 11, "pass_rate": 0.7333333333333333, "tool_selection_accuracy": 0.9333333333333333, "argument_accuracy": 0.8, "execution_success_rate": 0.8666666666666667, "tool_rate": 0.9333333333333333, "args_rate": 0.8, "exec_rate": 0.8666666666666667, "failure_types": { "wrong_arguments": 2, "wrong_tool": 1, "execution_error": 1 } }, "agent_single": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-020019-5e397a", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 12, "passed": 11, "correct": 11, "pass_rate": 0.9166666666666666, "avg_steps": 3.75, "avg_tool_calls": 2.75, "tool_calls_total": 33, "failed_tool_calls": 6, "repeated_tool_calls": 4, "failure_types": { "missing_required_step": 1 } }, "agent_multi": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-020055-80d568", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 8, "max_tokens": 256, "request_timeout_s": 300, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 10, "passed": 8, "correct": 8, "pass_rate": 0.8, "handoff_success_rate": 1.0, "role_participation_success_rate": 1.0, "avg_steps": 4.0, "avg_tool_calls": 3.0, "tool_calls_total": 30, "failed_tool_calls": 6, "invalid_tool_calls": 0, "handoffs": 11, "failure_types": { "missing_required_step": 1, "repeated_loop": 1 } } }, "provenance": { "hardware": "NVIDIA DGX Spark GB10", "reasoning_mode": "off", "raw_artifacts_public": false } }, { "model_id": "qwen3-6-35b-a3b-turbo", "model": "Qwen3.6 35B-A3B", "variant": "TURBO", "quantization": "TURBO", "benchmark_versions": { "performance": "existing-v1", "server_performance": "existing-v1", "knowledge": "1.2", "coding": "existing-v1", "tool_call": "1.1", "agent_single": "1.1", "agent_multi": "1.1" }, "suites": { "performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-172506-7319bf", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_type_k": "f16", "cache_type_v": "f16", "flash_attn": true, "gen_tokens": [ 512 ], "n_gpu_layers": 999, "prompt_tokens": [ 512, 2048, 8192, 32768 ], "repetitions": 5, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "metrics": { "pp": { "token_count": 32768, "mean_tps": 2241.159719, "stddev_tps": 5.84326, "repetitions": 5, "unit": "tokens/s" }, "tg": { "token_count": 512, "mean_tps": 77.200779, "stddev_tps": 0.158352, "repetitions": 5, "unit": "tokens/s" } }, "repetitions": 5 }, "server_performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-172758-ece708", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_prompt": false, "cache_type_k": "f16", "cache_type_v": "f16", "concurrencies": [ 1, 2, 4, 8 ], "ctx_size": 8192, "flash_attn": true, "gen_tokens": 512, "n_gpu_layers": 999, "parallel": 8, "repetitions": 3, "request_timeout_s": 300, "requests_per_concurrency": 4, "spec": "mtp", "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "formal": true, "concurrencies": [ 1, 2, 4, 8 ], "requests_per_concurrency": 4, "repetitions": 3, "gen_tokens": 512, "samples_count": 180, "conditions": [ { "concurrency": 1, "repetitions": 3, "requests_per_repetition": 4, "successful_requests": 12, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 87.02260140725951, "stddev": 0.5533180224375841, "min": 86.38426145639767, "max": 87.36527487827098 }, "per_request_generation_throughput_tps": { "mean": 87.04214076069144, "stddev": 1.2881923957272021, "min": 83.53603439084188, "max": 88.81211682733783 }, "latency_p50_s": { "mean": 5.876598739494511, "stddev": 0.02569072340322303, "min": 5.861747162991378, "max": 5.906263823497284 }, "latency_p95_s": { "mean": 5.9649730094703655, "stddev": 0.11632722709229934, "min": 5.888362644449808, "max": 6.098830077056482 }, "latency_p99_s": { "mean": 5.975554076832002, "stddev": 0.1281535376010528, "min": 5.891350807291456, "max": 6.123039231417643 } }, { "concurrency": 2, "repetitions": 3, "requests_per_repetition": 8, "successful_requests": 24, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 127.59116098099734, "stddev": 1.1855019143568115, "min": 126.28660082099626, "max": 128.60261633944438 }, "per_request_generation_throughput_tps": { "mean": 63.979269290686396, "stddev": 1.6877966375272822, "min": 61.188580754903654, "max": 65.86414535637843 }, "latency_p50_s": { "mean": 7.95912163216902, "stddev": 0.1517113909309586, "min": 7.826622403008514, "max": 8.12461494600575 }, "latency_p95_s": { "mean": 8.300998161835984, "stddev": 0.05114603374736976, "min": 8.2429473739081, "max": 8.339430802145216 }, "latency_p99_s": { "mean": 8.324049898236602, "stddev": 0.055190461041911706, "min": 8.260729497989377, "max": 8.361945655626187 } }, { "concurrency": 4, "repetitions": 3, "requests_per_repetition": 16, "successful_requests": 48, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 166.96544415267968, "stddev": 1.422354407240596, "min": 165.5816394243891, "max": 168.42344404016467 }, "per_request_generation_throughput_tps": { "mean": 42.57355976921814, "stddev": 1.7101586438817074, "min": 39.43410406592003, "max": 47.037676967857806 }, "latency_p50_s": { "mean": 11.961385855669505, "stddev": 0.08004781603213157, "min": 11.881807040495914, "max": 12.041894570509612 }, "latency_p95_s": { "mean": 12.738884412004458, "stddev": 0.17960641516210143, "min": 12.536335346256237, "max": 12.878744840498257 }, "latency_p99_s": { "mean": 12.896890772002129, "stddev": 0.018779441071774396, "min": 12.879592514502292, "max": 12.916864437059122 } }, { "concurrency": 8, "repetitions": 3, "requests_per_repetition": 32, "successful_requests": 96, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 217.335324572159, "stddev": 2.728223832425157, "min": 214.25898342438367, "max": 219.46111673915811 }, "per_request_generation_throughput_tps": { "mean": 27.651768839161992, "stddev": 1.6114645805202619, "min": 23.50293493514684, "max": 31.813700542417767 }, "latency_p50_s": { "mean": 18.704357515673717, "stddev": 0.33650963916248755, "min": 18.500875014506164, "max": 19.09277783950529 }, "latency_p95_s": { "mean": 20.077315226785625, "stddev": 0.5220745201902944, "min": 19.704256636556238, "max": 20.67394483990065 }, "latency_p99_s": { "mean": 20.618498689879733, "stddev": 0.8293100055747351, "min": 19.856341238030147, "max": 21.501665156432573 } } ] }, "knowledge": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-020127-747b4b", "evaluator_version": "1.2", "condition": { "dataset_version": 1.2, "max_tokens": 128, "request_timeout_s": 300, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.2, "total": 100, "correct": 94, "accuracy": 0.94, "pass_rate": 0.94, "categories": { "general": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "korea": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "math": { "total": 20, "correct": 16, "accuracy": 0.8, "pass_rate": 0.8 }, "science": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "logic": { "total": 20, "correct": 18, "accuracy": 0.9, "pass_rate": 0.9 } }, "difficulties": { "easy": { "total": 25, "correct": 25, "accuracy": 1.0, "pass_rate": 1.0 }, "medium": { "total": 40, "correct": 39, "accuracy": 0.975, "pass_rate": 0.975 }, "hard": { "total": 35, "correct": 30, "accuracy": 0.8571428571428571, "pass_rate": 0.8571428571428571 } } }, "coding": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-173719-61c62e", "evaluator_version": "existing-v1", "condition": { "dataset_version": 1, "request_timeout_s": 300.0, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": -1, "thinking_mode": "auto" }, "version": 1, "total": 12, "passed": 11, "pass_rate": 0.9166666666666666, "categories": { "basic": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "string": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "collection": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "edge_case": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 } }, "failure_types": { "syntax_error": 1 } }, "tool_call": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-020213-f0a325", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 15, "passed": 11, "pass_rate": 0.7333333333333333, "tool_selection_accuracy": 0.9333333333333333, "argument_accuracy": 0.8, "execution_success_rate": 0.8666666666666667, "tool_rate": 0.9333333333333333, "args_rate": 0.8, "exec_rate": 0.8666666666666667, "failure_types": { "wrong_arguments": 2, "wrong_tool": 1, "execution_error": 1 } }, "agent_single": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-020234-a0eaf2", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 12, "passed": 7, "correct": 7, "pass_rate": 0.5833333333333334, "avg_steps": 3.3333333333333335, "avg_tool_calls": 2.3333333333333335, "tool_calls_total": 28, "failed_tool_calls": 5, "repeated_tool_calls": 4, "failure_types": { "tool_error": 4, "missing_required_step": 1 } }, "agent_multi": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-020305-4be7f9", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 8, "max_tokens": 256, "request_timeout_s": 300, "spec_draft_n_max": 3, "spec_type": "mtp", "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 10, "passed": 8, "correct": 8, "pass_rate": 0.8, "handoff_success_rate": 1.0, "role_participation_success_rate": 1.0, "avg_steps": 4.0, "avg_tool_calls": 3.0, "tool_calls_total": 30, "failed_tool_calls": 6, "invalid_tool_calls": 0, "handoffs": 11, "failure_types": { "tool_error": 1, "repeated_loop": 1 } } }, "provenance": { "hardware": "NVIDIA DGX Spark GB10", "reasoning_mode": "off", "raw_artifacts_public": false } }, { "model_id": "qwen3-6-35b-a3b-q8", "model": "Qwen3.6 35B-A3B", "variant": "Q8", "quantization": "Q8", "benchmark_versions": { "performance": "existing-v1", "server_performance": "existing-v1", "knowledge": "1.2", "coding": "existing-v1", "tool_call": "1.1", "agent_single": "1.1", "agent_multi": "1.1" }, "suites": { "performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-174127-3d55c5", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_type_k": "f16", "cache_type_v": "f16", "flash_attn": true, "gen_tokens": [ 512 ], "n_gpu_layers": 999, "prompt_tokens": [ 512, 2048, 8192, 32768 ], "repetitions": 5, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "metrics": { "pp": { "token_count": 32768, "mean_tps": 1779.2207, "stddev_tps": 2.548781, "repetitions": 5, "unit": "tokens/s" }, "tg": { "token_count": 512, "mean_tps": 58.673607, "stddev_tps": 0.060877, "repetitions": 5, "unit": "tokens/s" } }, "repetitions": 5 }, "server_performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-174519-9fe463", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_prompt": false, "cache_type_k": "f16", "cache_type_v": "f16", "concurrencies": [ 1, 2, 4, 8 ], "ctx_size": 8192, "flash_attn": true, "gen_tokens": 512, "n_gpu_layers": 999, "parallel": 8, "repetitions": 3, "request_timeout_s": 300, "requests_per_concurrency": 4, "spec": "off", "spec_draft_n_max": 3, "spec_type": "none", "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "formal": true, "concurrencies": [ 1, 2, 4, 8 ], "requests_per_concurrency": 4, "repetitions": 3, "gen_tokens": 512, "samples_count": 180, "conditions": [ { "concurrency": 1, "repetitions": 3, "requests_per_repetition": 4, "successful_requests": 12, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 55.10062131943591, "stddev": 0.23406877460604136, "min": 54.87771641404748, "max": 55.34444916112258 }, "per_request_generation_throughput_tps": { "mean": 55.105242528007544, "stddev": 0.4838121780729247, "min": 54.058703770340266, "max": 55.857881051548155 }, "latency_p50_s": { "mean": 9.27228719816776, "stddev": 0.024356726790996958, "min": 9.25381297799322, "max": 9.29988942550699 }, "latency_p95_s": { "mean": 9.384710934052904, "stddev": 0.06142670219597031, "min": 9.325382616756542, "max": 9.448040398547892 }, "latency_p99_s": { "mean": 9.397401982813802, "stddev": 0.06877730212391608, "min": 9.329008012959093, "max": 9.466556312507018 } }, { "concurrency": 2, "repetitions": 3, "requests_per_repetition": 8, "successful_requests": 24, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 80.85476741124435, "stddev": 0.4107329986505433, "min": 80.38485920587098, "max": 81.14532101997693 }, "per_request_generation_throughput_tps": { "mean": 40.42924607607273, "stddev": 0.234247623040382, "min": 40.156450333636236, "max": 40.75788218721114 }, "latency_p50_s": { "mean": 12.656270372167151, "stddev": 0.07533706715636637, "min": 12.585272979500587, "max": 12.735302979002881 }, "latency_p95_s": { "mean": 12.724243387899817, "stddev": 0.03941934272653222, "min": 12.67887485004685, "max": 12.75011456925131 }, "latency_p99_s": { "mean": 12.724279266912053, "stddev": 0.03943046141358729, "min": 12.678894992406859, "max": 12.750127669048378 } }, { "concurrency": 4, "repetitions": 3, "requests_per_repetition": 16, "successful_requests": 48, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 119.69063340868647, "stddev": 3.6515711146081653, "min": 115.47698325930882, "max": 121.93100966842461 }, "per_request_generation_throughput_tps": { "mean": 30.130852981551218, "stddev": 2.6318308503311822, "min": 27.664365808523886, "max": 34.6221805117136 }, "latency_p50_s": { "mean": 17.35139990350096, "stddev": 0.9455522377529871, "min": 16.65107611699932, "max": 18.426978215000418 }, "latency_p95_s": { "mean": 18.43898111341453, "stddev": 0.05905473003134411, "min": 18.399308127500262, "max": 18.506848864741187 }, "latency_p99_s": { "mean": 18.4451837592181, "stddev": 0.055559723682130815, "min": 18.40058080310191, "max": 18.507420373746573 } }, { "concurrency": 8, "repetitions": 3, "requests_per_repetition": 32, "successful_requests": 96, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 182.63442849658853, "stddev": 4.335843131718906, "min": 178.68519327471017, "max": 187.27403982882998 }, "per_request_generation_throughput_tps": { "mean": 22.86697770933213, "stddev": 1.0136584980787244, "min": 21.33780847012079, "max": 24.41668354054183 }, "latency_p50_s": { "mean": 22.231728886499088, "stddev": 0.8746052848224335, "min": 21.24513878650032, "max": 22.911877850499877 }, "latency_p95_s": { "mean": 23.556560374716355, "stddev": 0.5192386049356897, "min": 22.98282433539716, "max": 23.994189618652307 }, "latency_p99_s": { "mean": 23.573708676946698, "stddev": 0.511164792854062, "min": 23.004948673780454, "max": 23.994739655666052 } } ] }, "knowledge": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-020336-b838f8", "evaluator_version": "1.2", "condition": { "dataset_version": 1.2, "max_tokens": 128, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.2, "total": 100, "correct": 95, "accuracy": 0.95, "pass_rate": 0.95, "categories": { "general": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "korea": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "math": { "total": 20, "correct": 17, "accuracy": 0.85, "pass_rate": 0.85 }, "science": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "logic": { "total": 20, "correct": 18, "accuracy": 0.9, "pass_rate": 0.9 } }, "difficulties": { "easy": { "total": 25, "correct": 25, "accuracy": 1.0, "pass_rate": 1.0 }, "medium": { "total": 40, "correct": 39, "accuracy": 0.975, "pass_rate": 0.975 }, "hard": { "total": 35, "correct": 31, "accuracy": 0.8857142857142857, "pass_rate": 0.8857142857142857 } } }, "coding": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-175911-e3d9fa", "evaluator_version": "existing-v1", "condition": { "dataset_version": 1, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto" }, "version": 1, "total": 12, "passed": 11, "pass_rate": 0.9166666666666666, "categories": { "basic": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "string": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "collection": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "edge_case": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 } }, "failure_types": { "syntax_error": 1 } }, "tool_call": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-020448-943315", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 15, "passed": 10, "pass_rate": 0.6666666666666666, "tool_selection_accuracy": 0.8666666666666667, "argument_accuracy": 0.7333333333333333, "execution_success_rate": 0.8, "tool_rate": 0.8666666666666667, "args_rate": 0.7333333333333333, "exec_rate": 0.8, "failure_types": { "wrong_arguments": 2, "missing_tool_call": 1, "wrong_tool": 1, "execution_error": 1 } }, "agent_single": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-020519-378732", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 12, "passed": 9, "correct": 9, "pass_rate": 0.75, "avg_steps": 3.6666666666666665, "avg_tool_calls": 2.6666666666666665, "tool_calls_total": 32, "failed_tool_calls": 7, "repeated_tool_calls": 4, "failure_types": { "tool_error": 2, "missing_required_step": 1 } }, "agent_multi": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-020615-282e39", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 8, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 10, "passed": 9, "correct": 9, "pass_rate": 0.9, "handoff_success_rate": 1.0, "role_participation_success_rate": 1.0, "avg_steps": 4.0, "avg_tool_calls": 3.0, "tool_calls_total": 30, "failed_tool_calls": 7, "invalid_tool_calls": 0, "handoffs": 11, "failure_types": { "missing_required_step": 1 } } }, "provenance": { "hardware": "NVIDIA DGX Spark GB10", "reasoning_mode": "off", "raw_artifacts_public": false } }, { "model_id": "qwen3-6-35b-a3b-apex", "model": "Qwen3.6 35B-A3B", "variant": "APEX", "quantization": "APEX", "benchmark_versions": { "performance": "existing-v1", "server_performance": "existing-v1", "knowledge": "1.2", "coding": "existing-v1", "tool_call": "1.1", "agent_single": "1.1", "agent_multi": "1.1" }, "suites": { "performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-180921-b11e6a", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_type_k": "f16", "cache_type_v": "f16", "flash_attn": true, "gen_tokens": [ 512 ], "n_gpu_layers": 999, "prompt_tokens": [ 512, 2048, 8192, 32768 ], "repetitions": 5, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "metrics": { "pp": { "token_count": 32768, "mean_tps": 1829.503614, "stddev_tps": 4.467092, "repetitions": 5, "unit": "tokens/s" }, "tg": { "token_count": 512, "mean_tps": 68.502242, "stddev_tps": 0.065986, "repetitions": 5, "unit": "tokens/s" } }, "repetitions": 5 }, "server_performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-181248-c9a2b9", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_prompt": false, "cache_type_k": "f16", "cache_type_v": "f16", "concurrencies": [ 1, 2, 4, 8 ], "ctx_size": 8192, "flash_attn": true, "gen_tokens": 512, "n_gpu_layers": 999, "parallel": 8, "repetitions": 3, "request_timeout_s": 300, "requests_per_concurrency": 4, "spec": "off", "spec_draft_n_max": 3, "spec_type": "none", "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "formal": true, "concurrencies": [ 1, 2, 4, 8 ], "requests_per_concurrency": 4, "repetitions": 3, "gen_tokens": 512, "samples_count": 180, "conditions": [ { "concurrency": 1, "repetitions": 3, "requests_per_repetition": 4, "successful_requests": 12, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 64.35582456223206, "stddev": 0.41886484217392533, "min": 63.94069102277045, "max": 64.77832356388713 }, "per_request_generation_throughput_tps": { "mean": 64.35932267142239, "stddev": 0.4896554553783383, "min": 63.7286026166304, "max": 65.43410511534721 }, "latency_p50_s": { "mean": 7.963173106329729, "stddev": 0.04654514154139293, "min": 7.915557315995102, "max": 8.008568094497605 }, "latency_p95_s": { "mean": 7.9931425497821085, "stddev": 0.03794795367567283, "min": 7.9571173298507345, "max": 8.032758049249969 }, "latency_p99_s": { "mean": 7.994847971022294, "stddev": 0.03777393159146471, "min": 7.9583826243711515, "max": 8.033806926650431 } }, { "concurrency": 2, "repetitions": 3, "requests_per_repetition": 8, "successful_requests": 24, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 98.53427686364759, "stddev": 0.8676062372064539, "min": 97.55062502928742, "max": 99.19060926826644 }, "per_request_generation_throughput_tps": { "mean": 49.27419099969773, "stddev": 0.620550288099998, "min": 47.43175391584129, "max": 49.79607457075806 }, "latency_p50_s": { "mean": 10.357187275495866, "stddev": 0.05760715258773722, "min": 10.307622847991297, "max": 10.420389661994705 }, "latency_p95_s": { "mean": 10.537006710950664, "stddev": 0.21859793080273537, "min": 10.395175136298349, "max": 10.788748128857696 }, "latency_p99_s": { "mean": 10.538654547259164, "stddev": 0.22111581343900588, "min": 10.395407850462652, "max": 10.79331475217943 } }, { "concurrency": 4, "repetitions": 3, "requests_per_repetition": 16, "successful_requests": 48, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 152.53027064424762, "stddev": 7.007744778632513, "min": 148.253142398441, "max": 160.61762601908973 }, "per_request_generation_throughput_tps": { "mean": 38.222631309704, "stddev": 2.262468978769683, "min": 34.091733879174484, "max": 40.482634384598974 }, "latency_p50_s": { "mean": 13.408217748668298, "stddev": 0.5713157496992832, "min": 12.750189885999134, "max": 13.777866453507158 }, "latency_p95_s": { "mean": 14.264144076666222, "stddev": 1.2489144152686311, "min": 12.822517706496, "max": 15.017702846005704 }, "latency_p99_s": { "mean": 14.26446911506961, "stddev": 1.2489989200765936, "min": 12.8227484149058, "max": 15.018183222803781 } }, { "concurrency": 8, "repetitions": 3, "requests_per_repetition": 32, "successful_requests": 96, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 195.28368127562678, "stddev": 2.3538982748741217, "min": 193.2711917721133, "max": 197.8720927654808 }, "per_request_generation_throughput_tps": { "mean": 24.416544805738525, "stddev": 0.36328409709094994, "min": 24.101657998772975, "max": 25.0075053885422 }, "latency_p50_s": { "mean": 21.000229261832526, "stddev": 0.34099386138369114, "min": 20.606524845497916, "max": 21.20202494249679 }, "latency_p95_s": { "mean": 21.189422866312572, "stddev": 0.07331304198600229, "min": 21.105716710044362, "max": 21.242219512299197 }, "latency_p99_s": { "mean": 21.19082993593227, "stddev": 0.07374769405954211, "min": 21.106473038860422, "max": 21.24309093456264 } } ] }, "knowledge": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-020707-041e50", "evaluator_version": "1.2", "condition": { "dataset_version": 1.2, "max_tokens": 128, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.2, "total": 100, "correct": 95, "accuracy": 0.95, "pass_rate": 0.95, "categories": { "general": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "korea": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "math": { "total": 20, "correct": 17, "accuracy": 0.85, "pass_rate": 0.85 }, "science": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "logic": { "total": 20, "correct": 18, "accuracy": 0.9, "pass_rate": 0.9 } }, "difficulties": { "easy": { "total": 25, "correct": 25, "accuracy": 1.0, "pass_rate": 1.0 }, "medium": { "total": 40, "correct": 39, "accuracy": 0.975, "pass_rate": 0.975 }, "hard": { "total": 35, "correct": 31, "accuracy": 0.8857142857142857, "pass_rate": 0.8857142857142857 } } }, "coding": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-182343-4283d9", "evaluator_version": "existing-v1", "condition": { "dataset_version": 1, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto" }, "version": 1, "total": 12, "passed": 10, "pass_rate": 0.8333333333333334, "categories": { "basic": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "string": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 }, "collection": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "edge_case": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 } }, "failure_types": { "syntax_error": 2 } }, "tool_call": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-020758-65179b", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 15, "passed": 11, "pass_rate": 0.7333333333333333, "tool_selection_accuracy": 0.9333333333333333, "argument_accuracy": 0.8, "execution_success_rate": 0.9333333333333333, "tool_rate": 0.9333333333333333, "args_rate": 0.8, "exec_rate": 0.9333333333333333, "failure_types": { "wrong_arguments": 2, "wrong_tool": 1, "final_answer_error": 1 } }, "agent_single": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-020824-1d79ba", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 12, "passed": 9, "correct": 9, "pass_rate": 0.75, "avg_steps": 3.6666666666666665, "avg_tool_calls": 2.6666666666666665, "tool_calls_total": 32, "failed_tool_calls": 7, "repeated_tool_calls": 5, "failure_types": { "missing_required_step": 2, "tool_error": 1 } }, "agent_multi": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-020910-33d8b1", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 8, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 10, "passed": 9, "correct": 9, "pass_rate": 0.9, "handoff_success_rate": 1.0, "role_participation_success_rate": 1.0, "avg_steps": 4.0, "avg_tool_calls": 3.0, "tool_calls_total": 30, "failed_tool_calls": 7, "invalid_tool_calls": 0, "handoffs": 11, "failure_types": { "tool_error": 1 } } }, "provenance": { "hardware": "NVIDIA DGX Spark GB10", "reasoning_mode": "off", "raw_artifacts_public": false } }, { "model_id": "qwen3-8-flash-next-ud-iq4-xs", "model": "Qwen3.8 Flash Next", "variant": "UD-IQ4_XS", "quantization": "UD-IQ4_XS", "benchmark_versions": { "performance": "existing-v1", "server_performance": "existing-v1", "knowledge": "1.2", "coding": "existing-v1", "tool_call": "1.1", "agent_single": "1.1", "agent_multi": "1.1" }, "suites": { "performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-223427-189e4d", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_type_k": "f16", "cache_type_v": "f16", "flash_attn": true, "gen_tokens": [ 512 ], "n_gpu_layers": 999, "prompt_tokens": [ 512, 2048, 8192, 32768 ], "repetitions": 5, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "metrics": { "pp": { "token_count": 32768, "mean_tps": 280.107968, "stddev_tps": 21.458376, "repetitions": 5, "unit": "tokens/s" }, "tg": { "token_count": 512, "mean_tps": 25.819479, "stddev_tps": 0.199371, "repetitions": 5, "unit": "tokens/s" } }, "repetitions": 5 }, "server_performance": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-225405-d84cc2", "evaluator_version": "existing-v1", "condition": { "batch_size": 2048, "cache_prompt": false, "cache_type_k": "f16", "cache_type_v": "f16", "concurrencies": [ 1, 2, 4, 8 ], "ctx_size": 8192, "flash_attn": true, "gen_tokens": 512, "n_gpu_layers": 999, "parallel": 8, "repetitions": 3, "request_timeout_s": 300, "requests_per_concurrency": 4, "spec": "off", "spec_draft_n_max": 3, "spec_type": "none", "thinking_budget": -1, "thinking_mode": "auto", "threads": 20, "ubatch_size": 512 }, "formal": true, "concurrencies": [ 1, 2, 4, 8 ], "requests_per_concurrency": 4, "repetitions": 3, "gen_tokens": 512, "samples_count": 180, "conditions": [ { "concurrency": 1, "repetitions": 3, "requests_per_repetition": 4, "successful_requests": 12, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 26.8182486493483, "stddev": 0.49890434867144756, "min": 26.24347571931715, "max": 27.13928936076516 }, "per_request_generation_throughput_tps": { "mean": 26.84387567677128, "stddev": 0.9129121803282568, "min": 23.954790897176373, "max": 27.1896633980315 }, "latency_p50_s": { "mean": 18.898438121167903, "stddev": 0.02745784355065091, "min": 18.86675476500386, "max": 18.91530784549832 }, "latency_p95_s": { "mean": 19.622039797965165, "stddev": 1.2015777403999108, "min": 18.87144940149228, "max": 21.007903949201136 }, "latency_p99_s": { "mean": 19.720071710525275, "stddev": 1.3698749910292674, "min": 18.87204613149064, "max": 21.300456825042055 } }, { "concurrency": 2, "repetitions": 3, "requests_per_repetition": 8, "successful_requests": 24, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 42.38992189361571, "stddev": 1.3967787618869894, "min": 40.79429311711878, "max": 43.391372023741006 }, "per_request_generation_throughput_tps": { "mean": 21.204917357267178, "stddev": 0.7109921015357036, "min": 19.847177811601114, "max": 21.76836622961006 }, "latency_p50_s": { "mean": 24.265734590167995, "stddev": 1.0203620629879009, "min": 23.595616344493465, "max": 25.44004629250412 }, "latency_p95_s": { "mean": 24.539385896782672, "stddev": 1.1006371989698138, "min": 23.683538448250328, "max": 25.78097177599193 }, "latency_p99_s": { "mean": 24.5438807012213, "stddev": 1.107762287970651, "min": 23.68383116724639, "max": 25.793889239988495 } }, { "concurrency": 4, "repetitions": 3, "requests_per_repetition": 16, "successful_requests": 48, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 63.47786429948999, "stddev": 2.3208133556722577, "min": 61.00287554247823, "max": 65.60530116005062 }, "per_request_generation_throughput_tps": { "mean": 15.873988411592128, "stddev": 0.5196365143692745, "min": 15.096146625158463, "max": 16.473211273966182 }, "latency_p50_s": { "mean": 32.340458113998466, "stddev": 1.212666513630434, "min": 31.191256833502848, "max": 33.60792409900023 }, "latency_p95_s": { "mean": 32.671440882668925, "stddev": 1.3257497801397273, "min": 31.276870356246945, "max": 33.915527997254685 }, "latency_p99_s": { "mean": 32.67172780827047, "stddev": 1.3258898729941864, "min": 31.27694696485123, "max": 33.91585743145697 } }, { "concurrency": 8, "repetitions": 3, "requests_per_repetition": 32, "successful_requests": 96, "failed_requests": 0, "failure_rate": 0.0, "aggregate_generation_throughput_tps": { "mean": 93.70121171378653, "stddev": 5.579850062845268, "min": 87.50186497015363, "max": 98.32101721406762 }, "per_request_generation_throughput_tps": { "mean": 11.736321561059745, "stddev": 0.7598584422627227, "min": 10.464603861746179, "max": 12.342842677872211 }, "latency_p50_s": { "mean": 43.718567230501016, "stddev": 3.5861146253863168, "min": 41.64333890900161, "max": 47.85945203150186 }, "latency_p95_s": { "mean": 45.936960805113145, "stddev": 3.756314453892477, "min": 41.72047352349764, "max": 48.926013465801226 }, "latency_p99_s": { "mean": 45.93800395517809, "stddev": 3.7560712223942394, "min": 41.721701823335316, "max": 48.926591434170696 } } ] }, "knowledge": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-020952-a4ffcc", "evaluator_version": "1.2", "condition": { "dataset_version": 1.2, "max_tokens": 128, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.2, "total": 100, "correct": 98, "accuracy": 0.98, "pass_rate": 0.98, "categories": { "general": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "korea": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "math": { "total": 20, "correct": 19, "accuracy": 0.95, "pass_rate": 0.95 }, "science": { "total": 20, "correct": 20, "accuracy": 1.0, "pass_rate": 1.0 }, "logic": { "total": 20, "correct": 19, "accuracy": 0.95, "pass_rate": 0.95 } }, "difficulties": { "easy": { "total": 25, "correct": 25, "accuracy": 1.0, "pass_rate": 1.0 }, "medium": { "total": 40, "correct": 39, "accuracy": 0.975, "pass_rate": 0.975 }, "hard": { "total": 35, "correct": 34, "accuracy": 0.9714285714285714, "pass_rate": 0.9714285714285714 } } }, "coding": { "status": "available", "source_type": "reused_historical", "source_run_id": "20260905-232058-c7aa39", "evaluator_version": "existing-v1", "condition": { "dataset_version": 1, "request_timeout_s": 300.0, "thinking_budget": -1, "thinking_mode": "auto" }, "version": 1, "total": 12, "passed": 10, "pass_rate": 0.8333333333333334, "categories": { "basic": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "string": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 }, "collection": { "total": 3, "passed": 3, "pass_rate": 1.0 }, "edge_case": { "total": 3, "passed": 2, "pass_rate": 0.6666666666666666 } }, "failure_types": { "syntax_error": 2 } }, "tool_call": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-021203-505271", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 15, "passed": 10, "pass_rate": 0.6666666666666666, "tool_selection_accuracy": 0.8666666666666667, "argument_accuracy": 0.8, "execution_success_rate": 0.9333333333333333, "tool_rate": 0.8666666666666667, "args_rate": 0.8, "exec_rate": 0.9333333333333333, "failure_types": { "wrong_arguments": 2, "wrong_tool": 2, "final_answer_error": 1 } }, "agent_single": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-021420-bc6cfc", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 6, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 12, "passed": 4, "correct": 4, "pass_rate": 0.3333333333333333, "avg_steps": 3.4166666666666665, "avg_tool_calls": 2.4166666666666665, "tool_calls_total": 29, "failed_tool_calls": 3, "repeated_tool_calls": 4, "failure_types": { "tool_error": 1, "final_answer_error": 3, "repeated_loop": 1, "missing_required_step": 3 } }, "agent_multi": { "status": "available", "source_type": "revalidated_evaluator", "source_run_id": "20260906-021716-919ebc", "evaluator_version": "1.1", "condition": { "dataset_version": 1, "max_steps": 8, "max_tokens": 256, "request_timeout_s": 300, "thinking_budget": 0, "thinking_mode": "no-think" }, "version": 1.1, "total": 10, "passed": 4, "correct": 4, "pass_rate": 0.4, "handoff_success_rate": 1.0, "role_participation_success_rate": 1.0, "avg_steps": 3.8, "avg_tool_calls": 2.8, "tool_calls_total": 28, "failed_tool_calls": 6, "invalid_tool_calls": 0, "handoffs": 12, "failure_types": { "tool_error": 4, "repeated_loop": 1, "missing_required_step": 1 } } }, "provenance": { "hardware": "NVIDIA DGX Spark GB10", "reasoning_mode": "off", "raw_artifacts_public": false } } ], "counts": { "models": 18, "suites": 7, "revalidated_evaluator_runs": 72, "reused_source_runs": 54, "source_run_references": 126 } }