{ "date": "2026-10-01", "harness": { "name": "JevBench open harness (MIT) by fstandhartinger", "commit": "bb05a335bc809e61b20c0f745d25499a82b326fc", "commit_date": "2026-09-29", "adapter": "typesafe (/v1/systemone)", "command": "python -m jevbench.cli run --tasks datasets/public/.jsonl --adapter typesafe --endpoint http://127.0.0.1: --key-env \"\" --model --results ... --raw-dir ... --ledger ..." }, "public_items": { "easy": 48, "original": 72, "hard": 111, "total": 231, "note": "231 = easy 48 + original 72 + hard 111, verified by line counts of datasets/public/*.jsonl; the standard and judge tiers have no public file here" }, "hardware": { "cpu": "Apple M5 Pro", "cores": 18, "ram_bytes": 51539607552, "os": "macOS (Darwin 25.4.0)", "device": "cpu (ONNX Runtime CPU provider)", "machine_busy": "YES: other agent sessions were running; load average 4-10 during runs", "requests": "strictly sequential, one item at a time, 3 full runs per system" }, "server": { "binary": "openjevx server v0.5.0 (installed release binary, sha256 prefix 5781928a2188) serving the model folder; scratch config and ports 21140/21141" }, "systems": { "openjevx": { "name": "OpenJevX v0.4.0 (8-bit ONNX, CPU)", "runs": 3, "results_identical_across_runs": true, "chance_accuracy": { "easy": 0.284, "original": 0.311, "hard": 0.336 }, "model_file": "openjevx-0.4.0/openjevx.w8.onnx", "model_sha256_prefix": "15dc80e05a22", "model_size_bytes": 598046561, "max_len": 512, "temperatures": { "choice": 1.111, "score": 1.132, "noul": 1.094 }, "truncated_items_at_cap": { "hard": 57, "easy": 0, "original": 0 }, "peak_rss_mb": 1679, "rss_after_runs_mb": 1380, "peak_rss_note": "macOS phys_footprint_peak via footprint(1), pid 50049", "per_run": { "x_1": { "overall": { "n": 231, "accuracy": 0.5844, "ece": 0.1209, "brier": 0.515, "p50_s": 0.2506, "p95_s": 0.7115 }, "tiers": { "easy": { "p50_s": 0.2191, "p95_s": 0.2427 }, "original": { "p50_s": 0.2213, "p95_s": 0.2506 }, "hard": { "p50_s": 0.695, "p95_s": 0.7212 } } }, "x_2": { "overall": { "n": 231, "accuracy": 0.5844, "ece": 0.1209, "brier": 0.515, "p50_s": 0.343, "p95_s": 0.7915 }, "tiers": { "easy": { "p50_s": 0.2372, "p95_s": 0.2708 }, "original": { "p50_s": 0.2407, "p95_s": 0.3614 }, "hard": { "p50_s": 0.7311, "p95_s": 0.8157 } } }, "x_3": { "overall": { "n": 231, "accuracy": 0.5844, "ece": 0.1209, "brier": 0.515, "p50_s": 0.2792, "p95_s": 0.7975 }, "tiers": { "easy": { "p50_s": 0.2492, "p95_s": 0.2736 }, "original": { "p50_s": 0.246, "p95_s": 0.275 }, "hard": { "p50_s": 0.7396, "p95_s": 0.8224 } } } }, "tiers": { "easy": { "n": 48, "n_correct": 48, "accuracy": 1.0, "accuracy_95ci_wilson": [ 0.926, 1.0 ], "chance_corrected_accuracy": 1.0, "harness_ece": 0.1282, "own_ece": 0.1282, "brier": 0.0391, "harness_brier": 0.0391, "mean_confidence": 0.8718, "confidently_wrong_ge0.8": 0, "n_conf_ge0.8": 41, "accuracy_by_confidence_band": { "conf>=0.9": { "n": 22, "acc": 1.0, "mean_conf": 0.9283 }, "0.8-0.9": { "n": 19, "acc": 1.0, "mean_conf": 0.8838 }, "0.6-0.8": { "n": 4, "acc": 1.0, "mean_conf": 0.7359 }, "<0.6": { "n": 3, "acc": 1.0, "mean_conf": 0.562 } }, "reliability_bins": [ { "bin": "0.0-0.1", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.1-0.2", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.2-0.3", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.3-0.4", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.4-0.5", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.5-0.6", "n": 3, "mean_conf": 0.562, "acc": 1.0 }, { "bin": "0.6-0.7", "n": 1, "mean_conf": 0.6226, "acc": 1.0 }, { "bin": "0.7-0.8", "n": 3, "mean_conf": 0.7736, "acc": 1.0 }, { "bin": "0.8-0.9", "n": 19, "mean_conf": 0.8838, "acc": 1.0 }, { "bin": "0.9-1.0", "n": 22, "mean_conf": 0.9283, "acc": 1.0 } ], "ordinal_mae_score_items": null, "per_family": { "extraction": { "n": 12, "accuracy": 1.0 }, "fact": { "n": 12, "accuracy": 1.0 }, "intent": { "n": 12, "accuracy": 1.0 }, "tool_selection": { "n": 12, "accuracy": 1.0 } }, "failed_requests": 0, "invalid_answers": 0 }, "original": { "n": 72, "n_correct": 51, "accuracy": 0.7083, "accuracy_95ci_wilson": [ 0.595, 0.801 ], "chance_corrected_accuracy": 0.5766, "harness_ece": 0.0794, "own_ece": 0.0794, "brier": 0.4003, "harness_brier": 0.4003, "mean_confidence": 0.7207, "confidently_wrong_ge0.8": 3, "n_conf_ge0.8": 26, "accuracy_by_confidence_band": { "conf>=0.9": { "n": 0, "acc": null, "mean_conf": null }, "0.8-0.9": { "n": 26, "acc": 0.8846, "mean_conf": 0.8539 }, "0.6-0.8": { "n": 31, "acc": 0.6129, "mean_conf": 0.7184 }, "<0.6": { "n": 15, "acc": 0.6, "mean_conf": 0.4947 } }, "reliability_bins": [ { "bin": "0.0-0.1", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.1-0.2", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.2-0.3", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.3-0.4", "n": 3, "mean_conf": 0.3451, "acc": 0.3333 }, { "bin": "0.4-0.5", "n": 2, "mean_conf": 0.4571, "acc": 0.5 }, { "bin": "0.5-0.6", "n": 10, "mean_conf": 0.5472, "acc": 0.7 }, { "bin": "0.6-0.7", "n": 10, "mean_conf": 0.6469, "acc": 0.5 }, { "bin": "0.7-0.8", "n": 21, "mean_conf": 0.7525, "acc": 0.6667 }, { "bin": "0.8-0.9", "n": 26, "mean_conf": 0.8539, "acc": 0.8846 }, { "bin": "0.9-1.0", "n": 0, "mean_conf": null, "acc": null } ], "ordinal_mae_score_items": 0.4107416666666666, "per_family": { "adequacy": { "n": 12, "accuracy": 0.583 }, "extraction": { "n": 12, "accuracy": 0.917 }, "intent": { "n": 12, "accuracy": 0.75 }, "ordinal": { "n": 12, "accuracy": 0.667 }, "policy": { "n": 12, "accuracy": 0.75 }, "routing": { "n": 12, "accuracy": 0.583 } }, "failed_requests": 0, "invalid_answers": 0 }, "hard": { "n": 111, "n_correct": 36, "accuracy": 0.3243, "accuracy_95ci_wilson": [ 0.244, 0.416 ], "chance_corrected_accuracy": -0.0179, "harness_ece": 0.259, "own_ece": 0.259, "brier": 0.7953, "harness_brier": 0.7953, "mean_confidence": 0.5833, "confidently_wrong_ge0.8": 9, "n_conf_ge0.8": 12, "accuracy_by_confidence_band": { "conf>=0.9": { "n": 0, "acc": null, "mean_conf": null }, "0.8-0.9": { "n": 12, "acc": 0.25, "mean_conf": 0.8474 }, "0.6-0.8": { "n": 45, "acc": 0.3778, "mean_conf": 0.7031 }, "<0.6": { "n": 54, "acc": 0.2963, "mean_conf": 0.4248 } }, "reliability_bins": [ { "bin": "0.0-0.1", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.1-0.2", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.2-0.3", "n": 5, "mean_conf": 0.262, "acc": 0.2 }, { "bin": "0.3-0.4", "n": 18, "mean_conf": 0.3528, "acc": 0.1667 }, { "bin": "0.4-0.5", "n": 16, "mean_conf": 0.4426, "acc": 0.375 }, { "bin": "0.5-0.6", "n": 15, "mean_conf": 0.5467, "acc": 0.4 }, { "bin": "0.6-0.7", "n": 24, "mean_conf": 0.6573, "acc": 0.375 }, { "bin": "0.7-0.8", "n": 21, "mean_conf": 0.7555, "acc": 0.381 }, { "bin": "0.8-0.9", "n": 12, "mean_conf": 0.8474, "acc": 0.25 }, { "bin": "0.9-1.0", "n": 0, "mean_conf": null, "acc": null } ], "ordinal_mae_score_items": 0.7047666666666667, "per_family": { "adversarial": { "n": 6, "accuracy": 0.5 }, "ambiguous": { "n": 7, "accuracy": 0.0 }, "judge_hard": { "n": 17, "accuracy": 0.471 }, "long_policy": { "n": 19, "accuracy": 0.368 }, "multi_hop": { "n": 18, "accuracy": 0.333 }, "probability": { "n": 10, "accuracy": 0.3 }, "routing_hard": { "n": 5, "accuracy": 0.4 }, "temporal_numeric": { "n": 15, "accuracy": 0.267 }, "tradeoff": { "n": 6, "accuracy": 0.333 }, "trap": { "n": 8, "accuracy": 0.125 } }, "failed_requests": 0, "invalid_answers": 0 } }, "overall": { "n": 231, "n_correct": 135, "accuracy": 0.5844, "accuracy_95ci_wilson": [ 0.52, 0.646 ], "macro_family_accuracy": 0.5286, "own_ece": 0.1209, "harness_ece": 0.1209, "brier": 0.515, "mean_confidence": 0.6861, "confidently_wrong_ge0.8": 12, "n_conf_ge0.8": 79, "accuracy_by_confidence_band": { "conf>=0.9": { "n": 22, "acc": 1.0, "mean_conf": 0.9283 }, "0.8-0.9": { "n": 57, "acc": 0.7895, "mean_conf": 0.8625 }, "0.6-0.8": { "n": 80, "acc": 0.5, "mean_conf": 0.7107 }, "<0.6": { "n": 72, "acc": 0.3889, "mean_conf": 0.4451 } }, "reliability_bins": [ { "bin": "0.0-0.1", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.1-0.2", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.2-0.3", "n": 5, "mean_conf": 0.262, "acc": 0.2 }, { "bin": "0.3-0.4", "n": 21, "mean_conf": 0.3517, "acc": 0.1905 }, { "bin": "0.4-0.5", "n": 18, "mean_conf": 0.4442, "acc": 0.3889 }, { "bin": "0.5-0.6", "n": 28, "mean_conf": 0.5485, "acc": 0.5714 }, { "bin": "0.6-0.7", "n": 35, "mean_conf": 0.6534, "acc": 0.4286 }, { "bin": "0.7-0.8", "n": 45, "mean_conf": 0.7553, "acc": 0.5556 }, { "bin": "0.8-0.9", "n": 57, "mean_conf": 0.8625, "acc": 0.7895 }, { "bin": "0.9-1.0", "n": 22, "mean_conf": 0.9283, "acc": 1.0 } ], "latency_s": { "p50": 0.2506, "p95": 0.7115, "mean": 0.3974, "max": 0.7279 } }, "abstain": "not applicable: the harness has no abstain action; every item gets an answer. Confidence below 0.6 is the only abstain-like signal." }, "laya_base": { "name": "Laya base 421M ModernBERT (8-bit ONNX, CPU, as quantized by openjevx repo quantize_w8.py)", "runs": 3, "results_identical_across_runs": true, "chance_accuracy": { "easy": 0.284, "original": 0.311, "hard": 0.336 }, "model_file": "site_work/openjevx-base.w8.onnx", "model_sha256_prefix": "1b2bd9969abe", "model_size_bytes": 598089196, "max_len": 1024, "temperatures": "bare-onnx back-compat defaults: choice 1.026 score 1.034 noul 1.148", "truncated_items_at_cap": { "hard": 37, "easy": 0, "original": 0 }, "peak_rss_mb": 1758, "rss_after_runs_mb": 1513, "peak_rss_note": "macOS phys_footprint_peak via footprint(1), pid 50050", "provenance": "local base-model ONNX built by the openjevx repo (convaiinnovations/laya exported then 8-bit weight-only quantized); not re-verified by us against upstream weights", "per_run": { "b_1": { "overall": { "n": 231, "accuracy": 0.5411, "ece": 0.2073, "brier": 0.6079, "p50_s": 0.2598, "p95_s": 1.6142 }, "tiers": { "easy": { "p50_s": 0.2216, "p95_s": 0.2564 }, "original": { "p50_s": 0.2286, "p95_s": 0.2589 }, "hard": { "p50_s": 0.8116, "p95_s": 1.6393 } } }, "b_2": { "overall": { "n": 231, "accuracy": 0.5411, "ece": 0.2073, "brier": 0.6079, "p50_s": 0.2912, "p95_s": 1.6659 }, "tiers": { "easy": { "p50_s": 0.2445, "p95_s": 0.2919 }, "original": { "p50_s": 0.2425, "p95_s": 0.2809 }, "hard": { "p50_s": 0.8172, "p95_s": 1.7308 } } }, "b_3": { "overall": { "n": 231, "accuracy": 0.5411, "ece": 0.2073, "brier": 0.6079, "p50_s": 0.3102, "p95_s": 1.6857 }, "tiers": { "easy": { "p50_s": 0.2494, "p95_s": 0.2663 }, "original": { "p50_s": 0.2509, "p95_s": 0.3102 }, "hard": { "p50_s": 0.8411, "p95_s": 1.7528 } } } }, "tiers": { "easy": { "n": 48, "n_correct": 46, "accuracy": 0.9583, "accuracy_95ci_wilson": [ 0.86, 0.988 ], "chance_corrected_accuracy": 0.9418, "harness_ece": 0.026, "own_ece": 0.026, "brier": 0.0567, "harness_brier": 0.0567, "mean_confidence": 0.9566, "confidently_wrong_ge0.8": 1, "n_conf_ge0.8": 45, "accuracy_by_confidence_band": { "conf>=0.9": { "n": 42, "acc": 1.0, "mean_conf": 0.9897 }, "0.8-0.9": { "n": 3, "acc": 0.6667, "mean_conf": 0.8279 }, "0.6-0.8": { "n": 1, "acc": 1.0, "mean_conf": 0.7692 }, "<0.6": { "n": 2, "acc": 0.5, "mean_conf": 0.5493 } }, "reliability_bins": [ { "bin": "0.0-0.1", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.1-0.2", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.2-0.3", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.3-0.4", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.4-0.5", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.5-0.6", "n": 2, "mean_conf": 0.5493, "acc": 0.5 }, { "bin": "0.6-0.7", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.7-0.8", "n": 1, "mean_conf": 0.7692, "acc": 1.0 }, { "bin": "0.8-0.9", "n": 3, "mean_conf": 0.8279, "acc": 0.6667 }, { "bin": "0.9-1.0", "n": 42, "mean_conf": 0.9897, "acc": 1.0 } ], "ordinal_mae_score_items": null, "per_family": { "extraction": { "n": 12, "accuracy": 1.0 }, "fact": { "n": 12, "accuracy": 0.833 }, "intent": { "n": 12, "accuracy": 1.0 }, "tool_selection": { "n": 12, "accuracy": 1.0 } }, "failed_requests": 0, "invalid_answers": 0 }, "original": { "n": 72, "n_correct": 51, "accuracy": 0.7083, "accuracy_95ci_wilson": [ 0.595, 0.801 ], "chance_corrected_accuracy": 0.5766, "harness_ece": 0.1673, "own_ece": 0.1673, "brier": 0.464, "harness_brier": 0.464, "mean_confidence": 0.7963, "confidently_wrong_ge0.8": 9, "n_conf_ge0.8": 40, "accuracy_by_confidence_band": { "conf>=0.9": { "n": 31, "acc": 0.8065, "mean_conf": 0.9551 }, "0.8-0.9": { "n": 9, "acc": 0.6667, "mean_conf": 0.8565 }, "0.6-0.8": { "n": 19, "acc": 0.5789, "mean_conf": 0.7042 }, "<0.6": { "n": 13, "acc": 0.6923, "mean_conf": 0.5107 } }, "reliability_bins": [ { "bin": "0.0-0.1", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.1-0.2", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.2-0.3", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.3-0.4", "n": 2, "mean_conf": 0.3812, "acc": 0.5 }, { "bin": "0.4-0.5", "n": 1, "mean_conf": 0.4959, "acc": 0.0 }, { "bin": "0.5-0.6", "n": 10, "mean_conf": 0.5381, "acc": 0.8 }, { "bin": "0.6-0.7", "n": 8, "mean_conf": 0.6421, "acc": 0.375 }, { "bin": "0.7-0.8", "n": 11, "mean_conf": 0.7493, "acc": 0.7273 }, { "bin": "0.8-0.9", "n": 9, "mean_conf": 0.8565, "acc": 0.6667 }, { "bin": "0.9-1.0", "n": 31, "mean_conf": 0.9551, "acc": 0.8065 } ], "ordinal_mae_score_items": 0.3750666666666667, "per_family": { "adequacy": { "n": 12, "accuracy": 0.667 }, "extraction": { "n": 12, "accuracy": 0.667 }, "intent": { "n": 12, "accuracy": 0.667 }, "ordinal": { "n": 12, "accuracy": 0.917 }, "policy": { "n": 12, "accuracy": 0.833 }, "routing": { "n": 12, "accuracy": 0.5 } }, "failed_requests": 0, "invalid_answers": 0 }, "hard": { "n": 111, "n_correct": 28, "accuracy": 0.2523, "accuracy_95ci_wilson": [ 0.181, 0.34 ], "chance_corrected_accuracy": -0.1264, "harness_ece": 0.3709, "own_ece": 0.3709, "brier": 0.9396, "harness_brier": 0.9396, "mean_confidence": 0.6213, "confidently_wrong_ge0.8": 21, "n_conf_ge0.8": 26, "accuracy_by_confidence_band": { "conf>=0.9": { "n": 16, "acc": 0.1875, "mean_conf": 0.9429 }, "0.8-0.9": { "n": 10, "acc": 0.2, "mean_conf": 0.8305 }, "0.6-0.8": { "n": 33, "acc": 0.3333, "mean_conf": 0.6983 }, "<0.6": { "n": 52, "acc": 0.2308, "mean_conf": 0.4332 } }, "reliability_bins": [ { "bin": "0.0-0.1", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.1-0.2", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.2-0.3", "n": 7, "mean_conf": 0.2688, "acc": 0.0 }, { "bin": "0.3-0.4", "n": 11, "mean_conf": 0.3543, "acc": 0.3636 }, { "bin": "0.4-0.5", "n": 20, "mean_conf": 0.4525, "acc": 0.25 }, { "bin": "0.5-0.6", "n": 14, "mean_conf": 0.5497, "acc": 0.2143 }, { "bin": "0.6-0.7", "n": 18, "mean_conf": 0.6628, "acc": 0.2778 }, { "bin": "0.7-0.8", "n": 15, "mean_conf": 0.741, "acc": 0.4 }, { "bin": "0.8-0.9", "n": 10, "mean_conf": 0.8305, "acc": 0.2 }, { "bin": "0.9-1.0", "n": 16, "mean_conf": 0.9429, "acc": 0.1875 } ], "ordinal_mae_score_items": 0.6396166666666666, "per_family": { "adversarial": { "n": 6, "accuracy": 0.5 }, "ambiguous": { "n": 7, "accuracy": 0.286 }, "judge_hard": { "n": 17, "accuracy": 0.235 }, "long_policy": { "n": 19, "accuracy": 0.158 }, "multi_hop": { "n": 18, "accuracy": 0.222 }, "probability": { "n": 10, "accuracy": 0.4 }, "routing_hard": { "n": 5, "accuracy": 0.2 }, "temporal_numeric": { "n": 15, "accuracy": 0.2 }, "tradeoff": { "n": 6, "accuracy": 0.667 }, "trap": { "n": 8, "accuracy": 0.0 } }, "failed_requests": 0, "invalid_answers": 0 } }, "overall": { "n": 231, "n_correct": 125, "accuracy": 0.5411, "accuracy_95ci_wilson": [ 0.477, 0.604 ], "macro_family_accuracy": 0.5158, "own_ece": 0.2073, "harness_ece": 0.2073, "brier": 0.6079, "mean_confidence": 0.7455, "confidently_wrong_ge0.8": 31, "n_conf_ge0.8": 111, "accuracy_by_confidence_band": { "conf>=0.9": { "n": 89, "acc": 0.7865, "mean_conf": 0.9692 }, "0.8-0.9": { "n": 22, "acc": 0.4545, "mean_conf": 0.8408 }, "0.6-0.8": { "n": 53, "acc": 0.434, "mean_conf": 0.7017 }, "<0.6": { "n": 67, "acc": 0.3284, "mean_conf": 0.4517 } }, "reliability_bins": [ { "bin": "0.0-0.1", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.1-0.2", "n": 0, "mean_conf": null, "acc": null }, { "bin": "0.2-0.3", "n": 7, "mean_conf": 0.2688, "acc": 0.0 }, { "bin": "0.3-0.4", "n": 13, "mean_conf": 0.3585, "acc": 0.3846 }, { "bin": "0.4-0.5", "n": 21, "mean_conf": 0.4546, "acc": 0.2381 }, { "bin": "0.5-0.6", "n": 26, "mean_conf": 0.5452, "acc": 0.4615 }, { "bin": "0.6-0.7", "n": 26, "mean_conf": 0.6564, "acc": 0.3077 }, { "bin": "0.7-0.8", "n": 27, "mean_conf": 0.7454, "acc": 0.5556 }, { "bin": "0.8-0.9", "n": 22, "mean_conf": 0.8408, "acc": 0.4545 }, { "bin": "0.9-1.0", "n": 89, "mean_conf": 0.9692, "acc": 0.7865 } ], "latency_s": { "p50": 0.2598, "p95": 1.6142, "mean": 0.5894, "max": 1.6945 } }, "abstain": "not applicable: the harness has no abstain action; every item gets an answer. Confidence below 0.6 is the only abstain-like signal." } }, "not_measured": [ "standard and judge tiers (no public files in this harness checkout)", "official composite JevBench Score (needs held-out items, speed/cost axes)", "Laya published numbers (to be quoted separately)", "GPU numbers" ] }