{
  "note": "bash-sft-03 fair baseline replacement",
  "evaluated_utc": "2026-08-31",
  "selection": {
    "stock_reason": "The exact public checkpoint from which the kept SFT lineage began.",
    "specialist_reason": "Selected before generation because its native task is natural-language-to-shell, its model card documents NL2Bash and macOS training examples, and its 0.9B scale is smaller than the 1.5B product model.",
    "specialist_model_card": "https://huggingface.co/AryaYT/nl2shell-0.8b"
  },
  "protocol": {
    "locked_components": 98,
    "same_effective_messages": true,
    "effective_prompts_sha256": "aca99dac129222eb6d566675f1bac7f560c0ead372017266084525463318d655",
    "prompt_mode": "contract-calibrated",
    "evaluator_contract_sha256": "a2630753defbc177c57893a5a8e6eab0a18e8a4735678a95de2ca3ed0a3f9cd3",
    "format_demo_sha256": "9fa4ffc9776afc24f1bc7d61b2013944c700740185145b1be99c1385355c471a",
    "max_input_tokens": 1024,
    "max_new_tokens": 256,
    "decoding": "greedy",
    "seed": 42,
    "generation_hardware": "one NVIDIA RTX 6000 Ada Generation",
    "generation_software": {
      "torch": "2.11.0+cu128",
      "transformers": "5.16.1"
    },
    "fixtures_available_during_generation": false,
    "execution_platform": "macOS 15.7.4 arm64, Bash 3.2.57",
    "completion_adapter": "strict",
    "scoring": "one raw Bash command; policy, syntax, exit status, exact stdout, exact stderr, and unchanged fixture filesystem must all pass"
  },
  "results": {
    "last40_wd01_matched": {
      "description": "The kept checkpoint rerun under the matched contract-calibrated prompt. The primary product-prompt result remains 92/98.",
      "parameters": 1543714304,
      "pass": 90,
      "n": 98,
      "raw_format_ok": 98,
      "policy_ok": 98,
      "syntax_ok": 98,
      "executed": 98,
      "filesystem_unchanged": 98,
      "exact_command": 82,
      "failure_reasons": {
        "exit_mismatch": 2,
        "stdout_mismatch": 6
      },
      "predictions_sha256": "3b24dec13e2cba808715c902ffe575f202e9f5c74767e8aeab2c6c4002ba0ebb",
      "unpublished_weights_sha256": "b3558c0f140a659f0c661bf9795ed59049cdfed5db42c1e3e8e8480e8ad901b2"
    },
    "stock_qwen25_coder_15b": {
      "hub_id": "Qwen/Qwen2.5-Coder-1.5B-Instruct",
      "revision": "2e1fd397ee46e1388853d2af2c993145b0f1098a",
      "parameters": 1543714304,
      "pass": 8,
      "n": 98,
      "raw_format_ok": 98,
      "policy_ok": 86,
      "syntax_ok": 85,
      "executed": 85,
      "filesystem_unchanged": 98,
      "exact_command": 0,
      "failure_reasons": {
        "absolute_or_home_path": 1,
        "bash_syntax": 1,
        "command_substitution": 1,
        "exit_mismatch": 61,
        "shell_lex_error": 2,
        "stdout_mismatch": 16,
        "tool_not_allowed": 6,
        "xargs_exec_not_allowed": 2
      },
      "predictions_sha256": "bde1bfdd6452f23176c65faaaf5cd3b259cdcc75508979793c90fd7f2210ab66"
    },
    "nl2shell_08b": {
      "hub_id": "AryaYT/nl2shell-0.8b",
      "revision": "d043800b678f8a879ee47219694eb2d00f048f5a",
      "license": "MIT",
      "parameters": 873438784,
      "model_card_training": "QLoRA on 8,130 NL2Bash pairs plus 40 synthetic macOS examples",
      "pass": 4,
      "n": 98,
      "raw_format_ok": 98,
      "policy_ok": 42,
      "syntax_ok": 42,
      "executed": 42,
      "filesystem_unchanged": 97,
      "exact_command": 0,
      "failure_reasons": {
        "absolute_or_home_path": 37,
        "banned_word": 1,
        "exit_mismatch": 24,
        "filesystem_changed": 1,
        "redirection": 1,
        "stdout_mismatch": 13,
        "tool_not_allowed": 11,
        "xargs_exec_not_allowed": 6
      },
      "predictions_sha256": "0d2e7f01b9840f177456d91d848bb9a2f9055c074efe72835f8eb3d98e23b003"
    }
  },
  "paired_component_bootstrap": {
    "seed": 42,
    "resamples": 10000,
    "last40_minus_stock_pass": {
      "delta": 0.8367346938775511,
      "ci95_low": 0.7551020408163265,
      "ci95_high": 0.9081632653061225
    },
    "nl2shell_minus_stock_pass": {
      "delta": -0.04081632653061224,
      "ci95_low": -0.10204081632653061,
      "ci95_high": 0.02040816326530612
    }
  },
  "boundaries": {
    "primary_product_prompt_pass": 92,
    "primary_product_prompt_n": 98,
    "matched_prompt_pass": 90,
    "matched_prompt_n": 98,
    "within_generator_development_set": true,
    "specialist_label_does_not_imply_contract_match": true,
    "raw_predictions_published": false,
    "prompts_or_fixtures_published": false,
    "evaluation_compute_destroyed_after_verified_pull": true
  }
}