{
  "schema_version": "bash-sft-02-fair-eval-summary-v1",
  "updated_date": "2026-08-21",
  "task_distribution": "98 locked representatives from distinct held-out GFR test components",
  "task_count": 98,
  "purpose": "separate adherence to the explicitly stated raw-command form from Bash semantics, add a neutral format demonstration, and disclose the judge's exact command-policy allowlist and restrictions while preserving the same execution judge",
  "judge": {
    "platform": "macOS 15.7.4, Bash 3.2",
    "checks": [
      "whole-response command acceptance",
      "read-only command policy",
      "Bash syntax",
      "execution",
      "expected exit code",
      "exact stdout",
      "exact stderr",
      "filesystem immutability"
    ],
    "timeout_seconds": 5,
    "unchanged_across_reported_conditions": true
  },
  "conditions": {
    "original_strict_reference": {
      "description": "historical product-contract score retained for transparency",
      "contract_confounded_for_semantic_comparison": true,
      "models": {
        "stock": {
          "raw_command_contract": 0,
          "passes": 0,
          "n": 98,
          "pass_rate": 0.0
        },
        "prior_session_2": {
          "raw_command_contract": 46,
          "passes": 0,
          "n": 98,
          "pass_rate": 0.0
        },
        "arm2": {
          "raw_command_contract": 98,
          "passes": 88,
          "n": 98,
          "pass_rate": 0.897959
        }
      }
    },
    "symmetric_whole_response_markdown_adapter": {
      "description": "deterministic adapter applied to every original completion before the unchanged judge",
      "accepts": "one whole-response command, raw or enclosed by one Markdown code fence",
      "selects_from_prose": false,
      "rewrites_shell": false,
      "models": {
        "stock": {
          "raw_command_contract": 0,
          "adapter_accepted": 98,
          "policy_accepted": 69,
          "syntax_valid": 69,
          "executed": 69,
          "passes": 9,
          "n": 98,
          "pass_rate": 0.091837
        },
        "prior_session_2": {
          "raw_command_contract": 46,
          "adapter_accepted": 97,
          "policy_accepted": 84,
          "syntax_valid": 84,
          "executed": 84,
          "passes": 10,
          "n": 98,
          "pass_rate": 0.102041
        },
        "arm2": {
          "raw_command_contract": 98,
          "adapter_accepted": 98,
          "policy_accepted": 98,
          "syntax_valid": 98,
          "executed": 98,
          "passes": 88,
          "n": 98,
          "pass_rate": 0.897959
        }
      },
      "arm2_minus_stock": {
        "pass_count_delta": 79,
        "pass_rate_delta": 0.806122,
        "paired_component_bootstrap_resamples": 10000,
        "bootstrap_seed": 42,
        "ci95_low": 0.714286,
        "ci95_high": 0.887755
      }
    },
    "identical_contract_calibrated_generation": {
      "description": "fresh generation from both pinned models under byte-identical effective prompts",
      "prompt_disclosures": {
        "exact_output_format": true,
        "allowed_command_policy": true,
        "neutral_noop_format_demonstration": true,
        "neutral_demonstration_assistant_output": "true"
      },
      "generation": {
        "decoding": "greedy",
        "seed": 42,
        "models_pinned": true,
        "separate_gpu_per_model": true,
        "gpu_type": "NVIDIA RTX 6000 Ada Generation"
      },
      "models": {
        "stock": {
          "raw_command_contract": 98,
          "policy_accepted": 86,
          "syntax_valid": 85,
          "executed": 85,
          "passes": 8,
          "n": 98,
          "pass_rate": 0.081633
        },
        "arm2": {
          "raw_command_contract": 98,
          "policy_accepted": 98,
          "syntax_valid": 98,
          "executed": 98,
          "passes": 89,
          "n": 98,
          "pass_rate": 0.908163
        }
      },
      "arm2_minus_stock": {
        "pass_count_delta": 81,
        "pass_rate_delta": 0.826531,
        "paired_component_bootstrap_resamples": 10000,
        "bootstrap_seed": 42,
        "ci95_low": 0.744898,
        "ci95_high": 0.897959
      }
    }
  },
  "cleanup": {
    "temporary_gpu_droplets": 2,
    "deleted_after_artifact_pull": 2,
    "remaining": 0
  },
  "interpretation": {
    "original_strict_stock_zero_is_semantic_baseline": false,
    "raw_format_confound_removed": true,
    "substantive_within_generator_gap_remains": true,
    "broad_bash_transfer_established": false
  },
  "public_artifact_boundary": {
    "aggregate_only": true,
    "includes_prompts": false,
    "includes_model_completions": false,
    "includes_fixtures": false,
    "includes_paths_or_machine_identifiers": false
  }
}