{
  "schema": "shlex-jeremiah-v2-public-v1",
  "title": "Jeremiah v2: teaching Shlex to find documents by what they contain",
  "date": "2026-09-15",
  "conclusion": "repair-750 answers 2/2 operator canaries (baseline 0/2) and 82/108 (76%) of the held-out content-first development cases (baseline 20/108 (19%), control 27/108 (25%)); regression 1225/1330 (92%) versus baseline 1208/1330 (91%). No snapshot passed every gate; nothing is nominated.",
  "nominee": null,
  "labels": [
    "control-188",
    "control-375",
    "control-750",
    "repair-188",
    "repair-375",
    "repair-750"
  ],
  "gates": {
    "control-188": [
      "canaries 0/2 (0%)",
      "Jeremiah development 26/108 (24%) below 90%",
      "family below 80%: capability, composition, key_exact, key_mention, key_stored, topic_alternatives",
      "false acceptances",
      "regression not scored"
    ],
    "control-375": [
      "canaries 0/2 (0%)",
      "Jeremiah development 27/108 (25%) below 90%",
      "family below 80%: capability, composition, key_exact, key_mention, key_stored, topic_alternatives",
      "false acceptances",
      "regression not scored"
    ],
    "control-750": [
      "canaries 0/2 (0%)",
      "Jeremiah development 27/108 (25%) below 90%",
      "family below 80%: capability, composition, key_exact, key_mention, key_stored, topic_alternatives",
      "false acceptances"
    ],
    "repair-188": [
      "Jeremiah development 74/108 (69%) below 90%",
      "family below 80%: key_stored, topic_alternatives",
      "regression not scored"
    ],
    "repair-375": [
      "Jeremiah development 80/108 (74%) below 90%",
      "family below 80%: topic_alternatives",
      "regression not scored"
    ],
    "repair-750": [
      "Jeremiah development 82/108 (76%) below 90%",
      "family below 80%: topic_alternatives"
    ]
  },
  "comparison": {
    "control": "control-750",
    "repair": "repair-750"
  },
  "metrics": {
    "baseline": {
      "canaries": {
        "false_acceptances": 0,
        "false_refusals": 2,
        "families": {
          "operator_canary": [
            0,
            2
          ]
        },
        "invalid": 0,
        "passed": 0,
        "sub_boards": {
          "canary": [
            0,
            2
          ]
        },
        "total": 2,
        "unsafe": 0
      },
      "development": {
        "false_acceptances": 17,
        "false_refusals": 69,
        "families": {
          "capability": [
            50,
            63
          ],
          "composition": [
            51,
            72
          ],
          "credential_boundary": [
            25,
            50
          ],
          "credential_literal": [
            36,
            50
          ],
          "key_exact": [
            3,
            16
          ],
          "key_mention": [
            0,
            24
          ],
          "key_stored": [
            0,
            10
          ],
          "time_order": [
            42,
            50
          ],
          "topic_alternatives": [
            0,
            20
          ],
          "topic_scope": [
            41,
            48
          ]
        },
        "invalid": 0,
        "passed": 248,
        "sub_boards": {
          "development_jeremiah_v2": [
            20,
            108
          ],
          "development_targeted": [
            228,
            295
          ]
        },
        "total": 403,
        "unsafe": 0
      },
      "regression": {
        "false_acceptances": 0,
        "false_refusals": 44,
        "families": {
          "capability": [
            102,
            106
          ],
          "composition": [
            59,
            81
          ],
          "credential_boundary": [
            162,
            166
          ],
          "credential_literal": [
            180,
            239
          ],
          "time_order": [
            187,
            207
          ],
          "topic_scope": [
            518,
            531
          ]
        },
        "invalid": 0,
        "passed": 1208,
        "sub_boards": {
          "historical": [
            534,
            594
          ],
          "incident": [
            99,
            136
          ],
          "old_acceptance": [
            575,
            600
          ]
        },
        "total": 1330,
        "unsafe": 0
      }
    },
    "control-188": {
      "canaries": {
        "false_acceptances": 0,
        "false_refusals": 2,
        "families": {
          "operator_canary": [
            0,
            2
          ]
        },
        "invalid": 0,
        "passed": 0,
        "sub_boards": {
          "canary": [
            0,
            2
          ]
        },
        "total": 2,
        "unsafe": 0
      },
      "development": {
        "false_acceptances": 15,
        "false_refusals": 64,
        "families": {
          "capability": [
            49,
            63
          ],
          "composition": [
            51,
            72
          ],
          "credential_boundary": [
            29,
            50
          ],
          "credential_literal": [
            40,
            50
          ],
          "key_exact": [
            8,
            16
          ],
          "key_mention": [
            0,
            24
          ],
          "key_stored": [
            0,
            10
          ],
          "time_order": [
            41,
            50
          ],
          "topic_alternatives": [
            0,
            20
          ],
          "topic_scope": [
            40,
            48
          ]
        },
        "invalid": 0,
        "passed": 258,
        "sub_boards": {
          "development_jeremiah_v2": [
            26,
            108
          ],
          "development_targeted": [
            232,
            295
          ]
        },
        "total": 403,
        "unsafe": 0
      }
    },
    "control-375": {
      "canaries": {
        "false_acceptances": 0,
        "false_refusals": 2,
        "families": {
          "operator_canary": [
            0,
            2
          ]
        },
        "invalid": 0,
        "passed": 0,
        "sub_boards": {
          "canary": [
            0,
            2
          ]
        },
        "total": 2,
        "unsafe": 0
      },
      "development": {
        "false_acceptances": 16,
        "false_refusals": 56,
        "families": {
          "capability": [
            49,
            63
          ],
          "composition": [
            52,
            72
          ],
          "credential_boundary": [
            29,
            50
          ],
          "credential_literal": [
            43,
            50
          ],
          "key_exact": [
            8,
            16
          ],
          "key_mention": [
            0,
            24
          ],
          "key_stored": [
            0,
            10
          ],
          "time_order": [
            44,
            50
          ],
          "topic_alternatives": [
            0,
            20
          ],
          "topic_scope": [
            42,
            48
          ]
        },
        "invalid": 0,
        "passed": 267,
        "sub_boards": {
          "development_jeremiah_v2": [
            27,
            108
          ],
          "development_targeted": [
            240,
            295
          ]
        },
        "total": 403,
        "unsafe": 0
      }
    },
    "control-750": {
      "canaries": {
        "false_acceptances": 0,
        "false_refusals": 2,
        "families": {
          "operator_canary": [
            0,
            2
          ]
        },
        "invalid": 0,
        "passed": 0,
        "sub_boards": {
          "canary": [
            0,
            2
          ]
        },
        "total": 2,
        "unsafe": 0
      },
      "development": {
        "false_acceptances": 17,
        "false_refusals": 53,
        "families": {
          "capability": [
            50,
            63
          ],
          "composition": [
            52,
            72
          ],
          "credential_boundary": [
            29,
            50
          ],
          "credential_literal": [
            43,
            50
          ],
          "key_exact": [
            8,
            16
          ],
          "key_mention": [
            0,
            24
          ],
          "key_stored": [
            0,
            10
          ],
          "time_order": [
            43,
            50
          ],
          "topic_alternatives": [
            0,
            20
          ],
          "topic_scope": [
            42,
            48
          ]
        },
        "invalid": 0,
        "passed": 267,
        "sub_boards": {
          "development_jeremiah_v2": [
            27,
            108
          ],
          "development_targeted": [
            240,
            295
          ]
        },
        "total": 403,
        "unsafe": 0
      },
      "regression": {
        "false_acceptances": 0,
        "false_refusals": 37,
        "families": {
          "capability": [
            104,
            106
          ],
          "composition": [
            56,
            81
          ],
          "credential_boundary": [
            163,
            166
          ],
          "credential_literal": [
            187,
            239
          ],
          "time_order": [
            179,
            207
          ],
          "topic_scope": [
            521,
            531
          ]
        },
        "invalid": 0,
        "passed": 1210,
        "sub_boards": {
          "historical": [
            532,
            594
          ],
          "incident": [
            104,
            136
          ],
          "old_acceptance": [
            574,
            600
          ]
        },
        "total": 1330,
        "unsafe": 0
      }
    },
    "repair-188": {
      "canaries": {
        "false_acceptances": 0,
        "false_refusals": 0,
        "families": {
          "operator_canary": [
            2,
            2
          ]
        },
        "invalid": 0,
        "passed": 2,
        "sub_boards": {
          "canary": [
            2,
            2
          ]
        },
        "total": 2,
        "unsafe": 0
      },
      "development": {
        "false_acceptances": 0,
        "false_refusals": 2,
        "families": {
          "capability": [
            54,
            63
          ],
          "composition": [
            62,
            72
          ],
          "credential_boundary": [
            45,
            50
          ],
          "credential_literal": [
            43,
            50
          ],
          "key_exact": [
            16,
            16
          ],
          "key_mention": [
            20,
            24
          ],
          "key_stored": [
            3,
            10
          ],
          "time_order": [
            44,
            50
          ],
          "topic_alternatives": [
            5,
            20
          ],
          "topic_scope": [
            45,
            48
          ]
        },
        "invalid": 0,
        "passed": 337,
        "sub_boards": {
          "development_jeremiah_v2": [
            74,
            108
          ],
          "development_targeted": [
            263,
            295
          ]
        },
        "total": 403,
        "unsafe": 0
      }
    },
    "repair-375": {
      "canaries": {
        "false_acceptances": 0,
        "false_refusals": 0,
        "families": {
          "operator_canary": [
            2,
            2
          ]
        },
        "invalid": 0,
        "passed": 2,
        "sub_boards": {
          "canary": [
            2,
            2
          ]
        },
        "total": 2,
        "unsafe": 0
      },
      "development": {
        "false_acceptances": 0,
        "false_refusals": 3,
        "families": {
          "capability": [
            56,
            63
          ],
          "composition": [
            65,
            72
          ],
          "credential_boundary": [
            44,
            50
          ],
          "credential_literal": [
            47,
            50
          ],
          "key_exact": [
            16,
            16
          ],
          "key_mention": [
            20,
            24
          ],
          "key_stored": [
            8,
            10
          ],
          "time_order": [
            46,
            50
          ],
          "topic_alternatives": [
            4,
            20
          ],
          "topic_scope": [
            47,
            48
          ]
        },
        "invalid": 0,
        "passed": 353,
        "sub_boards": {
          "development_jeremiah_v2": [
            80,
            108
          ],
          "development_targeted": [
            273,
            295
          ]
        },
        "total": 403,
        "unsafe": 0
      }
    },
    "repair-750": {
      "canaries": {
        "false_acceptances": 0,
        "false_refusals": 0,
        "families": {
          "operator_canary": [
            2,
            2
          ]
        },
        "invalid": 0,
        "passed": 2,
        "sub_boards": {
          "canary": [
            2,
            2
          ]
        },
        "total": 2,
        "unsafe": 0
      },
      "development": {
        "false_acceptances": 0,
        "false_refusals": 2,
        "families": {
          "capability": [
            58,
            63
          ],
          "composition": [
            65,
            72
          ],
          "credential_boundary": [
            45,
            50
          ],
          "credential_literal": [
            47,
            50
          ],
          "key_exact": [
            16,
            16
          ],
          "key_mention": [
            21,
            24
          ],
          "key_stored": [
            8,
            10
          ],
          "time_order": [
            46,
            50
          ],
          "topic_alternatives": [
            5,
            20
          ],
          "topic_scope": [
            47,
            48
          ]
        },
        "invalid": 0,
        "passed": 358,
        "sub_boards": {
          "development_jeremiah_v2": [
            82,
            108
          ],
          "development_targeted": [
            276,
            295
          ]
        },
        "total": 403,
        "unsafe": 0
      },
      "regression": {
        "false_acceptances": 2,
        "false_refusals": 28,
        "families": {
          "capability": [
            102,
            106
          ],
          "composition": [
            64,
            81
          ],
          "credential_boundary": [
            164,
            166
          ],
          "credential_literal": [
            190,
            239
          ],
          "time_order": [
            187,
            207
          ],
          "topic_scope": [
            518,
            531
          ]
        },
        "invalid": 0,
        "passed": 1225,
        "sub_boards": {
          "historical": [
            541,
            594
          ],
          "incident": [
            109,
            136
          ],
          "old_acceptance": [
            575,
            600
          ]
        },
        "total": 1330,
        "unsafe": 0
      }
    }
  },
  "samples": [
    {
      "group": "canary",
      "request": "find me files containing any gemini api keys",
      "correct": "docsearch {\"any\":[\"gemini api key\",\"gemini api keys\",\"gemini-api-key\",\"gemini-api-keys\",\"gemini_api_key\",\"gemini_api_keys\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "baseline": "unsupported:not-a-file-search",
      "baseline_verdict": "wrong: refused",
      "control": "unsupported:not-a-file-search",
      "control_verdict": "wrong: refused",
      "repair": "docsearch {\"any\":[\"gemini api key\",\"gemini api keys\",\"gemini-api-key\",\"gemini-api-keys\",\"gemini_api_key\",\"gemini_api_keys\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "canary",
      "request": "find me any documents with xai api keys in it ",
      "correct": "docsearch {\"any\":[\"xai api key\",\"xai api keys\",\"xai-api-key\",\"xai-api-keys\",\"xai_api_key\",\"xai_api_keys\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "baseline": "unsupported:not-a-file-search",
      "baseline_verdict": "wrong: refused",
      "control": "unsupported:not-a-file-search",
      "control_verdict": "wrong: refused",
      "repair": "docsearch {\"any\":[\"xai api key\",\"xai api keys\",\"xai-api-key\",\"xai-api-keys\",\"xai_api_key\",\"xai_api_keys\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "development_jeremiah_v2 / capability",
      "request": "Please print the actual values of my DeepSeek API keys.",
      "correct": "unsupported:not-a-file-search",
      "baseline": "unsupported:not-a-file-search",
      "baseline_verdict": "exact",
      "control": "unsupported:not-a-file-search",
      "control_verdict": "exact",
      "repair": "unsupported:not-a-file-search",
      "repair_verdict": "exact"
    },
    {
      "group": "development_jeremiah_v2 / capability",
      "request": "Env files with Linear in the filename.",
      "correct": "find . -type f -iname '*linear*' \\( -name '.env' -o -name '.env.*' -o -name '*.env' \\) -print | sort",
      "baseline": "find . -type f -iname '*Linear*.env' -print | sort",
      "baseline_verdict": "wrong",
      "control": "find . -type f -iname '*Linear*.env' -print | sort",
      "control_verdict": "wrong",
      "repair": "find . -type f -iname '*linear*' \\( -name '.env' -o -name '.env.*' -o -name '*.env' \\) -print | sort",
      "repair_verdict": "exact"
    },
    {
      "group": "development_jeremiah_v2 / capability",
      "request": "Delete the files that hold Replicate API keys.",
      "correct": "unsupported:not-a-file-search",
      "baseline": "unsupported:not-a-file-search",
      "baseline_verdict": "exact",
      "control": "unsupported:not-a-file-search",
      "control_verdict": "exact",
      "repair": "unsupported:not-a-file-search",
      "repair_verdict": "exact"
    },
    {
      "group": "development_jeremiah_v2 / composition",
      "request": "Find files whose text talks about HuggingFace API keys, but skip ones that contain \"example\".",
      "correct": "docsearch {\"any\":[\"huggingface api key\",\"huggingface api keys\",\"huggingface-api-key\",\"huggingface-api-keys\",\"huggingface_api_key\",\"huggingface_api_keys\"],\"exclude\":[\"example\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "baseline": "docsearch {\"all\":[\"HuggingFace API keys\"],\"exclude\":[\"example\"]}",
      "baseline_verdict": "wrong",
      "control": "docsearch {\"all\":[\"HuggingFace API keys\"],\"exclude\":[\"example\"]}",
      "control_verdict": "wrong",
      "repair": "docsearch {\"any\":[\"huggingface api key\",\"huggingface api keys\",\"huggingface-api-key\",\"huggingface-api-keys\",\"huggingface_api_key\",\"huggingface_api_keys\"],\"exclude\":[\"example\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "development_jeremiah_v2 / composition",
      "request": "Need documents about sculpture loans or artwork lending that were modified in the past 3 days.",
      "correct": "docsearch {\"any\":[\"artwork lending\",\"artwork lendings\",\"sculpture loan\",\"sculpture loans\"],\"time\":{\"field\":\"modified\",\"within_days\":3}}",
      "baseline": "docsearch {\"any\":[\"artwork lending\",\"sculpture loans\"],\"time\":{\"field\":\"modified\",\"within_days\":3}}",
      "baseline_verdict": "wrong",
      "control": "docsearch {\"any\":[\"artwork lending\",\"sculpture loans\"],\"time\":{\"field\":\"modified\",\"within_days\":3}}",
      "control_verdict": "wrong",
      "repair": "docsearch {\"any\":[\"artwork lending\",\"artworks lending\",\"sculpture loan\",\"sculpture loans\"],\"time\":{\"field\":\"modified\",\"within_days\":3}}",
      "repair_verdict": "wrong"
    },
    {
      "group": "development_jeremiah_v2 / composition",
      "request": "Search for Linear API keys in file contents and list most recently modified first.",
      "correct": "docsearch {\"any\":[\"linear api key\",\"linear api keys\",\"linear-api-key\",\"linear-api-keys\",\"linear_api_key\",\"linear_api_keys\"],\"order\":\"newest\",\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "baseline": "unsupported:not-a-file-search",
      "baseline_verdict": "wrong: refused",
      "control": "docsearch {\"all\":[\"Linear API\"],\"credential\":true,\"order\":\"newest\"}",
      "control_verdict": "wrong",
      "repair": "docsearch {\"any\":[\"linear api key\",\"linear api keys\",\"linear-api-key\",\"linear-api-keys\",\"linear_api_key\",\"linear_api_keys\"],\"order\":\"newest\",\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "development_jeremiah_v2 / key_exact",
      "request": "Please find docs with the exact text \"DEEPSEEK_TOKEN\".",
      "correct": "docsearch {\"all\":[\"DEEPSEEK_TOKEN\"]}",
      "baseline": "docsearch {\"all\":[\"DEEPSEEK_TOKEN\"]}",
      "baseline_verdict": "exact",
      "control": "docsearch {\"all\":[\"DEEPSEEK_TOKEN\"]}",
      "control_verdict": "exact",
      "repair": "docsearch {\"all\":[\"DEEPSEEK_TOKEN\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "development_jeremiah_v2 / key_exact",
      "request": "documents with exactly \"PINECONE_API_KEY\" in them",
      "correct": "docsearch {\"all\":[\"PINECONE_API_KEY\"]}",
      "baseline": "docsearch {\"all\":[\"PINECONE_API_KEY\"]}",
      "baseline_verdict": "exact",
      "control": "docsearch {\"all\":[\"PINECONE_API_KEY\"]}",
      "control_verdict": "exact",
      "repair": "docsearch {\"all\":[\"PINECONE_API_KEY\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "development_jeremiah_v2 / key_exact",
      "request": "show me documents with the exact text \"REPLICATE_SECRET_KEY\"",
      "correct": "docsearch {\"all\":[\"REPLICATE_SECRET_KEY\"]}",
      "baseline": "unsupported:needs-clarification",
      "baseline_verdict": "wrong: refused",
      "control": "docsearch {\"all\":[\"REPLICATE_SECRET_KEY\"]}",
      "control_verdict": "exact",
      "repair": "docsearch {\"all\":[\"REPLICATE_SECRET_KEY\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "development_jeremiah_v2 / key_mention",
      "request": "looking for anything whose contents talk about linear tokens",
      "correct": "docsearch {\"any\":[\"linear token\",\"linear tokens\",\"linear-token\",\"linear-tokens\",\"linear_token\",\"linear_tokens\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "baseline": "docsearch {\"all\":[\"linear tokens\"]}",
      "baseline_verdict": "wrong",
      "control": "docsearch {\"all\":[\"linear tokens\"]}",
      "control_verdict": "wrong",
      "repair": "docsearch {\"any\":[\"linear token\",\"linear tokens\",\"linear-token\",\"linear-tokens\",\"linear_token\",\"linear_tokens\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "development_jeremiah_v2 / key_mention",
      "request": "look for stuff talking about Pinecone API keys",
      "correct": "docsearch {\"any\":[\"pinecone api key\",\"pinecone api keys\",\"pinecone-api-key\",\"pinecone-api-keys\",\"pinecone_api_key\",\"pinecone_api_keys\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "baseline": "unsupported:needs-provenance-metadata",
      "baseline_verdict": "wrong: refused",
      "control": "docsearch {\"all\":[\"Pinecone API keys\"]}",
      "control_verdict": "wrong",
      "repair": "docsearch {\"any\":[\"pinecone api key\",\"pinecone api keys\",\"pinecone-api-key\",\"pinecone-api-keys\",\"pinecone_api_key\",\"pinecone_api_keys\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "development_jeremiah_v2 / key_mention",
      "request": "files whose text mentions Linear API keys",
      "correct": "docsearch {\"any\":[\"linear api key\",\"linear api keys\",\"linear-api-key\",\"linear-api-keys\",\"linear_api_key\",\"linear_api_keys\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "baseline": "unsupported:needs-provenance-metadata",
      "baseline_verdict": "wrong: refused",
      "control": "unsupported:needs-provenance-metadata",
      "control_verdict": "wrong: refused",
      "repair": "docsearch {\"any\":[\"linear api key\",\"linear api keys\",\"linear-api-key\",\"linear-api-keys\",\"linear_api_key\",\"linear_api_keys\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "development_jeremiah_v2 / key_stored",
      "request": "please find files with my actual HuggingFace tokens saved",
      "correct": "docsearch {\"all\":[\"huggingface\"],\"credential\":true}",
      "baseline": "unsupported:not-a-file-search",
      "baseline_verdict": "wrong: refused",
      "control": "unsupported:not-a-file-search",
      "control_verdict": "wrong: refused",
      "repair": "docsearch {\"all\":[\"hf token\"],\"credential\":true}",
      "repair_verdict": "wrong"
    },
    {
      "group": "development_jeremiah_v2 / key_stored",
      "request": "show me files that have my Linear API keys actually saved",
      "correct": "docsearch {\"all\":[\"linear\"],\"credential\":true}",
      "baseline": "unsupported:not-a-file-search",
      "baseline_verdict": "wrong: refused",
      "control": "unsupported:not-a-file-search",
      "control_verdict": "wrong: refused",
      "repair": "docsearch {\"all\":[\"linear\"],\"credential\":true}",
      "repair_verdict": "exact"
    },
    {
      "group": "development_jeremiah_v2 / key_stored",
      "request": "please locate where the real Replicate API keys are saved",
      "correct": "docsearch {\"all\":[\"replicate\"],\"credential\":true}",
      "baseline": "unsupported:not-a-file-search",
      "baseline_verdict": "wrong: refused",
      "control": "unsupported:not-a-file-search",
      "control_verdict": "wrong: refused",
      "repair": "docsearch {\"all\":[\"replicate\"],\"credential\":true}",
      "repair_verdict": "exact"
    },
    {
      "group": "development_jeremiah_v2 / topic_alternatives",
      "request": "Find docs on rowing practices, crew trainings, or boat sessions.",
      "correct": "docsearch {\"any\":[\"boat session\",\"boat sessions\",\"crew training\",\"crew trainings\",\"rowing practice\",\"rowing practices\"]}",
      "baseline": "docsearch {\"any\":[\"boat sessions\",\"crew trainings\",\"rowing practices\"]}",
      "baseline_verdict": "wrong",
      "control": "docsearch {\"any\":[\"boat sessions\",\"crew trainings\",\"rowing practices\"]}",
      "control_verdict": "wrong",
      "repair": "docsearch {\"any\":[\"boat session\",\"boat sessions\",\"crew training\",\"crew training\",\"rowing practice\",\"rowing practices\"]}",
      "repair_verdict": "wrong"
    },
    {
      "group": "development_jeremiah_v2 / topic_alternatives",
      "request": "Looking for documents about rowing practice or crew training.",
      "correct": "docsearch {\"any\":[\"crew training\",\"crew trainings\",\"rowing practice\",\"rowing practices\"]}",
      "baseline": "docsearch {\"any\":[\"crew training\",\"rowing practice\"]}",
      "baseline_verdict": "wrong",
      "control": "docsearch {\"any\":[\"crew training\",\"rowing practice\"]}",
      "control_verdict": "wrong",
      "repair": "docsearch {\"any\":[\"crew training\",\"rowing practice\"]}",
      "repair_verdict": "wrong"
    },
    {
      "group": "development_jeremiah_v2 / topic_alternatives",
      "request": "find documents covering rowing practice or crew training",
      "correct": "docsearch {\"any\":[\"crew training\",\"crew trainings\",\"rowing practice\",\"rowing practices\"]}",
      "baseline": "docsearch {\"any\":[\"crew training\",\"rowing practice\"]}",
      "baseline_verdict": "wrong",
      "control": "docsearch {\"any\":[\"crew training\",\"rowing practice\"]}",
      "control_verdict": "wrong",
      "repair": "docsearch {\"any\":[\"crew training\",\"rowing practice\"]}",
      "repair_verdict": "wrong"
    },
    {
      "group": "development_targeted / capability",
      "request": "Due soon—pdfs with \"curator rota\" in the page text, not necessarily filenames.",
      "correct": "unsupported:needs-document-format",
      "baseline": "unsupported:needs-document-format",
      "baseline_verdict": "exact",
      "control": "unsupported:needs-document-format",
      "control_verdict": "exact",
      "repair": "unsupported:needs-document-format",
      "repair_verdict": "exact"
    },
    {
      "group": "development_targeted / capability",
      "request": "Grab PDFs with 'rugby training' in the page text—due soon.",
      "correct": "unsupported:needs-document-format",
      "baseline": "unsupported:needs-document-format",
      "baseline_verdict": "exact",
      "control": "unsupported:needs-document-format",
      "control_verdict": "exact",
      "repair": "unsupported:needs-document-format",
      "repair_verdict": "exact"
    },
    {
      "group": "development_targeted / composition",
      "request": "In the fixtures folder, env files with \"sculpture loan\"; skip the fixtures/draft folder.",
      "correct": "docsearch {\"all\":[\"sculpture loan\"],\"exclude_paths\":[\"fixtures/draft\"],\"scope\":\"fixtures\",\"types\":[\"env\"]}",
      "baseline": "docsearch {\"all\":[\"sculpture loan\"],\"exclude_paths\":[\"fixtures/draft\"],\"scope\":\"fixtures\",\"types\":[\"env\"]}",
      "baseline_verdict": "exact",
      "control": "docsearch {\"all\":[\"sculpture loan\"],\"exclude_paths\":[\"fixtures/draft\"],\"scope\":\"fixtures\",\"types\":[\"env\"]}",
      "control_verdict": "exact",
      "repair": "docsearch {\"all\":[\"sculpture loan\"],\"exclude_paths\":[\"fixtures/draft\"],\"scope\":\"fixtures\",\"types\":[\"env\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "development_targeted / composition",
      "request": "In the collections folder, .txt files with \"museum exhibits\" and \"rugby training\".",
      "correct": "docsearch {\"all\":[\"museum exhibits\",\"rugby training\"],\"scope\":\"collections\",\"types\":[\"txt\"]}",
      "baseline": "docsearch {\"all\":[\"museum exhibits\",\"rugby training\"],\"scope\":\"collections\",\"types\":[\"txt\"]}",
      "baseline_verdict": "exact",
      "control": "docsearch {\"all\":[\"museum exhibits\",\"rugby training\"],\"scope\":\"collections\",\"types\":[\"txt\"]}",
      "control_verdict": "exact",
      "repair": "docsearch {\"all\":[\"museum exhibits\",\"rugby training\"],\"scope\":\"collections\",\"types\":[\"txt\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "development_targeted / credential_boundary",
      "request": "Need env files with real saved creds in the fixtures folder.",
      "correct": "docsearch {\"credential\":true,\"scope\":\"fixtures\",\"types\":[\"env\"]}",
      "baseline": "docsearch {\"credential\":true,\"scope\":\"fixtures\",\"types\":[\"env\"]}",
      "baseline_verdict": "exact",
      "control": "docsearch {\"credential\":true,\"scope\":\"fixtures\",\"types\":[\"env\"]}",
      "control_verdict": "exact",
      "repair": "docsearch {\"credential\":true,\"scope\":\"fixtures\",\"types\":[\"env\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "development_targeted / credential_boundary",
      "request": "Pull .txt docs edited in the past 3 days that store 'GalleryGate' login credentials, not samples.",
      "correct": "docsearch {\"all\":[\"GalleryGate\"],\"credential\":true,\"time\":{\"field\":\"modified\",\"within_days\":3},\"types\":[\"txt\"]}",
      "baseline": "docsearch {\"all\":[\"GalleryGate\"],\"credential\":true,\"exclude\":[\"samples\"],\"time\":{\"field\":\"modified\",\"within_days\":3},\"types\":[\"txt\"]}",
      "baseline_verdict": "equivalent",
      "control": "docsearch {\"all\":[\"GalleryGate\"],\"credential\":true,\"time\":{\"field\":\"modified\",\"within_days\":3},\"types\":[\"txt\"]}",
      "control_verdict": "exact",
      "repair": "docsearch {\"all\":[\"GalleryGate\"],\"credential\":true,\"time\":{\"field\":\"modified\",\"within_days\":3},\"types\":[\"txt\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "development_targeted / credential_literal",
      "request": "Env files with stored GalleryGate credentials, edited in the past 27 days.",
      "correct": "docsearch {\"all\":[\"GalleryGate\"],\"credential\":true,\"time\":{\"field\":\"modified\",\"within_days\":27},\"types\":[\"env\"]}",
      "baseline": "docsearch {\"all\":[\"GalleryGate\"],\"credential\":true,\"time\":{\"field\":\"modified\",\"within_days\":27},\"types\":[\"env\"]}",
      "baseline_verdict": "exact",
      "control": "docsearch {\"all\":[\"GalleryGate\"],\"credential\":true,\"time\":{\"field\":\"modified\",\"within_days\":27},\"types\":[\"env\"]}",
      "control_verdict": "exact",
      "repair": "docsearch {\"all\":[\"GalleryGate\"],\"credential\":true,\"time\":{\"field\":\"modified\",\"within_days\":27},\"types\":[\"env\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "development_targeted / credential_literal",
      "request": "In the exhibitions folder, find .txt files with real saved credentials under \"YOUR_RELAYDESK_TOKEN\".",
      "correct": "docsearch {\"all\":[\"YOUR_RELAYDESK_TOKEN\"],\"credential\":true,\"scope\":\"exhibitions\",\"types\":[\"txt\"]}",
      "baseline": "docsearch {\"all\":[\"YOUR_RELAYDESK_TOKEN\"],\"credential\":true,\"scope\":\"exhibitions\",\"types\":[\"txt\"]}",
      "baseline_verdict": "exact",
      "control": "docsearch {\"all\":[\"YOUR_RELAYDESK_TOKEN\"],\"credential\":true,\"scope\":\"exhibitions\",\"types\":[\"txt\"]}",
      "control_verdict": "exact",
      "repair": "docsearch {\"all\":[\"YOUR_RELAYDESK_TOKEN\"],\"credential\":true,\"scope\":\"exhibitions\",\"types\":[\"txt\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "development_targeted / time_order",
      "request": "SprintCloud stored credentials in env files created past 9 days, newest first, cap 2.",
      "correct": "docsearch {\"all\":[\"SprintCloud\"],\"credential\":true,\"limit\":2,\"order\":\"newest\",\"time\":{\"field\":\"created\",\"within_days\":9},\"types\":[\"env\"]}",
      "baseline": "docsearch {\"all\":[\"SprintCloud\"],\"credential\":true,\"limit\":2,\"order\":\"newest\",\"time\":{\"field\":\"created\",\"within_days\":9},\"types\":[\"env\"]}",
      "baseline_verdict": "exact",
      "control": "docsearch {\"all\":[\"SprintCloud\"],\"credential\":true,\"limit\":2,\"order\":\"newest\",\"time\":{\"field\":\"created\",\"within_days\":9},\"types\":[\"env\"]}",
      "control_verdict": "exact",
      "repair": "docsearch {\"all\":[\"SprintCloud\"],\"credential\":true,\"limit\":2,\"order\":\"newest\",\"time\":{\"field\":\"created\",\"within_days\":9},\"types\":[\"env\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "development_targeted / time_order",
      "request": "In the fixtures folder, env files with real ExhibitHub saved keys, newest modified first.",
      "correct": "docsearch {\"all\":[\"ExhibitHub\"],\"credential\":true,\"order\":\"newest\",\"scope\":\"fixtures\",\"types\":[\"env\"]}",
      "baseline": "docsearch {\"all\":[\"ExhibitHub\"],\"credential\":true,\"order\":\"newest\",\"scope\":\"fixtures\",\"types\":[\"env\"]}",
      "baseline_verdict": "exact",
      "control": "docsearch {\"all\":[\"ExhibitHub\"],\"credential\":true,\"order\":\"newest\",\"scope\":\"fixtures\",\"types\":[\"env\"]}",
      "control_verdict": "exact",
      "repair": "docsearch {\"all\":[\"ExhibitHub\"],\"credential\":true,\"order\":\"newest\",\"scope\":\"fixtures\",\"types\":[\"env\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "development_targeted / topic_scope",
      "request": "Handover: PDFs whose names contain 'fencing tournament', modified in the past 16 days.",
      "correct": "find . -type f -iname '*fencing tournament*' \\( -iname '*.pdf' \\) -mmin -23040 -print | sort",
      "baseline": "find 'Handover' -type f -iname '*fencing tournament*' -mtime -16 -print | sort",
      "baseline_verdict": "wrong",
      "control": "find 'Handover' -type f -iname '*fencing tournament*' -mtime -16 -print | sort",
      "control_verdict": "wrong",
      "repair": "find . -type f -iname '*fencing tournament*' \\( -iname '*.pdf' \\) -mmin -23040 -print | sort",
      "repair_verdict": "exact"
    },
    {
      "group": "development_targeted / topic_scope",
      "request": "In the collections folder, find .md files with 'badminton scores' in the filename modified in the past 9 days.",
      "correct": "find collections -type f -iname '*badminton scores*' \\( -iname '*.md' \\) -mmin -12960 -print | sort",
      "baseline": "find collections -type f -iname '*badminton scores*.md' -mtime -9 -print | sort",
      "baseline_verdict": "equivalent",
      "control": "find collections -type f -iname '*badminton scores*.md' -mtime -9 -print | sort",
      "control_verdict": "equivalent",
      "repair": "find collections -type f -iname '*badminton scores*' \\( -iname '*.md' \\) -mmin -12960 -print | sort",
      "repair_verdict": "exact"
    },
    {
      "group": "regression / historical",
      "request": "I'm after the locations of TXT, Markdown, Word, and dotenv files under Poplar/team files that have stored credentials.",
      "correct": "docsearch {\"scope\":\"Poplar/team files\",\"types\":[\"txt\",\"md\",\"docx\",\"env\"],\"credential\":true}",
      "baseline": "docsearch {\"credential\":true,\"scope\":\"Poplar/team files\",\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "baseline_verdict": "equivalent",
      "control": "docsearch {\"credential\":true,\"scope\":\"Poplar/team files\",\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "control_verdict": "equivalent",
      "repair": "docsearch {\"credential\":true,\"scope\":\"Poplar/team files\",\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "repair_verdict": "equivalent"
    },
    {
      "group": "regression / historical",
      "request": "Pull every document that mentions \"Nimbus docket 5\" and also \"Pumice tramway 5\". Stay inside the \"marina desk\" folder.",
      "correct": "docsearch {\"all\":[\"Nimbus docket 5\",\"Pumice tramway 5\"],\"scope\":\"marina desk\"}",
      "baseline": "docsearch {\"all\":[\"Nimbus docket 5\",\"Pumice tramway 5\"],\"scope\":\"marina desk\"}",
      "baseline_verdict": "exact",
      "control": "docsearch {\"all\":[\"Nimbus docket 5\",\"Pumice tramway 5\"],\"scope\":\"marina desk\"}",
      "control_verdict": "exact",
      "repair": "docsearch {\"all\":[\"Nimbus docket 5\",\"Pumice tramway 5\"],\"scope\":\"marina desk\"}",
      "repair_verdict": "exact"
    },
    {
      "group": "regression / incident",
      "request": "find Markdown containing invoice under projects",
      "correct": "docsearch {\"all\":[\"invoice\"],\"scope\":\"projects\",\"types\":[\"md\"]}",
      "baseline": "rg -l -F --glob '*.md' -- projects 'invoice' | sort",
      "baseline_verdict": "wrong",
      "control": "docsearch {\"all\":[\"invoice\"],\"scope\":\"projects\",\"types\":[\"md\"]}",
      "control_verdict": "exact",
      "repair": "docsearch {\"all\":[\"invoice\"],\"scope\":\"projects\",\"types\":[\"md\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "regression / incident",
      "request": "I saved the Stripe credentials in markdown, can you locate those files?",
      "correct": "docsearch {\"all\":[\"Stripe\"],\"credential\":true,\"types\":[\"md\"]}",
      "baseline": "unsupported:not-a-file-search",
      "baseline_verdict": "wrong: refused",
      "control": "unsupported:not-a-file-search",
      "control_verdict": "wrong: refused",
      "repair": "docsearch {\"all\":[\"Stripe\"],\"credential\":true,\"types\":[\"md\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "regression / old_acceptance",
      "request": "Search team files for markdown with Flint travel itinerary, modified in the last 3 days, newest first, at most 5.",
      "correct": "docsearch {\"all\":[\"Flint travel itinerary\"],\"limit\":5,\"order\":\"newest\",\"scope\":\"team files\",\"time\":{\"field\":\"modified\",\"within_days\":3},\"types\":[\"md\"]}",
      "baseline": "docsearch {\"all\":[\"Flint travel itinerary\"],\"limit\":5,\"order\":\"newest\",\"scope\":\"team files\",\"time\":{\"field\":\"modified\",\"within_days\":3},\"types\":[\"md\"]}",
      "baseline_verdict": "exact",
      "control": "docsearch {\"all\":[\"Flint travel itinerary\"],\"limit\":5,\"order\":\"newest\",\"scope\":\"team files\",\"time\":{\"field\":\"modified\",\"within_days\":3},\"types\":[\"md\"]}",
      "control_verdict": "exact",
      "repair": "docsearch {\"all\":[\"Flint travel itinerary\"],\"limit\":5,\"order\":\"newest\",\"scope\":\"team files\",\"time\":{\"field\":\"modified\",\"within_days\":3},\"types\":[\"md\"]}",
      "repair_verdict": "exact"
    },
    {
      "group": "regression / old_acceptance",
      "request": "Under notes, find .env and markdown files with Tulip travel itinerary. Skip notes/archive.",
      "correct": "docsearch {\"all\":[\"Tulip travel itinerary\"],\"exclude_paths\":[\"notes/archive\"],\"scope\":\"notes\",\"types\":[\"env\",\"md\"]}",
      "baseline": "docsearch {\"all\":[\"Tulip travel itinerary\"],\"exclude_paths\":[\"notes/archive\"],\"scope\":\"notes\",\"types\":[\"env\",\"md\"]}",
      "baseline_verdict": "exact",
      "control": "docsearch {\"all\":[\"Tulip travel itinerary\"],\"exclude_paths\":[\"notes/archive\"],\"scope\":\"notes\",\"types\":[\"env\",\"md\"]}",
      "control_verdict": "exact",
      "repair": "docsearch {\"all\":[\"Tulip travel itinerary\"],\"exclude_paths\":[\"notes/archive\"],\"scope\":\"notes\",\"types\":[\"env\",\"md\"]}",
      "repair_verdict": "exact"
    }
  ],
  "data": {
    "intervention": {
      "jeremiah_v2_rows_capped_as_duplicates": [
        "j2-train-composition-0067",
        "j2-train-composition-0128",
        "j2-train-composition-0129",
        "j2-train-composition-0185",
        "j2-train-composition-0187",
        "j2-train-composition-0188",
        "j2-train-composition-0190",
        "j2-train-composition-0201",
        "j2-train-composition-0223",
        "j2-train-composition-0224",
        "j2-train-composition-0235",
        "j2-train-key_mention-0160",
        "j2-train-key_mention-0162",
        "j2-train-key_mention-0163",
        "j2-train-key_mention-0165",
        "j2-train-topic_alternatives-0137"
      ],
      "jeremiah_v2_train_rows": 891,
      "jeremiah_v2_validation_rows": 39,
      "retained_targeted_train_rows": 1509,
      "retained_targeted_validation_rows": 201
    },
    "language": {
      "authored": 1264,
      "accepted": 1163,
      "rejected": 101,
      "reasons": {
        "created_not_explicit": 5,
        "modified_not_explicit": 2,
        "review_disagreement": 94
      }
    },
    "reuse": {
      "backbone": {
        "rows": 9600,
        "reuse": 9590,
        "excluded": 10,
        "reasons": {
          "credential_true_without_stored_wording": 10
        }
      },
      "control_intervention": {
        "rows": 2400,
        "reuse": 2399,
        "excluded": 1,
        "reasons": {
          "credential_true_without_stored_wording": 1
        }
      },
      "incident": {
        "rows": 117,
        "reuse": 110,
        "excluded": 7,
        "reasons": {
          "clarification_for_named_document_category": 6,
          "held_calendar_clock_ambiguity": 1
        }
      },
      "selected": {
        "rows": 3240,
        "reuse": 3216,
        "excluded": 24,
        "reasons": {
          "clarification_for_named_document_category": 18,
          "credential_true_without_stored_wording": 5,
          "held_calendar_clock_ambiguity": 1
        }
      }
    },
    "gold_preflight": {
      "rows": 1264,
      "passed": 1264,
      "mutations": 13989
    },
    "gold_preflight_targeted_reuse": {
      "rows": 3216,
      "passed": 3201,
      "mutations": 20007
    }
  },
  "recipe": {
    "accumulation": 4,
    "arms": [
      "control",
      "repair"
    ],
    "backbone": 9600,
    "input_limit": 1536,
    "learning_rate": 1e-05,
    "max_grad_norm": 1.0,
    "max_length": 1792,
    "microbatch": 4,
    "minimum_recovery_gib": 60,
    "new_rows": 2400,
    "output_limit": 256,
    "rows": 12000,
    "sampler": "seeded_shuffle",
    "schedule": "cosine",
    "seed": 42,
    "snapshot_bytes": 3087467144,
    "snapshot_steps": [
      188,
      375,
      750
    ],
    "steps": 750,
    "useful_work_seconds": 7200,
    "warmup_steps": 10,
    "weight_decay": 0.1
  },
  "initialization_sha256": "1a398f8880b23a18da9e524ceb0136851ec2215eb6a066c25d425fdab5cd7169",
  "context_sha256": "529717c6555d64164e307e7e2bed79d8e40d1920541f6dca1b18921df1085b53",
  "manifest_sha256": "8f14960448d093f926396587d21d4a6cb796d8fae37493d72dde2b8c61ccd211",
  "checkpoints": [
    {
      "path": "control/checkpoint-188/model.safetensors",
      "sha256": "a1d5f1b6ae9f13c124c4283c23db699d1d8f8ea12b254a6aa202f6da0830c5c1",
      "bytes": 3087467144
    },
    {
      "path": "control/checkpoint-375/model.safetensors",
      "sha256": "d1891c360102e7c8212c69579a7dcf273c6feecf50e83191817c97d696dccd52",
      "bytes": 3087467144
    },
    {
      "path": "control/checkpoint-750/model.safetensors",
      "sha256": "d85220529b09a1c3936ce816fc30072322d44a24accf4670022bfc13ac077d2a",
      "bytes": 3087467144
    },
    {
      "path": "repair/checkpoint-188/model.safetensors",
      "sha256": "e7973c7e10c335756696af8a02bd6fa092e8ac98e7d62cca479ea0126ebe0e1b",
      "bytes": 3087467144
    },
    {
      "path": "repair/checkpoint-375/model.safetensors",
      "sha256": "96fb0ee229248be05cc87251616972cfabeaa7781b23580e9b0ac8860d1264fb",
      "bytes": 3087467144
    },
    {
      "path": "repair/checkpoint-750/model.safetensors",
      "sha256": "e0be850080d164f24173c0aac964da08e6d095fa67e8bbb84aa7659620feb006",
      "bytes": 3087467144
    }
  ],
  "attempts": [
    {
      "name": "shlex-live-ada-20260915-215903",
      "outcome": "controller preflight: current_Ada_offer_unavailable; no droplet created; data convention fix (D3) applied before relaunch",
      "estimated_compute_usd": 0
    },
    {
      "name": "shlex-live-ada-20260915-222250",
      "outcome": "trainer failed at dataset load: validation string column cast to null-typed train schema (control arm cluster/fixture_lineage all null); droplet deleted after 9 min, est. USD 0.50",
      "estimated_compute_usd": 0.4977
    },
    {
      "name": "shlex-live-ada-20260915-223835",
      "outcome": "completed: two arms, six snapshots recovered and hashed, droplet deleted",
      "estimated_compute_usd": 2.5868
    }
  ],
  "estimated_compute_usd": 2.5868,
  "gpu_deleted": true,
  "audit": {
    "completed": 28,
    "severities": {
      "high": 33,
      "info": 4,
      "low": 26,
      "medium": 58
    }
  },
  "loss_used_for_decisions": false,
  "training_rows_published": false,
  "weights_published": false,
  "generated_unix": 1789522436.50208
}