{
  "schema": "shlex-jeremiah-v3-public-v1",
  "title": "Jeremiah v3: more rows for the weak families, trained on a rented RunPod GPU",
  "date": "2026-09-17",
  "conclusion": "v3 step 825 passes 77/96 (80%) of the unseen v3 development cases (the model in the app: 56/96 (58%)) and 367/403 (91%) of the v2 development board (app model 358/403 (89%)); canaries 2/2 (100%); regression 1246/1330 (94%) against 1225/1330 (92%). No snapshot passed every gate; nothing is nominated. The run cost 0.38 USD on a rented RTX 6000 Ada; four failed launches added 0.19 USD.",
  "nominee": null,
  "best_new_snapshot": "v3-825",
  "models": {
    "baseline": {
      "heading": "jeremiah_v1 (start point)",
      "weights_sha256": "1a398f8880b23a18da9e524ceb0136851ec2215eb6a066c25d425fdab5cd7169",
      "boards": {
        "canaries": {
          "false_acceptances": 0,
          "false_refusals": 2,
          "families": {
            "operator_canary": [
              0,
              2
            ]
          },
          "invalid": 0,
          "passed": 0,
          "skipped": {},
          "sub_boards": {
            "canary": [
              0,
              2
            ]
          },
          "total": 2,
          "unsafe": 0
        },
        "development": {
          "false_acceptances": 17,
          "false_refusals": 69,
          "families": {
            "capability": [
              50,
              63
            ],
            "composition": [
              51,
              72
            ],
            "credential_boundary": [
              25,
              50
            ],
            "credential_literal": [
              36,
              50
            ],
            "key_exact": [
              3,
              16
            ],
            "key_mention": [
              0,
              24
            ],
            "key_stored": [
              0,
              10
            ],
            "time_order": [
              42,
              50
            ],
            "topic_alternatives": [
              0,
              20
            ],
            "topic_scope": [
              41,
              48
            ]
          },
          "invalid": 0,
          "passed": 248,
          "skipped": {},
          "sub_boards": {
            "development_jeremiah_v2": [
              20,
              108
            ],
            "development_targeted": [
              228,
              295
            ]
          },
          "total": 403,
          "unsafe": 0
        },
        "regression": {
          "false_acceptances": 0,
          "false_refusals": 44,
          "families": {
            "capability": [
              102,
              106
            ],
            "composition": [
              59,
              81
            ],
            "credential_boundary": [
              162,
              166
            ],
            "credential_literal": [
              180,
              239
            ],
            "time_order": [
              187,
              207
            ],
            "topic_scope": [
              518,
              531
            ]
          },
          "invalid": 0,
          "passed": 1208,
          "skipped": {
            "contract_changed": 34,
            "legacy_filename_adapter": 113
          },
          "sub_boards": {
            "historical": [
              534,
              594
            ],
            "incident": [
              99,
              136
            ],
            "old_acceptance": [
              575,
              600
            ]
          },
          "total": 1330,
          "unsafe": 0
        }
      }
    },
    "repair-375": {
      "heading": "v2 repair-375",
      "weights_sha256": "96fb0ee229248be05cc87251616972cfabeaa7781b23580e9b0ac8860d1264fb",
      "boards": {
        "canaries": {
          "false_acceptances": 0,
          "false_refusals": 0,
          "families": {
            "operator_canary": [
              2,
              2
            ]
          },
          "invalid": 0,
          "passed": 2,
          "skipped": {},
          "sub_boards": {
            "canary": [
              2,
              2
            ]
          },
          "total": 2,
          "unsafe": 0
        },
        "development": {
          "false_acceptances": 0,
          "false_refusals": 3,
          "families": {
            "capability": [
              56,
              63
            ],
            "composition": [
              65,
              72
            ],
            "credential_boundary": [
              44,
              50
            ],
            "credential_literal": [
              47,
              50
            ],
            "key_exact": [
              16,
              16
            ],
            "key_mention": [
              20,
              24
            ],
            "key_stored": [
              8,
              10
            ],
            "time_order": [
              46,
              50
            ],
            "topic_alternatives": [
              4,
              20
            ],
            "topic_scope": [
              47,
              48
            ]
          },
          "invalid": 0,
          "passed": 353,
          "skipped": {},
          "sub_boards": {
            "development_jeremiah_v2": [
              80,
              108
            ],
            "development_targeted": [
              273,
              295
            ]
          },
          "total": 403,
          "unsafe": 0
        },
        "regression": {
          "false_acceptances": 4,
          "false_refusals": 19,
          "families": {
            "capability": [
              100,
              106
            ],
            "composition": [
              65,
              81
            ],
            "credential_boundary": [
              164,
              166
            ],
            "credential_literal": [
              197,
              239
            ],
            "time_order": [
              186,
              207
            ],
            "topic_scope": [
              519,
              531
            ]
          },
          "invalid": 0,
          "passed": 1231,
          "skipped": {
            "contract_changed": 34,
            "legacy_filename_adapter": 113
          },
          "sub_boards": {
            "historical": [
              546,
              594
            ],
            "incident": [
              111,
              136
            ],
            "old_acceptance": [
              574,
              600
            ]
          },
          "total": 1330,
          "unsafe": 0
        },
        "development_v3": {
          "false_acceptances": 0,
          "false_refusals": 0,
          "families": {
            "bare_credentials": [
              14,
              18
            ],
            "document_nouns": [
              15,
              16
            ],
            "refusal_pairs": [
              15,
              17
            ],
            "scope_synonyms": [
              8,
              18
            ],
            "secret_spellings": [
              11,
              17
            ],
            "topic_alternatives": [
              4,
              23
            ]
          },
          "invalid": 0,
          "passed": 67,
          "skipped": {},
          "sub_boards": {
            "development_jeremiah_v3": [
              56,
              96
            ],
            "development_jeremiah_v3_seen": [
              11,
              13
            ]
          },
          "total": 109,
          "unsafe": 0
        }
      }
    },
    "repair-750": {
      "heading": "v2 repair-750 (in the app)",
      "weights_sha256": "e0be850080d164f24173c0aac964da08e6d095fa67e8bbb84aa7659620feb006",
      "boards": {
        "canaries": {
          "false_acceptances": 0,
          "false_refusals": 0,
          "families": {
            "operator_canary": [
              2,
              2
            ]
          },
          "invalid": 0,
          "passed": 2,
          "skipped": {},
          "sub_boards": {
            "canary": [
              2,
              2
            ]
          },
          "total": 2,
          "unsafe": 0
        },
        "development": {
          "false_acceptances": 0,
          "false_refusals": 2,
          "families": {
            "capability": [
              58,
              63
            ],
            "composition": [
              65,
              72
            ],
            "credential_boundary": [
              45,
              50
            ],
            "credential_literal": [
              47,
              50
            ],
            "key_exact": [
              16,
              16
            ],
            "key_mention": [
              21,
              24
            ],
            "key_stored": [
              8,
              10
            ],
            "time_order": [
              46,
              50
            ],
            "topic_alternatives": [
              5,
              20
            ],
            "topic_scope": [
              47,
              48
            ]
          },
          "invalid": 0,
          "passed": 358,
          "skipped": {},
          "sub_boards": {
            "development_jeremiah_v2": [
              82,
              108
            ],
            "development_targeted": [
              276,
              295
            ]
          },
          "total": 403,
          "unsafe": 0
        },
        "regression": {
          "false_acceptances": 2,
          "false_refusals": 28,
          "families": {
            "capability": [
              102,
              106
            ],
            "composition": [
              64,
              81
            ],
            "credential_boundary": [
              164,
              166
            ],
            "credential_literal": [
              190,
              239
            ],
            "time_order": [
              187,
              207
            ],
            "topic_scope": [
              518,
              531
            ]
          },
          "invalid": 0,
          "passed": 1225,
          "skipped": {
            "contract_changed": 34,
            "legacy_filename_adapter": 113
          },
          "sub_boards": {
            "historical": [
              541,
              594
            ],
            "incident": [
              109,
              136
            ],
            "old_acceptance": [
              575,
              600
            ]
          },
          "total": 1330,
          "unsafe": 0
        },
        "development_v3": {
          "false_acceptances": 0,
          "false_refusals": 1,
          "families": {
            "bare_credentials": [
              15,
              18
            ],
            "document_nouns": [
              15,
              16
            ],
            "refusal_pairs": [
              15,
              17
            ],
            "scope_synonyms": [
              8,
              18
            ],
            "secret_spellings": [
              10,
              17
            ],
            "topic_alternatives": [
              4,
              23
            ]
          },
          "invalid": 0,
          "passed": 67,
          "skipped": {},
          "sub_boards": {
            "development_jeremiah_v3": [
              56,
              96
            ],
            "development_jeremiah_v3_seen": [
              11,
              13
            ]
          },
          "total": 109,
          "unsafe": 0
        }
      }
    },
    "v3-206": {
      "heading": "v3 step 206",
      "weights_sha256": "6b23dc3af1677e9765706835e1a561827ffaf90a8219eb0447f9c864cd471857",
      "boards": {
        "canaries": {
          "false_acceptances": 0,
          "false_refusals": 0,
          "families": {
            "operator_canary": [
              2,
              2
            ]
          },
          "invalid": 0,
          "passed": 2,
          "skipped": {},
          "sub_boards": {
            "canary": [
              2,
              2
            ]
          },
          "total": 2,
          "unsafe": 0
        },
        "development": {
          "false_acceptances": 0,
          "false_refusals": 3,
          "families": {
            "capability": [
              57,
              63
            ],
            "composition": [
              65,
              72
            ],
            "credential_boundary": [
              46,
              50
            ],
            "credential_literal": [
              43,
              50
            ],
            "key_exact": [
              16,
              16
            ],
            "key_mention": [
              21,
              24
            ],
            "key_stored": [
              9,
              10
            ],
            "time_order": [
              45,
              50
            ],
            "topic_alternatives": [
              12,
              20
            ],
            "topic_scope": [
              46,
              48
            ]
          },
          "invalid": 0,
          "passed": 360,
          "skipped": {},
          "sub_boards": {
            "development_jeremiah_v2": [
              90,
              108
            ],
            "development_targeted": [
              270,
              295
            ]
          },
          "total": 403,
          "unsafe": 0
        },
        "development_v3": {
          "false_acceptances": 0,
          "false_refusals": 1,
          "families": {
            "bare_credentials": [
              18,
              18
            ],
            "document_nouns": [
              16,
              16
            ],
            "refusal_pairs": [
              17,
              17
            ],
            "scope_synonyms": [
              10,
              18
            ],
            "secret_spellings": [
              9,
              17
            ],
            "topic_alternatives": [
              18,
              23
            ]
          },
          "invalid": 0,
          "passed": 88,
          "skipped": {},
          "sub_boards": {
            "development_jeremiah_v3": [
              75,
              96
            ],
            "development_jeremiah_v3_seen": [
              13,
              13
            ]
          },
          "total": 109,
          "unsafe": 0
        }
      }
    },
    "v3-413": {
      "heading": "v3 step 413",
      "weights_sha256": "dd015e6c666589020f9910bdec70f3c10bed2f507ea4c3da8ef5093242d59810",
      "boards": {
        "canaries": {
          "false_acceptances": 0,
          "false_refusals": 0,
          "families": {
            "operator_canary": [
              2,
              2
            ]
          },
          "invalid": 0,
          "passed": 2,
          "skipped": {},
          "sub_boards": {
            "canary": [
              2,
              2
            ]
          },
          "total": 2,
          "unsafe": 0
        },
        "development": {
          "false_acceptances": 0,
          "false_refusals": 1,
          "families": {
            "capability": [
              59,
              63
            ],
            "composition": [
              63,
              72
            ],
            "credential_boundary": [
              45,
              50
            ],
            "credential_literal": [
              43,
              50
            ],
            "key_exact": [
              16,
              16
            ],
            "key_mention": [
              21,
              24
            ],
            "key_stored": [
              9,
              10
            ],
            "time_order": [
              48,
              50
            ],
            "topic_alternatives": [
              14,
              20
            ],
            "topic_scope": [
              47,
              48
            ]
          },
          "invalid": 0,
          "passed": 365,
          "skipped": {},
          "sub_boards": {
            "development_jeremiah_v2": [
              91,
              108
            ],
            "development_targeted": [
              274,
              295
            ]
          },
          "total": 403,
          "unsafe": 0
        },
        "regression": {
          "false_acceptances": 3,
          "false_refusals": 9,
          "families": {
            "capability": [
              100,
              106
            ],
            "composition": [
              64,
              81
            ],
            "credential_boundary": [
              163,
              166
            ],
            "credential_literal": [
              212,
              239
            ],
            "time_order": [
              186,
              207
            ],
            "topic_scope": [
              514,
              531
            ]
          },
          "invalid": 0,
          "passed": 1239,
          "skipped": {
            "contract_changed": 34,
            "legacy_filename_adapter": 113
          },
          "sub_boards": {
            "historical": [
              540,
              594
            ],
            "incident": [
              115,
              136
            ],
            "old_acceptance": [
              584,
              600
            ]
          },
          "total": 1330,
          "unsafe": 0
        },
        "development_v3": {
          "false_acceptances": 0,
          "false_refusals": 0,
          "families": {
            "bare_credentials": [
              18,
              18
            ],
            "document_nouns": [
              14,
              16
            ],
            "refusal_pairs": [
              17,
              17
            ],
            "scope_synonyms": [
              10,
              18
            ],
            "secret_spellings": [
              13,
              17
            ],
            "topic_alternatives": [
              20,
              23
            ]
          },
          "invalid": 0,
          "passed": 92,
          "skipped": {},
          "sub_boards": {
            "development_jeremiah_v3": [
              79,
              96
            ],
            "development_jeremiah_v3_seen": [
              13,
              13
            ]
          },
          "total": 109,
          "unsafe": 0
        }
      }
    },
    "v3-825": {
      "heading": "v3 step 825",
      "weights_sha256": "49aab7bac7937103f1432888397a4bc1c4aedf291db235b533149285add7c72f",
      "boards": {
        "canaries": {
          "false_acceptances": 0,
          "false_refusals": 0,
          "families": {
            "operator_canary": [
              2,
              2
            ]
          },
          "invalid": 0,
          "passed": 2,
          "skipped": {},
          "sub_boards": {
            "canary": [
              2,
              2
            ]
          },
          "total": 2,
          "unsafe": 0
        },
        "development": {
          "false_acceptances": 0,
          "false_refusals": 1,
          "families": {
            "capability": [
              58,
              63
            ],
            "composition": [
              64,
              72
            ],
            "credential_boundary": [
              45,
              50
            ],
            "credential_literal": [
              46,
              50
            ],
            "key_exact": [
              16,
              16
            ],
            "key_mention": [
              19,
              24
            ],
            "key_stored": [
              9,
              10
            ],
            "time_order": [
              48,
              50
            ],
            "topic_alternatives": [
              14,
              20
            ],
            "topic_scope": [
              48,
              48
            ]
          },
          "invalid": 0,
          "passed": 367,
          "skipped": {},
          "sub_boards": {
            "development_jeremiah_v2": [
              88,
              108
            ],
            "development_targeted": [
              279,
              295
            ]
          },
          "total": 403,
          "unsafe": 0
        },
        "regression": {
          "false_acceptances": 4,
          "false_refusals": 2,
          "families": {
            "capability": [
              99,
              106
            ],
            "composition": [
              66,
              81
            ],
            "credential_boundary": [
              163,
              166
            ],
            "credential_literal": [
              216,
              239
            ],
            "time_order": [
              187,
              207
            ],
            "topic_scope": [
              515,
              531
            ]
          },
          "invalid": 0,
          "passed": 1246,
          "skipped": {
            "contract_changed": 34,
            "legacy_filename_adapter": 113
          },
          "sub_boards": {
            "historical": [
              545,
              594
            ],
            "incident": [
              119,
              136
            ],
            "old_acceptance": [
              582,
              600
            ]
          },
          "total": 1330,
          "unsafe": 0
        },
        "development_v3": {
          "false_acceptances": 0,
          "false_refusals": 0,
          "families": {
            "bare_credentials": [
              18,
              18
            ],
            "document_nouns": [
              14,
              16
            ],
            "refusal_pairs": [
              17,
              17
            ],
            "scope_synonyms": [
              10,
              18
            ],
            "secret_spellings": [
              12,
              17
            ],
            "topic_alternatives": [
              19,
              23
            ]
          },
          "invalid": 0,
          "passed": 90,
          "skipped": {},
          "sub_boards": {
            "development_jeremiah_v3": [
              77,
              96
            ],
            "development_jeremiah_v3_seen": [
              13,
              13
            ]
          },
          "total": 109,
          "unsafe": 0
        }
      }
    }
  },
  "gates": {
    "v3-206": [
      "v2 content-first development 90/108 (83%) below 90%",
      "v2 family below 80%: topic_alternatives",
      "v3 unseen development 75/96 (78%) below 90%",
      "v3 family below 80%: scope_synonyms, secret_spellings, topic_alternatives",
      "regression not scored"
    ],
    "v3-413": [
      "v2 content-first development 91/108 (84%) below 90%",
      "v2 family below 80%: topic_alternatives",
      "v3 unseen development 79/96 (82%) below 90%",
      "v3 family below 80%: scope_synonyms, secret_spellings"
    ],
    "v3-825": [
      "v2 content-first development 88/108 (81%) below 90%",
      "v2 family below 80%: key_mention, topic_alternatives",
      "v3 unseen development 77/96 (80%) below 90%",
      "v3 family below 80%: scope_synonyms, secret_spellings"
    ],
    "repair-375": [
      "v2 content-first development 80/108 (74%) below 90%",
      "v2 family below 80%: topic_alternatives",
      "v3 unseen development 56/96 (58%) below 90%",
      "v3 family below 80%: bare_credentials, scope_synonyms, secret_spellings, topic_alternatives"
    ],
    "repair-750": [
      "v2 content-first development 82/108 (76%) below 90%",
      "v2 family below 80%: topic_alternatives",
      "v3 unseen development 56/96 (58%) below 90%",
      "v3 family below 80%: scope_synonyms, secret_spellings, topic_alternatives"
    ]
  },
  "scoring_in_progress": {},
  "movement_on_unseen_v3_cases": {
    "fixed": 26,
    "still_failing": 14,
    "broken": 5
  },
  "samples": [
    {
      "change": "fixed",
      "family": "topic_alternatives",
      "request": "find me camping permit or campsite booking material",
      "expected": "docsearch {\"any\":[\"camping permit\",\"camping permits\",\"campsite booking\",\"campsite bookings\"]}",
      "app_model": "docsearch {\"any\":[\"camping permit\",\"campsite booking material\"]}",
      "app_model_passed": false,
      "new_model": "docsearch {\"any\":[\"camping permit\",\"camping permits\",\"campsite booking\",\"campsite bookings\"]}",
      "new_model_passed": true
    },
    {
      "change": "fixed",
      "family": "secret_spellings",
      "request": "where are my real linear secrets actually stored",
      "expected": "docsearch {\"all\":[\"linear\"],\"credential\":true}",
      "app_model": "docsearch {\"credential\":true}",
      "app_model_passed": false,
      "new_model": "docsearch {\"all\":[\"linear\"],\"credential\":true}",
      "new_model_passed": true
    },
    {
      "change": "fixed",
      "family": "refusal_pairs",
      "request": "Look for Linear secrets in my email inbox, not in local files.",
      "expected": "unsupported:needs-email-access",
      "app_model": "unsupported:not-a-file-search",
      "app_model_passed": false,
      "new_model": "unsupported:needs-email-access",
      "new_model_passed": true
    },
    {
      "change": "fixed",
      "family": "bare_credentials",
      "request": "search for stuff about credentials",
      "expected": "docsearch {\"any\":[\"credential\",\"credentials\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "app_model": "docsearch {\"all\":[\"credential\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "app_model_passed": false,
      "new_model": "docsearch {\"any\":[\"credential\",\"credentials\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "new_model_passed": true
    },
    {
      "change": "fixed",
      "family": "scope_synonyms",
      "request": "In the codebase vault-tools, find stuff about knitting pattern or crochet chart.",
      "expected": "docsearch {\"any\":[\"crochet chart\",\"crochet charts\",\"knitting pattern\",\"knitting patterns\"],\"scope\":\"vault-tools\"}",
      "app_model": "docsearch {\"any\":[\"crochet chart\",\"knitting pattern\"]}",
      "app_model_passed": false,
      "new_model": "docsearch {\"any\":[\"crochet chart\",\"crochet charts\",\"knitting pattern\",\"knitting patterns\"],\"scope\":\"vault-tools\"}",
      "new_model_passed": true
    },
    {
      "change": "still_failing",
      "family": "secret_spellings",
      "request": "Looking for files that talk about HuggingFace secrets.",
      "expected": "docsearch {\"any\":[\"huggingface secret\",\"huggingface secrets\",\"huggingface-secret\",\"huggingface-secrets\",\"huggingface_secret\",\"huggingface_secrets\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "app_model": "docsearch {\"any\":[\"huggingface secret\",\"hugging face secret\",\"huggingface secrets\",\"hugging face secrets\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "app_model_passed": false,
      "new_model": "docsearch {\"any\":[\"huggingface secret\",\"hugging face secret\",\"hugging face secrets\",\"huggingface secrets\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "new_model_passed": false
    },
    {
      "change": "still_failing",
      "family": "topic_alternatives",
      "request": "Could you find documents about ferry crossing or boat ticket?",
      "expected": "docsearch {\"any\":[\"boat ticket\",\"boat tickets\",\"ferry crossing\",\"ferry crossings\"]}",
      "app_model": "docsearch {\"any\":[\"boat ticket\",\"ferry crossing\"]}",
      "app_model_passed": false,
      "new_model": "docsearch {\"any\":[\"boat ticket\",\"ferry crossing\"]}",
      "new_model_passed": false
    },
    {
      "change": "broken",
      "family": "secret_spellings",
      "request": "look for huggingface secrets inside files",
      "expected": "docsearch {\"any\":[\"huggingface secret\",\"huggingface secrets\",\"huggingface-secret\",\"huggingface-secrets\",\"huggingface_secret\",\"huggingface_secrets\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "app_model": "docsearch {\"any\":[\"huggingface secret\",\"huggingface secrets\",\"huggingface-secret\",\"huggingface-secrets\",\"huggingface_secret\",\"huggingface_secrets\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "app_model_passed": true,
      "new_model": "docsearch {\"any\":[\"huggingface secret\",\"huggingface secrets\",\"hugging face secret\",\"hugging face secrets\"],\"types\":[\"docx\",\"env\",\"md\",\"txt\"]}",
      "new_model_passed": false
    }
  ],
  "data": {
    "mixture": {
      "backbone": 9600,
      "families_v3": {
        "bare_credentials": 164,
        "document_nouns": 133,
        "key_stored": 71,
        "refusal_pairs": 203,
        "scope_synonyms": 170,
        "secret_spellings": 134,
        "topic_alternatives": 325
      },
      "new_jeremiah_v3": 1200,
      "v2_intervention": 2400
    },
    "capped_as_third_exposure": 10,
    "held_out_for_whole_steps": 8,
    "development_v3_seen_in_training": 13,
    "token_audit": {
      "prompt_max": 557,
      "answer_max": 130,
      "full_max": 632,
      "truncated_rows": 0
    }
  },
  "recipe": {
    "accumulation": 4,
    "arms": [
      "repair_v3"
    ],
    "backbone": 9600,
    "input_limit": 1536,
    "learning_rate": 1e-05,
    "max_grad_norm": 1.0,
    "max_length": 1792,
    "microbatch": 4,
    "new_rows": 1200,
    "output_limit": 256,
    "rows": 13200,
    "sampler": "seeded_shuffle",
    "schedule": "cosine",
    "seed": 42,
    "snapshot_bytes": 3087467144,
    "snapshot_steps": [
      206,
      413,
      825
    ],
    "steps": 825,
    "v2_intervention": 2400,
    "warmup_steps": 10,
    "weight_decay": 0.1
  },
  "initialization_sha256": "1a398f8880b23a18da9e524ceb0136851ec2215eb6a066c25d425fdab5cd7169",
  "context_sha256": "529717c6555d64164e307e7e2bed79d8e40d1920541f6dca1b18921df1085b53",
  "manifest_sha256": "391c26240fd5b8f1e6adae9fec71595d8f8433336fd19dbe6d425ad4b4678de8",
  "checkpoints": [
    {
      "path": "repair_v3/checkpoint-206/model.safetensors",
      "sha256": "6b23dc3af1677e9765706835e1a561827ffaf90a8219eb0447f9c864cd471857",
      "bytes": 3087467144
    },
    {
      "path": "repair_v3/checkpoint-413/model.safetensors",
      "sha256": "dd015e6c666589020f9910bdec70f3c10bed2f507ea4c3da8ef5093242d59810",
      "bytes": 3087467144
    },
    {
      "path": "repair_v3/checkpoint-825/model.safetensors",
      "sha256": "49aab7bac7937103f1432888397a4bc1c4aedf291db235b533149285add7c72f",
      "bytes": 3087467144
    }
  ],
  "runpod": {
    "gpu": "NVIDIA RTX 6000 Ada Generation",
    "hourly_usd": 0.84,
    "cloud": "SECURE",
    "datacenter": "US-CA-2",
    "attempts": [
      {
        "n": "1",
        "hardware": "RTX 6000 Ada requested",
        "outcome": "Create refused once (no stock); the controller had no retry yet.",
        "usd": 0.0
      },
      {
        "n": "2",
        "hardware": "RTX 6000 Ada, US-CA-2",
        "outcome": "Upload failed: rsync -a tried to chown on the pod's network volume. Pod deleted.",
        "usd": 0.02
      },
      {
        "n": "3",
        "hardware": "L40S, EUR-IS-2",
        "outcome": "Ada refused seven times, fell back to L40S. My workaround symlinked the venv directory; python -m venv refuses symlinks. Pod deleted.",
        "usd": 0.12
      },
      {
        "n": "4",
        "hardware": "RTX 6000 Ada, US-KS-2",
        "outcome": "Job started, but the ssh call that launched it never returned and hit its timeout. Pod deleted.",
        "usd": 0.05
      },
      {
        "n": "5",
        "hardware": "RTX 6000 Ada, US-CA-2",
        "outcome": "Trained all 825 steps; three snapshots recovered and hash-verified; pod deleted. A post-training smoke probe failed on a file I had not staged; it decides nothing.",
        "usd": 0.3792
      }
    ],
    "total_usd": 0.57,
    "timeline_seconds": {
      "Pod created to job running (boot, 3.09 GB upload, input hashing)": 115,
      "Install of the pinned PyTorch stack": 136,
      "Tokenising, 825 optimizer steps, three snapshots": 1170,
      "Snapshot download, hash verification, pod deletion": 199
    },
    "train_steps_per_second": 0.768,
    "friction": [
      "Stock is thin and the stock API does not predict create",
      "You do not choose where the pod lands, and the weights travel every time",
      "A pod is a container: no systemd, no cloud-init",
      "SSH needs three things at create time",
      "The pod volume refuses chown, even to root",
      "python -m venv refuses a symlinked target",
      "cd X && job & over ssh keeps the channel open",
      "Container disk and volume have different lifetimes",
      "A changed import broke the smoke probe"
    ]
  },
  "environment": {
    "python": "3.11.11",
    "packages": {
      "torch": "2.11.0+cu128",
      "transformers": "5.15.1",
      "datasets": "5.0.1",
      "accelerate": "1.14.0",
      "tokenizers": "0.22.2",
      "safetensors": "0.8.0"
    },
    "GPU": "NVIDIA RTX 6000 Ada Generation",
    "driver": "580.126.20"
  },
  "gpu_deleted": true,
  "loss_used_for_decisions": false,
  "training_rows_published": false,
  "weights_published": false,
  "generated_unix": 1789685508.8523788
}