{
  "title": "Flow equivalence on generated SmarTest 7/8 programs",
  "source": "Origen-generated SmarTest 7/8 program pairs, scored against the SmarTest 7 source",
  "recorded": [
    "2026-09-30",
    "2026-10-08"
  ],
  "inputSha256": "5ee2b4edec9a1804f27a885e7290dfaa420f14e61cd480601f22c8b3ebf98625",
  "corpus": {
    "programs": 28,
    "scored": 26,
    "facts": 9889,
    "generator": "origen_testers"
  },
  "arms": [
    {
      "id": "direct",
      "label": "Direct port",
      "model": "sonnet",
      "programs": 26,
      "facts": 9888,
      "wrongFacts": 16,
      "silentErrors": 4,
      "programsWithSilentError": 3,
      "flowEquivalent": 20,
      "flowScenarios": 1770,
      "flowScenariosEqual": 1629,
      "trapsFlagged": 10,
      "trapsMustFlag": 10,
      "costUsd": 5.29
    },
    {
      "id": "llm_ir",
      "label": "Through the intent record",
      "model": "sonnet",
      "programs": 26,
      "facts": 9889,
      "wrongFacts": 0,
      "silentErrors": 0,
      "programsWithSilentError": 0,
      "flowEquivalent": 23,
      "flowScenarios": 1768,
      "flowScenariosEqual": 1702,
      "trapsFlagged": 10,
      "trapsMustFlag": 10,
      "costUsd": 5.49
    },
    {
      "id": "rules",
      "label": "Rules through the intent record",
      "model": "sonnet",
      "programs": 26,
      "facts": 9889,
      "wrongFacts": 0,
      "silentErrors": 0,
      "programsWithSilentError": 0,
      "flowEquivalent": 25,
      "flowScenarios": 1768,
      "flowScenariosEqual": 1721,
      "trapsFlagged": 10,
      "trapsMustFlag": 10,
      "costUsd": 0.02
    }
  ],
  "checker": {
    "caught": 846,
    "injected": 846,
    "faultClasses": 20,
    "readersAgreeFacts": 4661,
    "readersAgreePairs": 15,
    "branchesCovered": 1272,
    "branches": 1302
  },
  "generatorDefects": [
    {
      "what": "Low and high limits exchanged in the limits table",
      "where": "2 shipped programs, 14 of 14 comparable limits",
      "programs": [
        "ship_test",
        "ship_prb1"
      ]
    },
    {
      "what": "Testing continues after a fail bin",
      "where": "15 of 15 generated programs under documented SmarTest 8 binning"
    }
  ],
  "board": {
    "program": "gen_l3_01",
    "tests": 17,
    "inputFlags": 1,
    "columns": [
      "source",
      "generator",
      "port"
    ],
    "columnLabels": {
      "source": "Source \u00b7 SmarTest 7",
      "generator": "Generator's own SmarTest 8",
      "port": "Our port \u00b7 SmarTest 8"
    },
    "testList": [
      {
        "name": "isb",
        "number": "30010",
        "bin": [
          "111",
          "8"
        ]
      },
      {
        "name": "iddq_scan",
        "number": "30015",
        "bin": [
          "112",
          "5"
        ]
      },
      {
        "name": "dps_conn",
        "number": "30025",
        "bin": [
          "117",
          "6"
        ]
      },
      {
        "name": "out_dc",
        "number": "30026",
        "bin": [
          "118",
          "5"
        ]
      },
      {
        "name": "isb_1",
        "number": "30031",
        "bin": [
          "119",
          "7"
        ]
      },
      {
        "name": "vol_voh",
        "number": "30036",
        "bin": null
      },
      {
        "name": "dps_conn_1",
        "number": "30041",
        "bin": [
          "120",
          "12"
        ]
      },
      {
        "name": "iddq",
        "number": "30042",
        "bin": [
          "121",
          "9"
        ]
      },
      {
        "name": "icc",
        "number": "30052",
        "bin": [
          "123",
          "5"
        ]
      },
      {
        "name": "pmu",
        "number": "30062",
        "bin": null
      },
      {
        "name": "idd_sleep",
        "number": "30072",
        "bin": [
          "138",
          "11"
        ]
      },
      {
        "name": "ileak",
        "number": "30082",
        "bin": [
          "139",
          "9"
        ]
      },
      {
        "name": "ibias",
        "number": "30092",
        "bin": null
      },
      {
        "name": "ileak_1",
        "number": "30093",
        "bin": [
          "140",
          "9"
        ]
      },
      {
        "name": "contact",
        "number": "30103",
        "bin": [
          "145",
          "9"
        ]
      },
      {
        "name": "supply_status",
        "number": "30108",
        "bin": [
          "146",
          "6"
        ]
      },
      {
        "name": "dps_stat",
        "number": "30113",
        "bin": [
          "156",
          "8"
        ]
      }
    ],
    "rows": [
      {
        "fail": null,
        "label": "No failure",
        "source": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "1",
          "hard": "1"
        },
        "generator": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "1",
          "hard": ""
        },
        "port": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "1",
          "hard": "1"
        }
      },
      {
        "fail": 0,
        "label": "isb fails",
        "source": {
          "trace": [
            0
          ],
          "soft": "111",
          "hard": "8"
        },
        "generator": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "111",
          "hard": "8"
        },
        "port": {
          "trace": [
            0
          ],
          "soft": "111",
          "hard": "8"
        }
      },
      {
        "fail": 1,
        "label": "iddq_scan fails",
        "source": {
          "trace": [
            0,
            1
          ],
          "soft": "112",
          "hard": "5"
        },
        "generator": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "112",
          "hard": "5"
        },
        "port": {
          "trace": [
            0,
            1
          ],
          "soft": "112",
          "hard": "5"
        }
      },
      {
        "fail": 2,
        "label": "dps_conn fails",
        "source": {
          "trace": [
            0,
            1,
            2
          ],
          "soft": "117",
          "hard": "6"
        },
        "generator": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "117",
          "hard": "6"
        },
        "port": {
          "trace": [
            0,
            1,
            2
          ],
          "soft": "117",
          "hard": "6"
        }
      },
      {
        "fail": 3,
        "label": "out_dc fails",
        "source": {
          "trace": [
            0,
            1,
            2,
            3
          ],
          "soft": "118",
          "hard": "5"
        },
        "generator": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "118",
          "hard": "5"
        },
        "port": {
          "trace": [
            0,
            1,
            2,
            3
          ],
          "soft": "118",
          "hard": "5"
        }
      },
      {
        "fail": 4,
        "label": "isb_1 fails",
        "source": {
          "trace": [
            0,
            1,
            2,
            3,
            4
          ],
          "soft": "119",
          "hard": "7"
        },
        "generator": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "119",
          "hard": "7"
        },
        "port": {
          "trace": [
            0,
            1,
            2,
            3,
            4
          ],
          "soft": "119",
          "hard": "7"
        }
      },
      {
        "fail": 5,
        "label": "vol_voh fails",
        "source": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "1",
          "hard": "1"
        },
        "generator": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "1",
          "hard": ""
        },
        "port": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "1",
          "hard": "1"
        }
      },
      {
        "fail": 6,
        "label": "dps_conn_1 fails",
        "source": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6
          ],
          "soft": "120",
          "hard": "12"
        },
        "generator": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "120",
          "hard": "12"
        },
        "port": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6
          ],
          "soft": "120",
          "hard": "12"
        }
      },
      {
        "fail": 7,
        "label": "iddq fails",
        "source": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7
          ],
          "soft": "121",
          "hard": "9"
        },
        "generator": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "121",
          "hard": "9"
        },
        "port": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7
          ],
          "soft": "121",
          "hard": "9"
        }
      },
      {
        "fail": 8,
        "label": "icc fails",
        "source": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8
          ],
          "soft": "123",
          "hard": "5"
        },
        "generator": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "123",
          "hard": "5"
        },
        "port": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8
          ],
          "soft": "123",
          "hard": "5"
        }
      },
      {
        "fail": 9,
        "label": "pmu fails",
        "source": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            10,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "1",
          "hard": "1"
        },
        "generator": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            10,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "1",
          "hard": ""
        },
        "port": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            10,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "1",
          "hard": "1"
        }
      },
      {
        "fail": 10,
        "label": "idd_sleep fails",
        "source": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "1",
          "hard": "1"
        },
        "generator": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "1",
          "hard": ""
        },
        "port": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "1",
          "hard": "1"
        }
      },
      {
        "fail": 11,
        "label": "ileak fails",
        "source": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11
          ],
          "soft": "139",
          "hard": "9"
        },
        "generator": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "139",
          "hard": "9"
        },
        "port": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11
          ],
          "soft": "139",
          "hard": "9"
        }
      },
      {
        "fail": 12,
        "label": "ibias fails",
        "source": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "1",
          "hard": "1"
        },
        "generator": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "1",
          "hard": ""
        },
        "port": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "1",
          "hard": "1"
        }
      },
      {
        "fail": 13,
        "label": "ileak_1 fails",
        "source": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13
          ],
          "soft": "140",
          "hard": "9"
        },
        "generator": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "140",
          "hard": "9"
        },
        "port": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13
          ],
          "soft": "140",
          "hard": "9"
        }
      },
      {
        "fail": 14,
        "label": "contact fails",
        "source": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14
          ],
          "soft": "145",
          "hard": "9"
        },
        "generator": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "145",
          "hard": "9"
        },
        "port": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14
          ],
          "soft": "145",
          "hard": "9"
        }
      },
      {
        "fail": 15,
        "label": "supply_status fails",
        "source": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15
          ],
          "soft": "146",
          "hard": "6"
        },
        "generator": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "146",
          "hard": "6"
        },
        "port": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15
          ],
          "soft": "146",
          "hard": "6"
        }
      },
      {
        "fail": 16,
        "label": "dps_stat fails",
        "source": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "156",
          "hard": "8"
        },
        "generator": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "156",
          "hard": "8"
        },
        "port": {
          "trace": [
            0,
            1,
            2,
            3,
            4,
            5,
            6,
            7,
            8,
            9,
            11,
            12,
            13,
            14,
            15,
            16
          ],
          "soft": "156",
          "hard": "8"
        }
      }
    ],
    "differ": {
      "generator": {
        "cases": 296,
        "of": 308,
        "sameBin": 265,
        "extraTestsMax": 16
      },
      "port": {
        "cases": 0,
        "of": 308,
        "sameBin": 0,
        "extraTestsMax": 0
      }
    }
  }
}