{
  "extractor": {
    "regionsPerPage": {
      "median": 4,
      "p95": 12,
      "max": 12
    },
    "regionKinds": {
      "section": 594,
      "form": 116,
      "nav": 148,
      "header": 178,
      "page": 72,
      "footer": 139,
      "other": 47,
      "main": 31,
      "aside": 28,
      "overlay": 4,
      "dialog": 5
    },
    "truthRegionFoundRate": 0.9815950920245399,
    "truthRegionKinds": {
      "section": 51,
      "form": 41,
      "header": 22,
      "nav": 17,
      "footer": 5,
      "aside": 5,
      "main": 11,
      "other": 3,
      "page": 5,
      "overlay": 1,
      "dialog": 2
    },
    "truthRegionSize": {
      "median": 4,
      "p95": 44
    },
    "decisionPoints": 252,
    "steps": 164,
    "journeys": 88,
    "truthStatus": {
      "ok": 163,
      "cut-off": 1
    },
    "truthMatchVia": {
      "self": 159,
      "label-for": 2,
      "ancestor": 3
    },
    "truthNotInCandidatesRate": 0,
    "truthCutOffRate": 0.006097560975609756,
    "candidatesPerPage": {
      "median": 21.5,
      "p95": 150,
      "max": 150
    },
    "candidatesBeforeCap": {
      "median": 21.5,
      "p95": 180.34999999999997,
      "max": 388
    },
    "stateTokens": {
      "median": 522.5,
      "p95": 6456.45,
      "max": 6820
    },
    "perSite": {
      "demo-app": {
        "decisionPoints": 29,
        "steps": 19,
        "journeys": 10,
        "truthOk": 19,
        "notInCandidates": 0,
        "cutOff": 0
      },
      "playwright-docs": {
        "decisionPoints": 13,
        "steps": 7,
        "journeys": 6,
        "truthOk": 6,
        "notInCandidates": 0,
        "cutOff": 1
      },
      "todomvc": {
        "decisionPoints": 23,
        "steps": 17,
        "journeys": 6,
        "truthOk": 17,
        "notInCandidates": 0,
        "cutOff": 0
      },
      "quotes-toscrape": {
        "decisionPoints": 12,
        "steps": 7,
        "journeys": 5,
        "truthOk": 7,
        "notInCandidates": 0,
        "cutOff": 0
      },
      "the-internet": {
        "decisionPoints": 32,
        "steps": 19,
        "journeys": 13,
        "truthOk": 19,
        "notInCandidates": 0,
        "cutOff": 0
      },
      "saucedemo": {
        "decisionPoints": 38,
        "steps": 28,
        "journeys": 10,
        "truthOk": 28,
        "notInCandidates": 0,
        "cutOff": 0
      },
      "books-toscrape": {
        "decisionPoints": 14,
        "steps": 8,
        "journeys": 6,
        "truthOk": 8,
        "notInCandidates": 0,
        "cutOff": 0
      },
      "client-site": {
        "decisionPoints": 54,
        "steps": 36,
        "journeys": 18,
        "truthOk": 36,
        "notInCandidates": 0,
        "cutOff": 0
      },
      "automationexercise": {
        "decisionPoints": 17,
        "steps": 11,
        "journeys": 6,
        "truthOk": 11,
        "notInCandidates": 0,
        "cutOff": 0
      },
      "demoqa": {
        "decisionPoints": 20,
        "steps": 12,
        "journeys": 8,
        "truthOk": 12,
        "notInCandidates": 0,
        "cutOff": 0
      }
    }
  },
  "backends": {
    "jev-flat": {
      "answered": 252,
      "errors": 0,
      "steps": 164,
      "stepsTruthInCandidates": 163,
      "top1": 0.9695121951219512,
      "top1InCandidates": 0.9754601226993865,
      "top3InCandidates": 1,
      "top3N": 151,
      "perSite": {
        "demo-app": {
          "steps": 19,
          "correct": 19,
          "inCandidates": 19,
          "correctInCandidates": 19,
          "top1": 1,
          "top1InCandidates": 1
        },
        "playwright-docs": {
          "steps": 7,
          "correct": 6,
          "inCandidates": 6,
          "correctInCandidates": 6,
          "top1": 0.8571428571428571,
          "top1InCandidates": 1
        },
        "todomvc": {
          "steps": 17,
          "correct": 17,
          "inCandidates": 17,
          "correctInCandidates": 17,
          "top1": 1,
          "top1InCandidates": 1
        },
        "quotes-toscrape": {
          "steps": 7,
          "correct": 7,
          "inCandidates": 7,
          "correctInCandidates": 7,
          "top1": 1,
          "top1InCandidates": 1
        },
        "the-internet": {
          "steps": 19,
          "correct": 19,
          "inCandidates": 19,
          "correctInCandidates": 19,
          "top1": 1,
          "top1InCandidates": 1
        },
        "saucedemo": {
          "steps": 28,
          "correct": 28,
          "inCandidates": 28,
          "correctInCandidates": 28,
          "top1": 1,
          "top1InCandidates": 1
        },
        "books-toscrape": {
          "steps": 8,
          "correct": 8,
          "inCandidates": 8,
          "correctInCandidates": 8,
          "top1": 1,
          "top1InCandidates": 1
        },
        "client-site": {
          "steps": 36,
          "correct": 34,
          "inCandidates": 36,
          "correctInCandidates": 34,
          "top1": 0.9444444444444444,
          "top1InCandidates": 0.9444444444444444
        },
        "automationexercise": {
          "steps": 11,
          "correct": 11,
          "inCandidates": 11,
          "correctInCandidates": 11,
          "top1": 1,
          "top1InCandidates": 1
        },
        "demoqa": {
          "steps": 12,
          "correct": 10,
          "inCandidates": 12,
          "correctInCandidates": 10,
          "top1": 0.8333333333333334,
          "top1InCandidates": 0.8333333333333334
        }
      },
      "actionAccuracy": 0.9817073170731707,
      "actionN": 164,
      "goalMetAccuracy": 0.9206349206349206,
      "goalMetFalsePositiveRate": 0,
      "goalMetTruePositiveRate": 0.7727272727272727,
      "goalMetN": 252,
      "calibration": {
        "bins": [
          {
            "range": "0.0-0.2",
            "n": 0,
            "correct": 0,
            "confSum": 0
          },
          {
            "range": "0.2-0.4",
            "n": 0,
            "correct": 0,
            "confSum": 0
          },
          {
            "range": "0.4-0.6",
            "n": 4,
            "correct": 3,
            "confSum": 2.18,
            "accuracy": 0.75,
            "avgConfidence": 0.545
          },
          {
            "range": "0.6-0.8",
            "n": 7,
            "correct": 6,
            "confSum": 4.86,
            "accuracy": 0.8571428571428571,
            "avgConfidence": 0.6942857142857143
          },
          {
            "range": "0.8-1.0",
            "n": 152,
            "correct": 150,
            "confSum": 148.80000000000004,
            "accuracy": 0.9868421052631579,
            "avgConfidence": 0.9789473684210529
          }
        ],
        "ece": 0.01938650306748437,
        "n": 163
      },
      "escalation": [
        {
          "threshold": 0.5,
          "accuracy": 0.9754601226993865,
          "escalatedFraction": 0.006134969325153374,
          "n": 163
        },
        {
          "threshold": 0.55,
          "accuracy": 0.9754601226993865,
          "escalatedFraction": 0.006134969325153374,
          "n": 163
        },
        {
          "threshold": 0.6,
          "accuracy": 0.9815950920245399,
          "escalatedFraction": 0.024539877300613498,
          "n": 163
        },
        {
          "threshold": 0.65,
          "accuracy": 0.9815950920245399,
          "escalatedFraction": 0.03067484662576687,
          "n": 163
        },
        {
          "threshold": 0.7,
          "accuracy": 0.9815950920245399,
          "escalatedFraction": 0.049079754601226995,
          "n": 163
        },
        {
          "threshold": 0.75,
          "accuracy": 0.9815950920245399,
          "escalatedFraction": 0.06134969325153374,
          "n": 163
        },
        {
          "threshold": 0.8,
          "accuracy": 0.9815950920245399,
          "escalatedFraction": 0.06748466257668712,
          "n": 163
        },
        {
          "threshold": 0.85,
          "accuracy": 0.9815950920245399,
          "escalatedFraction": 0.09202453987730061,
          "n": 163
        },
        {
          "threshold": 0.9,
          "accuracy": 0.9815950920245399,
          "escalatedFraction": 0.13496932515337423,
          "n": 163
        },
        {
          "threshold": 0.95,
          "accuracy": 0.9754601226993865,
          "escalatedFraction": 0.1901840490797546,
          "n": 163
        }
      ],
      "latencyMs": {
        "median": 349.5,
        "p95": 524,
        "n": 252
      },
      "latencyParts": [
        null,
        null
      ],
      "roundTrips": 1,
      "requestBytes": {
        "median": 5966,
        "p95": 55179.35,
        "max": 58071
      },
      "regionAccuracy": null,
      "regionN": 0,
      "elementGivenRegion": null,
      "altTop1": null,
      "retried": 0,
      "costPer1000": 0.22883033333333327,
      "totalCostUsd": 0.05766524399999999,
      "inputTokens": {
        "median": 2136.5,
        "p95": 18657.45
      },
      "misses": [
        {
          "id": "demoqa-checkbox-desktop#1",
          "site": "demoqa",
          "page": {
            "title": "demosite",
            "path": "/checkbox"
          },
          "goal": "Expand the Home node of the tree and tick the 'Desktop' checkbox.",
          "step": 1,
          "truth": "c4 clickable \".rc-tree-switcher\"",
          "chosen": "c3 clickable \"Home\"",
          "topProbability": 0.59,
          "truthRegion": "r3",
          "chosenRegion": "r3"
        },
        {
          "id": "demoqa-checkbox-desktop#2",
          "site": "demoqa",
          "page": {
            "title": "demosite",
            "path": "/checkbox"
          },
          "goal": "Expand the Home node of the tree and tick the 'Desktop' checkbox.",
          "step": 2,
          "truth": "c7 clickable \"Desktop\"",
          "chosen": "c9 checkbox \"Select Desktop\"",
          "topProbability": 0.85,
          "truthRegion": "r3",
          "chosenRegion": "r3"
        }
      ]
    },
    "jev-hier": {
      "answered": 252,
      "errors": 0,
      "steps": 164,
      "stepsTruthInCandidates": 163,
      "top1": 0.9390243902439024,
      "top1InCandidates": 0.9447852760736196,
      "top3InCandidates": 0.957983193277311,
      "top3N": 119,
      "perSite": {
        "demo-app": {
          "steps": 19,
          "correct": 19,
          "inCandidates": 19,
          "correctInCandidates": 19,
          "top1": 1,
          "top1InCandidates": 1
        },
        "playwright-docs": {
          "steps": 7,
          "correct": 6,
          "inCandidates": 6,
          "correctInCandidates": 6,
          "top1": 0.8571428571428571,
          "top1InCandidates": 1
        },
        "todomvc": {
          "steps": 17,
          "correct": 17,
          "inCandidates": 17,
          "correctInCandidates": 17,
          "top1": 1,
          "top1InCandidates": 1
        },
        "quotes-toscrape": {
          "steps": 7,
          "correct": 7,
          "inCandidates": 7,
          "correctInCandidates": 7,
          "top1": 1,
          "top1InCandidates": 1
        },
        "the-internet": {
          "steps": 19,
          "correct": 19,
          "inCandidates": 19,
          "correctInCandidates": 19,
          "top1": 1,
          "top1InCandidates": 1
        },
        "saucedemo": {
          "steps": 28,
          "correct": 28,
          "inCandidates": 28,
          "correctInCandidates": 28,
          "top1": 1,
          "top1InCandidates": 1
        },
        "books-toscrape": {
          "steps": 8,
          "correct": 7,
          "inCandidates": 8,
          "correctInCandidates": 7,
          "top1": 0.875,
          "top1InCandidates": 0.875
        },
        "client-site": {
          "steps": 36,
          "correct": 32,
          "inCandidates": 36,
          "correctInCandidates": 32,
          "top1": 0.8888888888888888,
          "top1InCandidates": 0.8888888888888888
        },
        "automationexercise": {
          "steps": 11,
          "correct": 10,
          "inCandidates": 11,
          "correctInCandidates": 10,
          "top1": 0.9090909090909091,
          "top1InCandidates": 0.9090909090909091
        },
        "demoqa": {
          "steps": 12,
          "correct": 9,
          "inCandidates": 12,
          "correctInCandidates": 9,
          "top1": 0.75,
          "top1InCandidates": 0.75
        }
      },
      "actionAccuracy": 0.9634146341463414,
      "actionN": 164,
      "goalMetAccuracy": 0.9126984126984127,
      "goalMetFalsePositiveRate": 0,
      "goalMetTruePositiveRate": 0.75,
      "goalMetN": 252,
      "calibration": {
        "bins": [
          {
            "range": "0.0-0.2",
            "n": 0,
            "correct": 0,
            "confSum": 0
          },
          {
            "range": "0.2-0.4",
            "n": 1,
            "correct": 1,
            "confSum": 0.38,
            "accuracy": 1,
            "avgConfidence": 0.38
          },
          {
            "range": "0.4-0.6",
            "n": 5,
            "correct": 3,
            "confSum": 2.5214999999999996,
            "accuracy": 0.6,
            "avgConfidence": 0.5043
          },
          {
            "range": "0.6-0.8",
            "n": 12,
            "correct": 11,
            "confSum": 8.250900000000001,
            "accuracy": 0.9166666666666666,
            "avgConfidence": 0.6875750000000002
          },
          {
            "range": "0.8-1.0",
            "n": 145,
            "correct": 139,
            "confSum": 141.51439999999994,
            "accuracy": 0.9586206896551724,
            "avgConfidence": 0.9759613793103444
          }
        ],
        "ece": 0.03903067484662538,
        "n": 163
      },
      "escalation": [
        {
          "threshold": 0.5,
          "accuracy": 0.950920245398773,
          "escalatedFraction": 0.024539877300613498,
          "n": 163
        },
        {
          "threshold": 0.55,
          "accuracy": 0.9570552147239264,
          "escalatedFraction": 0.03067484662576687,
          "n": 163
        },
        {
          "threshold": 0.6,
          "accuracy": 0.9570552147239264,
          "escalatedFraction": 0.03680981595092025,
          "n": 163
        },
        {
          "threshold": 0.65,
          "accuracy": 0.9570552147239264,
          "escalatedFraction": 0.049079754601226995,
          "n": 163
        },
        {
          "threshold": 0.7,
          "accuracy": 0.950920245398773,
          "escalatedFraction": 0.07975460122699386,
          "n": 163
        },
        {
          "threshold": 0.75,
          "accuracy": 0.950920245398773,
          "escalatedFraction": 0.09815950920245399,
          "n": 163
        },
        {
          "threshold": 0.8,
          "accuracy": 0.9570552147239264,
          "escalatedFraction": 0.11042944785276074,
          "n": 163
        },
        {
          "threshold": 0.85,
          "accuracy": 0.9570552147239264,
          "escalatedFraction": 0.13496932515337423,
          "n": 163
        },
        {
          "threshold": 0.9,
          "accuracy": 0.9693251533742331,
          "escalatedFraction": 0.15950920245398773,
          "n": 163
        },
        {
          "threshold": 0.95,
          "accuracy": 0.9754601226993865,
          "escalatedFraction": 0.25153374233128833,
          "n": 163
        }
      ],
      "latencyMs": {
        "median": 641.5,
        "p95": 863.45,
        "n": 252
      },
      "latencyParts": [
        {
          "median": 309,
          "p95": 467.6
        },
        {
          "median": 316,
          "p95": 471.6
        }
      ],
      "roundTrips": 2,
      "requestBytes": {
        "median": 4837.5,
        "p95": 17573.64999999999,
        "max": 37513
      },
      "regionAccuracy": 0.9447852760736196,
      "regionN": 163,
      "elementGivenRegion": 0.9805194805194806,
      "altTop1": null,
      "retried": 0,
      "costPer1000": 0.11512433333333337,
      "totalCostUsd": 0.029011332000000008,
      "inputTokens": {
        "median": 1942.5,
        "p95": 6312.5999999999985
      },
      "misses": [
        {
          "id": "books-poetry-open-second-book#2",
          "site": "books-toscrape",
          "page": {
            "title": "Poetry | Books to Scrape - Sandbox",
            "path": "/catalogue/category/books/poetry_23/index.html"
          },
          "goal": "Open the Poetry category and open the second book listed.",
          "step": 2,
          "truth": "c29 link \"The Black Maria\" (row: \u00a352.15 In stock Add to basket)",
          "chosen": "c28 link \"A Light in the ...\" (row: \u00a351.77 In stock Add to basket)",
          "topProbability": 0.5044,
          "truthRegion": "r4",
          "chosenRegion": "r3"
        },
        {
          "id": "demoqa-checkbox-desktop#1",
          "site": "demoqa",
          "page": {
            "title": "demosite",
            "path": "/checkbox"
          },
          "goal": "Expand the Home node of the tree and tick the 'Desktop' checkbox.",
          "step": 1,
          "truth": "c4 clickable \".rc-tree-switcher\"",
          "chosen": "c3 clickable \"Home\"",
          "topProbability": 0.9009,
          "truthRegion": "r3",
          "chosenRegion": "r3"
        },
        {
          "id": "demoqa-checkbox-desktop#2",
          "site": "demoqa",
          "page": {
            "title": "demosite",
            "path": "/checkbox"
          },
          "goal": "Expand the Home node of the tree and tick the 'Desktop' checkbox.",
          "step": 2,
          "truth": "c7 clickable \"Desktop\"",
          "chosen": "c9 checkbox \"Select Desktop\"",
          "topProbability": 0.84,
          "truthRegion": "r3",
          "chosenRegion": "r3"
        },
        {
          "id": "ae-view-first-product#3",
          "site": "automationexercise",
          "page": {
            "title": "Automation Exercise - All Products",
            "path": "/products"
          },
          "goal": "View the details of the first product in the product list.",
          "step": 3,
          "truth": "c32 link \"View Product\" (row: Rs. 500 Blue Top Add to cart Rs. 500 Blue Top Add to cart)",
          "chosen": "c34 link \"View Product\" (row: Rs. 400 Men Tshirt Add to cart Rs. 400 Men Tshirt Add to ca\u2026)",
          "topProbability": 0.882,
          "truthRegion": "r12",
          "chosenRegion": "r6"
        },
        {
          "id": "demoqa-old-select-green#1",
          "site": "demoqa",
          "page": {
            "title": "demosite",
            "path": "/select-menu"
          },
          "goal": "Choose 'Green' in the old style select menu.",
          "step": 1,
          "truth": "c8 combobox \"Red\"",
          "chosen": "c4 combobox \"react-select-2-input\" (row: Select Option)",
          "topProbability": 0.47,
          "truthRegion": "r3",
          "chosenRegion": "r3"
        }
      ]
    },
    "jev-fanout": {
      "answered": 252,
      "errors": 0,
      "steps": 164,
      "stepsTruthInCandidates": 163,
      "top1": 0.9451219512195121,
      "top1InCandidates": 0.950920245398773,
      "top3InCandidates": 0.9663865546218487,
      "top3N": 119,
      "perSite": {
        "demo-app": {
          "steps": 19,
          "correct": 19,
          "inCandidates": 19,
          "correctInCandidates": 19,
          "top1": 1,
          "top1InCandidates": 1
        },
        "playwright-docs": {
          "steps": 7,
          "correct": 6,
          "inCandidates": 6,
          "correctInCandidates": 6,
          "top1": 0.8571428571428571,
          "top1InCandidates": 1
        },
        "todomvc": {
          "steps": 17,
          "correct": 17,
          "inCandidates": 17,
          "correctInCandidates": 17,
          "top1": 1,
          "top1InCandidates": 1
        },
        "quotes-toscrape": {
          "steps": 7,
          "correct": 7,
          "inCandidates": 7,
          "correctInCandidates": 7,
          "top1": 1,
          "top1InCandidates": 1
        },
        "the-internet": {
          "steps": 19,
          "correct": 18,
          "inCandidates": 19,
          "correctInCandidates": 18,
          "top1": 0.9473684210526315,
          "top1InCandidates": 0.9473684210526315
        },
        "saucedemo": {
          "steps": 28,
          "correct": 28,
          "inCandidates": 28,
          "correctInCandidates": 28,
          "top1": 1,
          "top1InCandidates": 1
        },
        "books-toscrape": {
          "steps": 8,
          "correct": 8,
          "inCandidates": 8,
          "correctInCandidates": 8,
          "top1": 1,
          "top1InCandidates": 1
        },
        "client-site": {
          "steps": 36,
          "correct": 31,
          "inCandidates": 36,
          "correctInCandidates": 31,
          "top1": 0.8611111111111112,
          "top1InCandidates": 0.8611111111111112
        },
        "automationexercise": {
          "steps": 11,
          "correct": 11,
          "inCandidates": 11,
          "correctInCandidates": 11,
          "top1": 1,
          "top1InCandidates": 1
        },
        "demoqa": {
          "steps": 12,
          "correct": 10,
          "inCandidates": 12,
          "correctInCandidates": 10,
          "top1": 0.8333333333333334,
          "top1InCandidates": 0.8333333333333334
        }
      },
      "actionAccuracy": 0.975609756097561,
      "actionN": 164,
      "goalMetAccuracy": 0.9126984126984127,
      "goalMetFalsePositiveRate": 0,
      "goalMetTruePositiveRate": 0.75,
      "goalMetN": 252,
      "calibration": {
        "bins": [
          {
            "range": "0.0-0.2",
            "n": 0,
            "correct": 0,
            "confSum": 0
          },
          {
            "range": "0.2-0.4",
            "n": 0,
            "correct": 0,
            "confSum": 0
          },
          {
            "range": "0.4-0.6",
            "n": 2,
            "correct": 1,
            "confSum": 1.0799999999999998,
            "accuracy": 0.5,
            "avgConfidence": 0.5399999999999999
          },
          {
            "range": "0.6-0.8",
            "n": 9,
            "correct": 6,
            "confSum": 6.5291000000000015,
            "accuracy": 0.6666666666666666,
            "avgConfidence": 0.7254555555555557
          },
          {
            "range": "0.8-1.0",
            "n": 152,
            "correct": 148,
            "confSum": 147.7049,
            "accuracy": 0.9736842105263158,
            "avgConfidence": 0.9717427631578948
          }
        ],
        "ece": 0.005547239263803696,
        "n": 163
      },
      "escalation": [
        {
          "threshold": 0.5,
          "accuracy": 0.950920245398773,
          "escalatedFraction": 0.006134969325153374,
          "n": 163
        },
        {
          "threshold": 0.55,
          "accuracy": 0.950920245398773,
          "escalatedFraction": 0.006134969325153374,
          "n": 163
        },
        {
          "threshold": 0.6,
          "accuracy": 0.9570552147239264,
          "escalatedFraction": 0.012269938650306749,
          "n": 163
        },
        {
          "threshold": 0.65,
          "accuracy": 0.9570552147239264,
          "escalatedFraction": 0.024539877300613498,
          "n": 163
        },
        {
          "threshold": 0.7,
          "accuracy": 0.9570552147239264,
          "escalatedFraction": 0.03067484662576687,
          "n": 163
        },
        {
          "threshold": 0.75,
          "accuracy": 0.9631901840490797,
          "escalatedFraction": 0.03680981595092025,
          "n": 163
        },
        {
          "threshold": 0.8,
          "accuracy": 0.9754601226993865,
          "escalatedFraction": 0.06748466257668712,
          "n": 163
        },
        {
          "threshold": 0.85,
          "accuracy": 0.9754601226993865,
          "escalatedFraction": 0.1165644171779141,
          "n": 163
        },
        {
          "threshold": 0.9,
          "accuracy": 0.9815950920245399,
          "escalatedFraction": 0.15950920245398773,
          "n": 163
        },
        {
          "threshold": 0.95,
          "accuracy": 0.9815950920245399,
          "escalatedFraction": 0.2331288343558282,
          "n": 163
        }
      ],
      "latencyMs": {
        "median": 359,
        "p95": 547.6999999999999,
        "n": 252
      },
      "latencyParts": [
        null,
        null
      ],
      "roundTrips": 1,
      "requestBytes": {
        "median": 8322,
        "p95": 62375.35,
        "max": 70397
      },
      "regionAccuracy": 0.950920245398773,
      "regionN": 163,
      "elementGivenRegion": 0.9741935483870968,
      "altTop1": 0.950920245398773,
      "retried": 0,
      "costPer1000": 0.2790536666666669,
      "totalCostUsd": 0.07032152400000007,
      "inputTokens": {
        "median": 2867.5,
        "p95": 20704.45
      },
      "misses": [
        {
          "id": "the-internet-dynamic-controls#1",
          "site": "the-internet",
          "page": {
            "title": "The Internet",
            "path": "/dynamic_controls"
          },
          "goal": "On the dynamic controls page, remove the checkbox and then enable the text input.",
          "step": 1,
          "truth": "c2 button \"Remove\" (row: Dynamic Controls This example demonstrates when elements (e\u2026)",
          "chosen": "c1 checkbox \"\" (row: Dynamic Controls This example demonstrates when elements (e\u2026)",
          "topProbability": 0.89,
          "truthRegion": "r1",
          "chosenRegion": "r1"
        },
        {
          "id": "demoqa-checkbox-desktop#1",
          "site": "demoqa",
          "page": {
            "title": "demosite",
            "path": "/checkbox"
          },
          "goal": "Expand the Home node of the tree and tick the 'Desktop' checkbox.",
          "step": 1,
          "truth": "c4 clickable \".rc-tree-switcher\"",
          "chosen": "c3 clickable \"Home\"",
          "topProbability": 0.76,
          "truthRegion": "r3",
          "chosenRegion": "r3"
        },
        {
          "id": "demoqa-checkbox-desktop#2",
          "site": "demoqa",
          "page": {
            "title": "demosite",
            "path": "/checkbox"
          },
          "goal": "Expand the Home node of the tree and tick the 'Desktop' checkbox.",
          "step": 2,
          "truth": "c7 clickable \"Desktop\"",
          "chosen": "c9 checkbox \"Select Desktop\"",
          "topProbability": 0.8,
          "truthRegion": "r3",
          "chosenRegion": "r3"
        }
      ]
    },
    "haiku": {
      "answered": 252,
      "errors": 0,
      "steps": 164,
      "stepsTruthInCandidates": 163,
      "top1": 0.9695121951219512,
      "top1InCandidates": 0.9754601226993865,
      "top3InCandidates": null,
      "top3N": 0,
      "perSite": {
        "demo-app": {
          "steps": 19,
          "correct": 19,
          "inCandidates": 19,
          "correctInCandidates": 19,
          "top1": 1,
          "top1InCandidates": 1
        },
        "playwright-docs": {
          "steps": 7,
          "correct": 6,
          "inCandidates": 6,
          "correctInCandidates": 6,
          "top1": 0.8571428571428571,
          "top1InCandidates": 1
        },
        "todomvc": {
          "steps": 17,
          "correct": 17,
          "inCandidates": 17,
          "correctInCandidates": 17,
          "top1": 1,
          "top1InCandidates": 1
        },
        "quotes-toscrape": {
          "steps": 7,
          "correct": 7,
          "inCandidates": 7,
          "correctInCandidates": 7,
          "top1": 1,
          "top1InCandidates": 1
        },
        "the-internet": {
          "steps": 19,
          "correct": 19,
          "inCandidates": 19,
          "correctInCandidates": 19,
          "top1": 1,
          "top1InCandidates": 1
        },
        "saucedemo": {
          "steps": 28,
          "correct": 27,
          "inCandidates": 28,
          "correctInCandidates": 27,
          "top1": 0.9642857142857143,
          "top1InCandidates": 0.9642857142857143
        },
        "books-toscrape": {
          "steps": 8,
          "correct": 8,
          "inCandidates": 8,
          "correctInCandidates": 8,
          "top1": 1,
          "top1InCandidates": 1
        },
        "client-site": {
          "steps": 36,
          "correct": 34,
          "inCandidates": 36,
          "correctInCandidates": 34,
          "top1": 0.9444444444444444,
          "top1InCandidates": 0.9444444444444444
        },
        "automationexercise": {
          "steps": 11,
          "correct": 11,
          "inCandidates": 11,
          "correctInCandidates": 11,
          "top1": 1,
          "top1InCandidates": 1
        },
        "demoqa": {
          "steps": 12,
          "correct": 11,
          "inCandidates": 12,
          "correctInCandidates": 11,
          "top1": 0.9166666666666666,
          "top1InCandidates": 0.9166666666666666
        }
      },
      "actionAccuracy": 0.9634146341463414,
      "actionN": 164,
      "goalMetAccuracy": 0.8968253968253969,
      "goalMetFalsePositiveRate": 0,
      "goalMetTruePositiveRate": 0.7045454545454546,
      "goalMetN": 252,
      "calibration": {
        "bins": [
          {
            "range": "0.0-0.2",
            "n": 0,
            "correct": 0,
            "confSum": 0
          },
          {
            "range": "0.2-0.4",
            "n": 0,
            "correct": 0,
            "confSum": 0
          },
          {
            "range": "0.4-0.6",
            "n": 0,
            "correct": 0,
            "confSum": 0
          },
          {
            "range": "0.6-0.8",
            "n": 0,
            "correct": 0,
            "confSum": 0
          },
          {
            "range": "0.8-1.0",
            "n": 163,
            "correct": 159,
            "confSum": 156.3599999999998,
            "accuracy": 0.9754601226993865,
            "avgConfidence": 0.9592638036809803
          }
        ],
        "ece": 0.016196319018406236,
        "n": 163
      },
      "escalation": [
        {
          "threshold": 0.5,
          "accuracy": 0.9754601226993865,
          "escalatedFraction": 0,
          "n": 163
        },
        {
          "threshold": 0.55,
          "accuracy": 0.9754601226993865,
          "escalatedFraction": 0,
          "n": 163
        },
        {
          "threshold": 0.6,
          "accuracy": 0.9754601226993865,
          "escalatedFraction": 0,
          "n": 163
        },
        {
          "threshold": 0.65,
          "accuracy": 0.9754601226993865,
          "escalatedFraction": 0,
          "n": 163
        },
        {
          "threshold": 0.7,
          "accuracy": 0.9754601226993865,
          "escalatedFraction": 0,
          "n": 163
        },
        {
          "threshold": 0.75,
          "accuracy": 0.9754601226993865,
          "escalatedFraction": 0,
          "n": 163
        },
        {
          "threshold": 0.8,
          "accuracy": 0.9754601226993865,
          "escalatedFraction": 0,
          "n": 163
        },
        {
          "threshold": 0.85,
          "accuracy": 0.9754601226993865,
          "escalatedFraction": 0,
          "n": 163
        },
        {
          "threshold": 0.9,
          "accuracy": 0.9815950920245399,
          "escalatedFraction": 0.012269938650306749,
          "n": 163
        },
        {
          "threshold": 0.95,
          "accuracy": 0.9815950920245399,
          "escalatedFraction": 0.012269938650306749,
          "n": 163
        }
      ],
      "latencyMs": {
        "median": 1643,
        "p95": 3287.35,
        "n": 252
      },
      "latencyParts": [
        null,
        null
      ],
      "roundTrips": 1,
      "requestBytes": null,
      "regionAccuracy": null,
      "regionN": 0,
      "elementGivenRegion": null,
      "altTop1": null,
      "retried": 1,
      "costPer1000": 3.0278055555555574,
      "totalCostUsd": 0.7630070000000004,
      "inputTokens": {
        "median": 970,
        "p95": 8200.05
      },
      "misses": [
        {
          "id": "saucedemo-two-items-cart#5",
          "site": "saucedemo",
          "page": {
            "title": "Swag Labs",
            "path": "/inventory.html"
          },
          "goal": "Add both the Sauce Labs Fleece Jacket and the Sauce Labs Onesie to the cart and open the cart.",
          "step": 5,
          "truth": "c20 button \"Add to cart\" (row: Sauce Labs Onesie Rib snap infant onesie for the junior aut\u2026)",
          "chosen": "c15 button \"Remove\" (row: Sauce Labs Fleece Jacket It's not every day that you come a\u2026)",
          "topProbability": 0.85,
          "truthRegion": "r2",
          "chosenRegion": "r2"
        },
        {
          "id": "demoqa-checkbox-desktop#2",
          "site": "demoqa",
          "page": {
            "title": "demosite",
            "path": "/checkbox"
          },
          "goal": "Expand the Home node of the tree and tick the 'Desktop' checkbox.",
          "step": 2,
          "truth": "c7 clickable \"Desktop\"",
          "chosen": "c9 checkbox \"Select Desktop\"",
          "topProbability": 0.95,
          "truthRegion": "r3",
          "chosenRegion": "r3"
        }
      ]
    },
    "haiku-hier": {
      "answered": 252,
      "errors": 0,
      "steps": 164,
      "stepsTruthInCandidates": 163,
      "top1": 0.9146341463414634,
      "top1InCandidates": 0.9202453987730062,
      "top3InCandidates": null,
      "top3N": 0,
      "perSite": {
        "demo-app": {
          "steps": 19,
          "correct": 19,
          "inCandidates": 19,
          "correctInCandidates": 19,
          "top1": 1,
          "top1InCandidates": 1
        },
        "playwright-docs": {
          "steps": 7,
          "correct": 6,
          "inCandidates": 6,
          "correctInCandidates": 6,
          "top1": 0.8571428571428571,
          "top1InCandidates": 1
        },
        "todomvc": {
          "steps": 17,
          "correct": 14,
          "inCandidates": 17,
          "correctInCandidates": 14,
          "top1": 0.8235294117647058,
          "top1InCandidates": 0.8235294117647058
        },
        "quotes-toscrape": {
          "steps": 7,
          "correct": 7,
          "inCandidates": 7,
          "correctInCandidates": 7,
          "top1": 1,
          "top1InCandidates": 1
        },
        "the-internet": {
          "steps": 19,
          "correct": 19,
          "inCandidates": 19,
          "correctInCandidates": 19,
          "top1": 1,
          "top1InCandidates": 1
        },
        "saucedemo": {
          "steps": 28,
          "correct": 27,
          "inCandidates": 28,
          "correctInCandidates": 27,
          "top1": 0.9642857142857143,
          "top1InCandidates": 0.9642857142857143
        },
        "books-toscrape": {
          "steps": 8,
          "correct": 7,
          "inCandidates": 8,
          "correctInCandidates": 7,
          "top1": 0.875,
          "top1InCandidates": 0.875
        },
        "client-site": {
          "steps": 36,
          "correct": 31,
          "inCandidates": 36,
          "correctInCandidates": 31,
          "top1": 0.8611111111111112,
          "top1InCandidates": 0.8611111111111112
        },
        "automationexercise": {
          "steps": 11,
          "correct": 10,
          "inCandidates": 11,
          "correctInCandidates": 10,
          "top1": 0.9090909090909091,
          "top1InCandidates": 0.9090909090909091
        },
        "demoqa": {
          "steps": 12,
          "correct": 10,
          "inCandidates": 12,
          "correctInCandidates": 10,
          "top1": 0.8333333333333334,
          "top1InCandidates": 0.8333333333333334
        }
      },
      "actionAccuracy": 0.9631901840490797,
      "actionN": 163,
      "goalMetAccuracy": 0.8174603174603174,
      "goalMetFalsePositiveRate": 0.036585365853658534,
      "goalMetTruePositiveRate": 0.5454545454545454,
      "goalMetN": 252,
      "calibration": {
        "bins": [
          {
            "range": "0.0-0.2",
            "n": 2,
            "correct": 0,
            "confSum": 0.2375,
            "accuracy": 0,
            "avgConfidence": 0.11875
          },
          {
            "range": "0.2-0.4",
            "n": 0,
            "correct": 0,
            "confSum": 0
          },
          {
            "range": "0.4-0.6",
            "n": 0,
            "correct": 0,
            "confSum": 0
          },
          {
            "range": "0.6-0.8",
            "n": 1,
            "correct": 1,
            "confSum": 0.7124999999999999,
            "accuracy": 1,
            "avgConfidence": 0.7124999999999999
          },
          {
            "range": "0.8-1.0",
            "n": 160,
            "correct": 149,
            "confSum": 144.09600000000037,
            "accuracy": 0.93125,
            "avgConfidence": 0.9006000000000023
          }
        ],
        "ece": 0.03330674846625545,
        "n": 163
      },
      "escalation": [
        {
          "threshold": 0.5,
          "accuracy": 0.9325153374233128,
          "escalatedFraction": 0.012269938650306749,
          "n": 163
        },
        {
          "threshold": 0.55,
          "accuracy": 0.9325153374233128,
          "escalatedFraction": 0.012269938650306749,
          "n": 163
        },
        {
          "threshold": 0.6,
          "accuracy": 0.9325153374233128,
          "escalatedFraction": 0.012269938650306749,
          "n": 163
        },
        {
          "threshold": 0.65,
          "accuracy": 0.9325153374233128,
          "escalatedFraction": 0.012269938650306749,
          "n": 163
        },
        {
          "threshold": 0.7,
          "accuracy": 0.9325153374233128,
          "escalatedFraction": 0.012269938650306749,
          "n": 163
        },
        {
          "threshold": 0.75,
          "accuracy": 0.9325153374233128,
          "escalatedFraction": 0.018404907975460124,
          "n": 163
        },
        {
          "threshold": 0.8,
          "accuracy": 0.9325153374233128,
          "escalatedFraction": 0.018404907975460124,
          "n": 163
        },
        {
          "threshold": 0.85,
          "accuracy": 0.9447852760736196,
          "escalatedFraction": 0.05521472392638037,
          "n": 163
        },
        {
          "threshold": 0.9,
          "accuracy": 0.9447852760736196,
          "escalatedFraction": 0.05521472392638037,
          "n": 163
        },
        {
          "threshold": 0.95,
          "accuracy": 0.9754601226993865,
          "escalatedFraction": 1,
          "n": 163
        }
      ],
      "latencyMs": {
        "median": 2926,
        "p95": 5030.599999999999,
        "n": 252
      },
      "latencyParts": [
        {
          "median": 1675,
          "p95": 2455.95
        },
        {
          "median": 1120.5,
          "p95": 2872.85
        }
      ],
      "roundTrips": 2,
      "requestBytes": null,
      "regionAccuracy": 0.9263803680981595,
      "regionN": 163,
      "elementGivenRegion": 0.9735099337748344,
      "altTop1": null,
      "retried": 2,
      "costPer1000": 2.1707658730158745,
      "totalCostUsd": 0.5470330000000004,
      "inputTokens": {
        "median": 877.5,
        "p95": 2994.049999999997
      },
      "misses": [
        {
          "id": "todomvc-complete-one#3",
          "site": "todomvc",
          "page": {
            "title": "React \u2022 TodoMVC",
            "path": "/todomvc/#/"
          },
          "goal": "Add the todo 'Buy milk' and mark it as completed.",
          "step": 3,
          "truth": "c4 checkbox \"Toggle Todo\" (row: Buy milk)",
          "chosen": "c2 textbox \"What needs to be done?\"",
          "topProbability": 0.9025,
          "truthRegion": "r3",
          "chosenRegion": "r2"
        },
        {
          "id": "todomvc-clear-completed#3",
          "site": "todomvc",
          "page": {
            "title": "React \u2022 TodoMVC",
            "path": "/todomvc/#/"
          },
          "goal": "Add 'Buy milk', mark it completed, then clear the completed todos.",
          "step": 3,
          "truth": "c4 checkbox \"Toggle Todo\" (row: Buy milk)",
          "chosen": "c2 textbox \"What needs to be done?\"",
          "topProbability": 0.9025,
          "truthRegion": "r3",
          "chosenRegion": "r2"
        },
        {
          "id": "todomvc-completed-filter#3",
          "site": "todomvc",
          "page": {
            "title": "React \u2022 TodoMVC",
            "path": "/todomvc/#/"
          },
          "goal": "Add 'Buy milk', mark it completed, then open the Completed filter.",
          "step": 3,
          "truth": "c4 checkbox \"Toggle Todo\" (row: Buy milk)",
          "chosen": "c2 textbox \"What needs to be done?\"",
          "topProbability": 0.9025,
          "truthRegion": "r3",
          "chosenRegion": "r2"
        },
        {
          "id": "saucedemo-two-items-cart#5",
          "site": "saucedemo",
          "page": {
            "title": "Swag Labs",
            "path": "/inventory.html"
          },
          "goal": "Add both the Sauce Labs Fleece Jacket and the Sauce Labs Onesie to the cart and open the cart.",
          "step": 5,
          "truth": "c20 button \"Add to cart\" (row: Sauce Labs Onesie Rib snap infant onesie for the junior aut\u2026)",
          "chosen": "c15 button \"Remove\" (row: Sauce Labs Fleece Jacket It's not every day that you come a\u2026)",
          "topProbability": 0.8075,
          "truthRegion": "r2",
          "chosenRegion": "r2"
        },
        {
          "id": "books-poetry-open-second-book#2",
          "site": "books-toscrape",
          "page": {
            "title": "Poetry | Books to Scrape - Sandbox",
            "path": "/catalogue/category/books/poetry_23/index.html"
          },
          "goal": "Open the Poetry category and open the second book listed.",
          "step": 2,
          "truth": "c29 link \"The Black Maria\" (row: \u00a352.15 In stock Add to basket)",
          "chosen": "c16 link \"Shakespeare's Sonnets\" (row: Shakespeare's Sonnets \u00a320.66 In stock Add to basket)",
          "topProbability": 0.9025,
          "truthRegion": "r4",
          "chosenRegion": "r4"
        },
        {
          "id": "demoqa-checkbox-desktop#2",
          "site": "demoqa",
          "page": {
            "title": "demosite",
            "path": "/checkbox"
          },
          "goal": "Expand the Home node of the tree and tick the 'Desktop' checkbox.",
          "step": 2,
          "truth": "c7 clickable \"Desktop\"",
          "chosen": "c9 checkbox \"Select Desktop\"",
          "topProbability": 0.9025,
          "truthRegion": "r3",
          "chosenRegion": "r3"
        },
        {
          "id": "ae-view-first-product#3",
          "site": "automationexercise",
          "page": {
            "title": "Automation Exercise - All Products",
            "path": "/products"
          },
          "goal": "View the details of the first product in the product list.",
          "step": 3,
          "truth": "c32 link \"View Product\" (row: Rs. 500 Blue Top Add to cart Rs. 500 Blue Top Add to cart)",
          "chosen": "c34 link \"View Product\" (row: Rs. 400 Men Tshirt Add to cart Rs. 400 Men Tshirt Add to ca\u2026)",
          "topProbability": 0.9025,
          "truthRegion": "r12",
          "chosenRegion": "r6"
        },
        {
          "id": "demoqa-old-select-green#1",
          "site": "demoqa",
          "page": {
            "title": "demosite",
            "path": "/select-menu"
          },
          "goal": "Choose 'Green' in the old style select menu.",
          "step": 1,
          "truth": "c8 combobox \"Red\"",
          "chosen": "c13 listbox \"cars\" (row: Standard multi select)",
          "topProbability": 0.8075,
          "truthRegion": "r3",
          "chosenRegion": "r3"
        }
      ]
    },
    "sonnet": {
      "answered": 252,
      "errors": 0,
      "steps": 164,
      "stepsTruthInCandidates": 163,
      "top1": 0.9695121951219512,
      "top1InCandidates": 0.9754601226993865,
      "top3InCandidates": null,
      "top3N": 0,
      "perSite": {
        "demo-app": {
          "steps": 19,
          "correct": 19,
          "inCandidates": 19,
          "correctInCandidates": 19,
          "top1": 1,
          "top1InCandidates": 1
        },
        "playwright-docs": {
          "steps": 7,
          "correct": 6,
          "inCandidates": 6,
          "correctInCandidates": 6,
          "top1": 0.8571428571428571,
          "top1InCandidates": 1
        },
        "todomvc": {
          "steps": 17,
          "correct": 16,
          "inCandidates": 17,
          "correctInCandidates": 16,
          "top1": 0.9411764705882353,
          "top1InCandidates": 0.9411764705882353
        },
        "quotes-toscrape": {
          "steps": 7,
          "correct": 7,
          "inCandidates": 7,
          "correctInCandidates": 7,
          "top1": 1,
          "top1InCandidates": 1
        },
        "the-internet": {
          "steps": 19,
          "correct": 19,
          "inCandidates": 19,
          "correctInCandidates": 19,
          "top1": 1,
          "top1InCandidates": 1
        },
        "saucedemo": {
          "steps": 28,
          "correct": 28,
          "inCandidates": 28,
          "correctInCandidates": 28,
          "top1": 1,
          "top1InCandidates": 1
        },
        "books-toscrape": {
          "steps": 8,
          "correct": 8,
          "inCandidates": 8,
          "correctInCandidates": 8,
          "top1": 1,
          "top1InCandidates": 1
        },
        "client-site": {
          "steps": 36,
          "correct": 34,
          "inCandidates": 36,
          "correctInCandidates": 34,
          "top1": 0.9444444444444444,
          "top1InCandidates": 0.9444444444444444
        },
        "automationexercise": {
          "steps": 11,
          "correct": 11,
          "inCandidates": 11,
          "correctInCandidates": 11,
          "top1": 1,
          "top1InCandidates": 1
        },
        "demoqa": {
          "steps": 12,
          "correct": 11,
          "inCandidates": 12,
          "correctInCandidates": 11,
          "top1": 0.9166666666666666,
          "top1InCandidates": 0.9166666666666666
        }
      },
      "actionAccuracy": 0.975609756097561,
      "actionN": 164,
      "goalMetAccuracy": 0.9285714285714286,
      "goalMetFalsePositiveRate": 0,
      "goalMetTruePositiveRate": 0.7954545454545454,
      "goalMetN": 252,
      "calibration": {
        "bins": [
          {
            "range": "0.0-0.2",
            "n": 0,
            "correct": 0,
            "confSum": 0
          },
          {
            "range": "0.2-0.4",
            "n": 0,
            "correct": 0,
            "confSum": 0
          },
          {
            "range": "0.4-0.6",
            "n": 0,
            "correct": 0,
            "confSum": 0
          },
          {
            "range": "0.6-0.8",
            "n": 3,
            "correct": 2,
            "confSum": 2.1,
            "accuracy": 0.6666666666666666,
            "avgConfidence": 0.7000000000000001
          },
          {
            "range": "0.8-1.0",
            "n": 160,
            "correct": 157,
            "confSum": 151.64,
            "accuracy": 0.98125,
            "avgConfidence": 0.9477499999999999
          }
        ],
        "ece": 0.03349693251533751,
        "n": 163
      },
      "escalation": null,
      "latencyMs": {
        "median": 1403.5,
        "p95": 1824.6499999999999,
        "n": 252
      },
      "latencyParts": [
        null,
        null
      ],
      "roundTrips": 1,
      "requestBytes": null,
      "regionAccuracy": null,
      "regionN": 0,
      "elementGivenRegion": null,
      "altTop1": null,
      "retried": 0,
      "costPer1000": 6.758277777777776,
      "totalCostUsd": 1.7030859999999997,
      "inputTokens": {
        "median": 1193.5,
        "p95": 10694.3
      },
      "misses": [
        {
          "id": "todomvc-clear-completed#3",
          "site": "todomvc",
          "page": {
            "title": "React \u2022 TodoMVC",
            "path": "/todomvc/#/"
          },
          "goal": "Add 'Buy milk', mark it completed, then clear the completed todos.",
          "step": 3,
          "truth": "c4 checkbox \"Toggle Todo\" (row: Buy milk)",
          "chosen": "c2 textbox \"What needs to be done?\"",
          "topProbability": 0.6,
          "truthRegion": "r3",
          "chosenRegion": "r2"
        },
        {
          "id": "demoqa-checkbox-desktop#2",
          "site": "demoqa",
          "page": {
            "title": "demosite",
            "path": "/checkbox"
          },
          "goal": "Expand the Home node of the tree and tick the 'Desktop' checkbox.",
          "step": 2,
          "truth": "c7 clickable \"Desktop\"",
          "chosen": "c9 checkbox \"Select Desktop\"",
          "topProbability": 0.95,
          "truthRegion": "r3",
          "chosenRegion": "r3"
        }
      ]
    }
  },
  "backendInfo": {
    "jev-flat": {
      "model": "jev-1.13.0",
      "endpoint": "https://api.typesafe.ai/v1/systemone"
    },
    "jev-hier": {
      "model": "jev-1.13.0",
      "endpoint": "https://api.typesafe.ai/v1/systemone"
    },
    "jev-fanout": {
      "model": "jev-1.13.0",
      "endpoint": "https://api.typesafe.ai/v1/systemone"
    },
    "haiku": {
      "model": "claude-haiku-4-5-20251001"
    },
    "haiku-hier": {
      "model": "claude-haiku-4-5-20251001"
    },
    "sonnet": {
      "model": "claude-sonnet-5"
    }
  }
}