{
  "campaignId": "fc-d89e429d2781",
  "counts": {
    "attempts": 150,
    "cells": 15,
    "certifiedPasses": 93,
    "exams": 5,
    "graded": 148,
    "invalid": 0,
    "laneErrors": 2,
    "lanes": 3,
    "modelFails": 55,
    "withheld": 0
  },
  "disclosure": {
    "public.swe.martinblech-xmltodict-issue-257": {
      "class": "public_source",
      "rule": "public-source disclosure (ownership public_source declared by engine registry, upstream https://github.com/martinblech/xmltodict under MIT; family public_oss_swe): the submitted change is a patch against a public repository, so a redacted excerpt of it is published."
    },
    "vvdex.browser.order-desk-1": {
      "class": "state",
      "rule": "state disclosure (ownership vvdex_proprietary \u2014 engine registry; family browser): the exam is VVDex evaluation IP and the graded output is the application's own state and audit trail, so no submission content, diff or state is published."
    },
    "vvdex.harness.fault-recovery-1": {
      "class": "answer_key",
      "rule": "answer-key disclosure (ownership vvdex_proprietary \u2014 engine registry; family harness_eval): the exam is VVDex evaluation IP and the graded output is an answer or a keyed value, so no submission content, diff, model claim or hidden-assertion name is published."
    },
    "vvdex.knowledge.grounded-rag-1": {
      "class": "answer_key",
      "rule": "answer-key disclosure (ownership vvdex_proprietary \u2014 engine registry; family rag_grounding): the exam is VVDex evaluation IP and the graded output is an answer or a keyed value, so no submission content, diff, model claim or hidden-assertion name is published."
    },
    "vvdex.knowledge.memory-fact-update-1": {
      "class": "answer_key",
      "rule": "answer-key disclosure (ownership vvdex_proprietary \u2014 engine registry; family long_term_memory): the exam is VVDex evaluation IP and the graded output is an answer or a keyed value, so no submission content, diff, model claim or hidden-assertion name is published."
    }
  },
  "families": [
    {
      "envId": "public.swe.martinblech-xmltodict-issue-257",
      "family": "public_oss_swe",
      "label": "Software repair (public OSS SWE)",
      "short": "SWE",
      "what": "A real software-engineering task taken from a public open-source repository: the model is given the codebase and a written issue, and must change the code so that the behaviour described in the issue is fixed."
    },
    {
      "envId": "vvdex.knowledge.grounded-rag-1",
      "family": "rag_grounding",
      "label": "Grounded retrieval (RAG)",
      "short": "RAG",
      "what": "Grounded question answering over a small document collection."
    },
    {
      "envId": "vvdex.knowledge.memory-fact-update-1",
      "family": "long_term_memory",
      "label": "Cross-session memory update",
      "short": "Memory",
      "what": "A state-and-memory task across sessions: a fact was recorded earlier, an authoritative source has since changed it, and the model must answer from the newer authoritative roster rather than the stale carried memory, and acknowledge the change."
    },
    {
      "envId": "vvdex.harness.fault-recovery-1",
      "family": "harness_eval",
      "label": "Tool and harness fault recovery",
      "short": "Harness",
      "what": "A code-fix task carried out through a deliberately unreliable tool loop."
    },
    {
      "envId": "vvdex.browser.order-desk-1",
      "family": "browser",
      "label": "Browser and computer use",
      "short": "Browser",
      "what": "A task inside a real web application, driven through a real headless browser."
    }
  ],
  "lanes": [
    {
      "certifiedPasses": 50,
      "countable": 50,
      "counts": {
        "certified_pass": 50,
        "invalid": 0,
        "lane_error": 0,
        "model_fail": 0,
        "withheld": 0
      },
      "lane": "cli/codex",
      "rate": 1.0,
      "rateInterval95": [
        0.9287,
        1.0
      ]
    },
    {
      "certifiedPasses": 40,
      "countable": 50,
      "counts": {
        "certified_pass": 40,
        "invalid": 0,
        "lane_error": 0,
        "model_fail": 10,
        "withheld": 0
      },
      "lane": "cli/claude-sonnet",
      "rate": 0.8,
      "rateInterval95": [
        0.6696,
        0.8876
      ]
    },
    {
      "certifiedPasses": 3,
      "countable": 48,
      "counts": {
        "certified_pass": 3,
        "invalid": 0,
        "lane_error": 2,
        "model_fail": 45,
        "withheld": 0
      },
      "lane": "codestral-latest",
      "rate": 0.0625,
      "rateInterval95": [
        0.0215,
        0.1684
      ]
    }
  ],
  "perExamMatrix": {
    "exams": [
      {
        "envId": "public.swe.martinblech-xmltodict-issue-257",
        "evalId": "fr-20260902-6616b188",
        "family": "public_oss_swe",
        "short": "SWE",
        "title": "Streaming items must equal what the full parse would have built"
      },
      {
        "envId": "vvdex.knowledge.grounded-rag-1",
        "evalId": "fr-20260902-2e7f4171",
        "family": "rag_grounding",
        "short": "RAG",
        "title": "Answering questions from a document collection with outdated and distractor sources, and refusing when the evidence is insufficient"
      },
      {
        "envId": "vvdex.knowledge.memory-fact-update-1",
        "evalId": "fr-20260901-46eb9548",
        "family": "long_term_memory",
        "short": "Memory",
        "title": "Updating a cross-session fact: answer from the newer authoritative roster, not stale carried memory"
      },
      {
        "envId": "vvdex.harness.fault-recovery-1",
        "evalId": "fr-20260902-a6f12b64",
        "family": "harness_eval",
        "short": "Harness",
        "title": "Harness recovery under a faulted tool loop"
      },
      {
        "envId": "vvdex.browser.order-desk-1",
        "evalId": "fr-20260902-6e618c05",
        "family": "browser",
        "short": "Browser",
        "title": "Correct one order in an admin application, through a real browser"
      }
    ],
    "rows": [
      {
        "cells": {
          "public.swe.martinblech-xmltodict-issue-257": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 10,
            "text": "10/10",
            "withheld": 0
          },
          "vvdex.browser.order-desk-1": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 10,
            "text": "10/10",
            "withheld": 0
          },
          "vvdex.harness.fault-recovery-1": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 10,
            "text": "10/10",
            "withheld": 0
          },
          "vvdex.knowledge.grounded-rag-1": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 10,
            "text": "10/10",
            "withheld": 0
          },
          "vvdex.knowledge.memory-fact-update-1": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 10,
            "text": "10/10",
            "withheld": 0
          }
        },
        "certifiedPasses": 50,
        "lane": "cli/codex"
      },
      {
        "cells": {
          "public.swe.martinblech-xmltodict-issue-257": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 9,
            "text": "9/10",
            "withheld": 0
          },
          "vvdex.browser.order-desk-1": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 10,
            "text": "10/10",
            "withheld": 0
          },
          "vvdex.harness.fault-recovery-1": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 10,
            "text": "10/10",
            "withheld": 0
          },
          "vvdex.knowledge.grounded-rag-1": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 1,
            "text": "1/10",
            "withheld": 0
          },
          "vvdex.knowledge.memory-fact-update-1": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 10,
            "text": "10/10",
            "withheld": 0
          }
        },
        "certifiedPasses": 40,
        "lane": "cli/claude-sonnet"
      },
      {
        "cells": {
          "public.swe.martinblech-xmltodict-issue-257": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 0,
            "text": "0/10",
            "withheld": 0
          },
          "vvdex.browser.order-desk-1": {
            "graded": 9,
            "invalid": 0,
            "laneErrors": 1,
            "passes": 1,
            "text": "1/9 + 1 lane error",
            "withheld": 0
          },
          "vvdex.harness.fault-recovery-1": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 0,
            "text": "0/10",
            "withheld": 0
          },
          "vvdex.knowledge.grounded-rag-1": {
            "graded": 9,
            "invalid": 0,
            "laneErrors": 1,
            "passes": 0,
            "text": "0/9 + 1 lane error",
            "withheld": 0
          },
          "vvdex.knowledge.memory-fact-update-1": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 2,
            "text": "2/10",
            "withheld": 0
          }
        },
        "certifiedPasses": 3,
        "lane": "codestral-latest"
      }
    ]
  },
  "recordCount": 5,
  "records": [
    {
      "canonicalBytesPublished": false,
      "certificationReceiptSha256": "87ba9833637f004939e1a7b2d3a69b2d0e0e37b1040ef5d403eb9c392695bee0",
      "disclosureClass": "public_source",
      "environmentId": "public.swe.martinblech-xmltodict-issue-257",
      "evalId": "fr-20260902-6616b188",
      "examFingerprint": "aa41977120e6ecdddb5c4fc603e8ab23cb272726a86750119969ff4dfc90e297",
      "recordDigest": "138425da59aabc18aa208c339aa41919f387e1ecb80fb725cfed8f7d6ba78713",
      "retired_for_future_evaluation": null
    },
    {
      "canonicalBytesPublished": false,
      "certificationReceiptSha256": "a446ba3064d1580c3f2816fe8295226b196eb89fe9da20c1608a186cd1c7856b",
      "disclosureClass": "state",
      "environmentId": "vvdex.browser.order-desk-1",
      "evalId": "fr-20260902-6e618c05",
      "examFingerprint": "8d404ee5d5ab71456de3d3df0b8ca69271d8810767a694f71c6e45e72fc347bf",
      "recordDigest": "59f94854d78f7ce174866ff114505b2b41ca2333f287f40b604fbb38cc267da5",
      "retired_for_future_evaluation": null
    },
    {
      "canonicalBytesPublished": false,
      "certificationReceiptSha256": "a8c8b313038678a3b8e92f57a8ed0a87f1b389dffe84b4e27e83ee4575648ebc",
      "disclosureClass": "answer_key",
      "environmentId": "vvdex.harness.fault-recovery-1",
      "evalId": "fr-20260902-a6f12b64",
      "examFingerprint": "4a206fbfe8fb70fb3e37943705065a8599fab492e495dca326279781ee960b1e",
      "recordDigest": "d5a8264aa0c20dd67c14879856a744e2ff1ab79da35ea8bcea92b4ba1ea14d5f",
      "retired_for_future_evaluation": null
    },
    {
      "canonicalBytesPublished": false,
      "certificationReceiptSha256": "95deabd2469ddea08d716e0cc5c2c966b874be73743798020aeedaa2fd99d70f",
      "disclosureClass": "answer_key",
      "environmentId": "vvdex.knowledge.grounded-rag-1",
      "evalId": "fr-20260902-2e7f4171",
      "examFingerprint": "4880a527ef97ca2f3fe6b57be026c3dc1be51e9e104d79fc13a3e558331dda92",
      "recordDigest": "157ec324a925c0b10fc8769d01f628b6143c40a00fa2dc70ea309a70e55a37a8",
      "retired_for_future_evaluation": {
        "environmentId": "vvdex.knowledge.grounded-rag-1",
        "historicalResults": "Historical sealed results on this revision are unaffected: the disclosure happened after those runs, so it could not have influenced them.",
        "reason": "post-run public disclosure of answer-bearing submission content",
        "retiredOn": "2026-09-02",
        "version": "0.1.0"
      }
    },
    {
      "canonicalBytesPublished": false,
      "certificationReceiptSha256": "0e21831617f6d3b3680450adc93ec0aa0367a79dfc6ed07538d8a69a05a104ee",
      "disclosureClass": "answer_key",
      "environmentId": "vvdex.knowledge.memory-fact-update-1",
      "evalId": "fr-20260901-46eb9548",
      "examFingerprint": "2052eec0ff2d3044dc269e3599ae1bfaa45d0d8cd5df9554796105538d226c21",
      "recordDigest": "1127e8356d42d31767a3ddc4d6c6cd0a515f5bbf7239ef0a82129e00fc2e6900",
      "retired_for_future_evaluation": {
        "environmentId": "vvdex.knowledge.memory-fact-update-1",
        "historicalResults": "Historical sealed results on this revision are unaffected: the disclosure happened after those runs, so it could not have influenced them.",
        "reason": "post-run public disclosure of answer-bearing submission content",
        "retiredOn": "2026-09-02",
        "version": "0.1.0"
      }
    }
  ],
  "refused": 0,
  "retired_for_future_evaluation": {
    "vvdex.knowledge.grounded-rag-1": {
      "environmentId": "vvdex.knowledge.grounded-rag-1",
      "historicalResults": "Historical sealed results on this revision are unaffected: the disclosure happened after those runs, so it could not have influenced them.",
      "reason": "post-run public disclosure of answer-bearing submission content",
      "retiredOn": "2026-09-02",
      "version": "0.1.0"
    },
    "vvdex.knowledge.memory-fact-update-1": {
      "environmentId": "vvdex.knowledge.memory-fact-update-1",
      "historicalResults": "Historical sealed results on this revision are unaffected: the disclosure happened after those runs, so it could not have influenced them.",
      "reason": "post-run public disclosure of answer-bearing submission content",
      "retiredOn": "2026-09-02",
      "version": "0.1.0"
    }
  },
  "schema": "vvdex.forge.campaign-summary/v1",
  "suspectRecords": [],
  "withheldByHarness": 0
}