{
  "campaignId": "fc-3194f803055c",
  "counts": {
    "attempts": 90,
    "cells": 9,
    "certifiedPasses": 52,
    "exams": 3,
    "graded": 89,
    "invalid": 0,
    "laneErrors": 1,
    "lanes": 3,
    "modelFails": 37,
    "withheld": 0
  },
  "disclosure": {
    "public.swe.martinblech-xmltodict-issue-257": {
      "class": "public_source",
      "rule": "public-source disclosure (ownership public_source declared by engine registry, upstream https://github.com/martinblech/xmltodict under MIT; family public_oss_swe): the submitted change is a patch against a public repository, so a redacted excerpt of it is published."
    },
    "vvdex.knowledge.grounded-rag-1": {
      "class": "answer_key",
      "rule": "answer-key disclosure (ownership vvdex_proprietary \u2014 engine registry; family rag_grounding): the exam is VVDex evaluation IP and the graded output is an answer or a keyed value, so no submission content, diff, model claim or hidden-assertion name is published."
    },
    "vvdex.knowledge.memory-fact-update-1": {
      "class": "answer_key",
      "rule": "answer-key disclosure (ownership vvdex_proprietary \u2014 engine registry; family long_term_memory): the exam is VVDex evaluation IP and the graded output is an answer or a keyed value, so no submission content, diff, model claim or hidden-assertion name is published."
    }
  },
  "families": [
    {
      "envId": "public.swe.martinblech-xmltodict-issue-257",
      "family": "public_oss_swe",
      "label": "Software repair (public OSS SWE)",
      "short": "SWE",
      "what": "A real software-engineering task taken from a public open-source repository: the model is given the codebase and a written issue, and must change the code so that the behaviour described in the issue is fixed."
    },
    {
      "envId": "vvdex.knowledge.grounded-rag-1",
      "family": "rag_grounding",
      "label": "Grounded retrieval (RAG)",
      "short": "RAG",
      "what": "Grounded question answering over a small document collection."
    },
    {
      "envId": "vvdex.knowledge.memory-fact-update-1",
      "family": "long_term_memory",
      "label": "Cross-session memory update",
      "short": "Memory",
      "what": "A state-and-memory task across sessions: a fact was recorded earlier, an authoritative source has since changed it, and the model must answer from the newer authoritative roster rather than the stale carried memory, and acknowledge the change."
    }
  ],
  "lanes": [
    {
      "certifiedPasses": 30,
      "countable": 30,
      "counts": {
        "certified_pass": 30,
        "invalid": 0,
        "lane_error": 0,
        "model_fail": 0,
        "withheld": 0
      },
      "lane": "cli/codex",
      "rate": 1.0,
      "rateInterval95": [
        0.8865,
        1.0
      ]
    },
    {
      "certifiedPasses": 20,
      "countable": 30,
      "counts": {
        "certified_pass": 20,
        "invalid": 0,
        "lane_error": 0,
        "model_fail": 10,
        "withheld": 0
      },
      "lane": "cli/claude-sonnet",
      "rate": 0.6666666666666666,
      "rateInterval95": [
        0.4878,
        0.8077
      ]
    },
    {
      "certifiedPasses": 2,
      "countable": 29,
      "counts": {
        "certified_pass": 2,
        "invalid": 0,
        "lane_error": 1,
        "model_fail": 27,
        "withheld": 0
      },
      "lane": "codestral-latest",
      "rate": 0.06896551724137931,
      "rateInterval95": [
        0.0191,
        0.2196
      ]
    }
  ],
  "perExamMatrix": {
    "exams": [
      {
        "envId": "public.swe.martinblech-xmltodict-issue-257",
        "evalId": "fr-20260902-6616b188",
        "family": "public_oss_swe",
        "short": "SWE",
        "title": "Streaming items must equal what the full parse would have built"
      },
      {
        "envId": "vvdex.knowledge.grounded-rag-1",
        "evalId": "fr-20260902-2e7f4171",
        "family": "rag_grounding",
        "short": "RAG",
        "title": "Answering questions from a document collection with outdated and distractor sources, and refusing when the evidence is insufficient"
      },
      {
        "envId": "vvdex.knowledge.memory-fact-update-1",
        "evalId": "fr-20260901-46eb9548",
        "family": "long_term_memory",
        "short": "Memory",
        "title": "Updating a cross-session fact: answer from the newer authoritative roster, not stale carried memory"
      }
    ],
    "rows": [
      {
        "cells": {
          "public.swe.martinblech-xmltodict-issue-257": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 10,
            "text": "10/10",
            "withheld": 0
          },
          "vvdex.knowledge.grounded-rag-1": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 10,
            "text": "10/10",
            "withheld": 0
          },
          "vvdex.knowledge.memory-fact-update-1": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 10,
            "text": "10/10",
            "withheld": 0
          }
        },
        "certifiedPasses": 30,
        "lane": "cli/codex"
      },
      {
        "cells": {
          "public.swe.martinblech-xmltodict-issue-257": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 9,
            "text": "9/10",
            "withheld": 0
          },
          "vvdex.knowledge.grounded-rag-1": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 1,
            "text": "1/10",
            "withheld": 0
          },
          "vvdex.knowledge.memory-fact-update-1": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 10,
            "text": "10/10",
            "withheld": 0
          }
        },
        "certifiedPasses": 20,
        "lane": "cli/claude-sonnet"
      },
      {
        "cells": {
          "public.swe.martinblech-xmltodict-issue-257": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 0,
            "text": "0/10",
            "withheld": 0
          },
          "vvdex.knowledge.grounded-rag-1": {
            "graded": 9,
            "invalid": 0,
            "laneErrors": 1,
            "passes": 0,
            "text": "0/9 + 1 lane error",
            "withheld": 0
          },
          "vvdex.knowledge.memory-fact-update-1": {
            "graded": 10,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 2,
            "text": "2/10",
            "withheld": 0
          }
        },
        "certifiedPasses": 2,
        "lane": "codestral-latest"
      }
    ]
  },
  "recordCount": 3,
  "records": [
    {
      "canonicalBytesPublished": false,
      "certificationReceiptSha256": "87ba9833637f004939e1a7b2d3a69b2d0e0e37b1040ef5d403eb9c392695bee0",
      "disclosureClass": "public_source",
      "environmentId": "public.swe.martinblech-xmltodict-issue-257",
      "evalId": "fr-20260902-6616b188",
      "examFingerprint": "aa41977120e6ecdddb5c4fc603e8ab23cb272726a86750119969ff4dfc90e297",
      "recordDigest": "138425da59aabc18aa208c339aa41919f387e1ecb80fb725cfed8f7d6ba78713",
      "retired_for_future_evaluation": null
    },
    {
      "canonicalBytesPublished": false,
      "certificationReceiptSha256": "95deabd2469ddea08d716e0cc5c2c966b874be73743798020aeedaa2fd99d70f",
      "disclosureClass": "answer_key",
      "environmentId": "vvdex.knowledge.grounded-rag-1",
      "evalId": "fr-20260902-2e7f4171",
      "examFingerprint": "4880a527ef97ca2f3fe6b57be026c3dc1be51e9e104d79fc13a3e558331dda92",
      "recordDigest": "157ec324a925c0b10fc8769d01f628b6143c40a00fa2dc70ea309a70e55a37a8",
      "retired_for_future_evaluation": {
        "environmentId": "vvdex.knowledge.grounded-rag-1",
        "historicalResults": "Historical sealed results on this revision are unaffected: the disclosure happened after those runs, so it could not have influenced them.",
        "reason": "post-run public disclosure of answer-bearing submission content",
        "retiredOn": "2026-09-02",
        "version": "0.1.0"
      }
    },
    {
      "canonicalBytesPublished": false,
      "certificationReceiptSha256": "0e21831617f6d3b3680450adc93ec0aa0367a79dfc6ed07538d8a69a05a104ee",
      "disclosureClass": "answer_key",
      "environmentId": "vvdex.knowledge.memory-fact-update-1",
      "evalId": "fr-20260901-46eb9548",
      "examFingerprint": "2052eec0ff2d3044dc269e3599ae1bfaa45d0d8cd5df9554796105538d226c21",
      "recordDigest": "1127e8356d42d31767a3ddc4d6c6cd0a515f5bbf7239ef0a82129e00fc2e6900",
      "retired_for_future_evaluation": {
        "environmentId": "vvdex.knowledge.memory-fact-update-1",
        "historicalResults": "Historical sealed results on this revision are unaffected: the disclosure happened after those runs, so it could not have influenced them.",
        "reason": "post-run public disclosure of answer-bearing submission content",
        "retiredOn": "2026-09-02",
        "version": "0.1.0"
      }
    }
  ],
  "refused": 0,
  "retired_for_future_evaluation": {
    "vvdex.knowledge.grounded-rag-1": {
      "environmentId": "vvdex.knowledge.grounded-rag-1",
      "historicalResults": "Historical sealed results on this revision are unaffected: the disclosure happened after those runs, so it could not have influenced them.",
      "reason": "post-run public disclosure of answer-bearing submission content",
      "retiredOn": "2026-09-02",
      "version": "0.1.0"
    },
    "vvdex.knowledge.memory-fact-update-1": {
      "environmentId": "vvdex.knowledge.memory-fact-update-1",
      "historicalResults": "Historical sealed results on this revision are unaffected: the disclosure happened after those runs, so it could not have influenced them.",
      "reason": "post-run public disclosure of answer-bearing submission content",
      "retiredOn": "2026-09-02",
      "version": "0.1.0"
    }
  },
  "schema": "vvdex.forge.campaign-summary/v1",
  "suspectRecords": [],
  "withheldByHarness": 0
}