{
  "campaignId": "fc-8626f712e26f",
  "counts": {
    "attempts": 20,
    "cells": 20,
    "certifiedPasses": 8,
    "exams": 5,
    "graded": 16,
    "invalid": 0,
    "laneErrors": 4,
    "lanes": 4,
    "modelFails": 8,
    "withheld": 0
  },
  "disclosure": {
    "public.swe.martinblech-xmltodict-issue-257": {
      "class": "public_source",
      "rule": "public-source disclosure (ownership public_source declared by engine registry, upstream https://github.com/martinblech/xmltodict under MIT; family public_oss_swe): the submitted change is a patch against a public repository, so a redacted excerpt of it is published."
    },
    "public.swe.pytoolz-toolz-issue-602": {
      "class": "answer_key",
      "rule": "answer-key disclosure (ownership vvdex_proprietary \u2014 no declaration \u2014 proprietary by default; family public_oss_swe): the exam is VVDex evaluation IP and the graded output is an answer or a keyed value, so no submission content, diff, model claim or hidden-assertion name is published."
    },
    "public.swe.r1chardj0n3s-parse-issue-102": {
      "class": "answer_key",
      "rule": "answer-key disclosure (ownership vvdex_proprietary \u2014 no declaration \u2014 proprietary by default; family public_oss_swe): the exam is VVDex evaluation IP and the graded output is an answer or a keyed value, so no submission content, diff, model claim or hidden-assertion name is published."
    },
    "vvdex.knowledge.grounded-rag-1": {
      "class": "answer_key",
      "rule": "answer-key disclosure (ownership vvdex_proprietary \u2014 engine registry; family rag_grounding): the exam is VVDex evaluation IP and the graded output is an answer or a keyed value, so no submission content, diff, model claim or hidden-assertion name is published."
    },
    "vvdex.knowledge.memory-fact-update-1": {
      "class": "answer_key",
      "rule": "answer-key disclosure (ownership vvdex_proprietary \u2014 engine registry; family long_term_memory): the exam is VVDex evaluation IP and the graded output is an answer or a keyed value, so no submission content, diff, model claim or hidden-assertion name is published."
    }
  },
  "families": [
    {
      "envId": "public.swe.martinblech-xmltodict-issue-257",
      "family": "public_oss_swe",
      "label": "Software repair (public OSS SWE)",
      "short": "SWE \u00b7 xmltodict-257",
      "what": "A real software-engineering task taken from a public open-source repository: the model is given the codebase and a written issue, and must change the code so that the behaviour described in the issue is fixed."
    },
    {
      "envId": "public.swe.pytoolz-toolz-issue-602",
      "family": "public_oss_swe",
      "label": "Software repair (public OSS SWE)",
      "short": "SWE \u00b7 toolz-602",
      "what": "A real software-engineering task taken from a public open-source repository: the model is given the codebase and a written issue, and must change the code so that the behaviour described in the issue is fixed."
    },
    {
      "envId": "public.swe.r1chardj0n3s-parse-issue-102",
      "family": "public_oss_swe",
      "label": "Software repair (public OSS SWE)",
      "short": "SWE \u00b7 parse-102",
      "what": "A real software-engineering task taken from a public open-source repository: the model is given the codebase and a written issue, and must change the code so that the behaviour described in the issue is fixed."
    },
    {
      "envId": "vvdex.knowledge.grounded-rag-1",
      "family": "rag_grounding",
      "label": "Grounded retrieval (RAG)",
      "short": "RAG",
      "what": "Grounded question answering over a small document collection."
    },
    {
      "envId": "vvdex.knowledge.memory-fact-update-1",
      "family": "long_term_memory",
      "label": "Cross-session memory update",
      "short": "Memory",
      "what": "A state-and-memory task across sessions: a fact was recorded earlier, an authoritative source has since changed it, and the model must answer from the newer authoritative roster rather than the stale carried memory, and acknowledge the change."
    }
  ],
  "lanes": [
    {
      "certifiedPasses": 4,
      "countable": 5,
      "counts": {
        "certified_pass": 4,
        "invalid": 0,
        "lane_error": 0,
        "model_fail": 1,
        "withheld": 0
      },
      "lane": "cli/claude-sonnet"
    },
    {
      "certifiedPasses": 3,
      "countable": 5,
      "counts": {
        "certified_pass": 3,
        "invalid": 0,
        "lane_error": 0,
        "model_fail": 2,
        "withheld": 0
      },
      "lane": "cli/codex"
    },
    {
      "certifiedPasses": 1,
      "countable": 1,
      "counts": {
        "certified_pass": 1,
        "invalid": 0,
        "lane_error": 4,
        "model_fail": 0,
        "withheld": 0
      },
      "lane": "openai/gpt-oss-120b"
    },
    {
      "certifiedPasses": 0,
      "countable": 5,
      "counts": {
        "certified_pass": 0,
        "invalid": 0,
        "lane_error": 0,
        "model_fail": 5,
        "withheld": 0
      },
      "lane": "codestral-latest"
    }
  ],
  "perExamMatrix": {
    "exams": [
      {
        "envId": "public.swe.martinblech-xmltodict-issue-257",
        "evalId": "fr-20260901-a27316a3",
        "family": "public_oss_swe",
        "short": "SWE \u00b7 xmltodict-257",
        "title": "Streaming items must equal what the full parse would have built"
      },
      {
        "envId": "public.swe.pytoolz-toolz-issue-602",
        "evalId": "fr-20260901-88c3cdf1",
        "family": "public_oss_swe",
        "short": "SWE \u00b7 toolz-602",
        "title": "Trusting an inaccurate __len__ in partition_all"
      },
      {
        "envId": "public.swe.r1chardj0n3s-parse-issue-102",
        "evalId": "fr-20260901-4b1e3251",
        "family": "public_oss_swe",
        "short": "SWE \u00b7 parse-102",
        "title": "Signs and base indicators in the integer converter"
      },
      {
        "envId": "vvdex.knowledge.grounded-rag-1",
        "evalId": "fr-20260901-718db345",
        "family": "rag_grounding",
        "short": "RAG",
        "title": "Answering questions from a document collection with outdated and distractor sources, and refusing when the evidence is insufficient"
      },
      {
        "envId": "vvdex.knowledge.memory-fact-update-1",
        "evalId": "fr-20260901-109cfa3c",
        "family": "long_term_memory",
        "short": "Memory",
        "title": "Updating a cross-session fact: answer from the newer authoritative roster, not stale carried memory"
      }
    ],
    "rows": [
      {
        "cells": {
          "public.swe.martinblech-xmltodict-issue-257": {
            "graded": 1,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 1,
            "text": "1/1",
            "withheld": 0
          },
          "public.swe.pytoolz-toolz-issue-602": {
            "graded": 1,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 1,
            "text": "1/1",
            "withheld": 0
          },
          "public.swe.r1chardj0n3s-parse-issue-102": {
            "graded": 1,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 1,
            "text": "1/1",
            "withheld": 0
          },
          "vvdex.knowledge.grounded-rag-1": {
            "graded": 1,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 0,
            "text": "0/1",
            "withheld": 0
          },
          "vvdex.knowledge.memory-fact-update-1": {
            "graded": 1,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 1,
            "text": "1/1",
            "withheld": 0
          }
        },
        "certifiedPasses": 4,
        "lane": "cli/claude-sonnet"
      },
      {
        "cells": {
          "public.swe.martinblech-xmltodict-issue-257": {
            "graded": 1,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 1,
            "text": "1/1",
            "withheld": 0
          },
          "public.swe.pytoolz-toolz-issue-602": {
            "graded": 1,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 0,
            "text": "0/1",
            "withheld": 0
          },
          "public.swe.r1chardj0n3s-parse-issue-102": {
            "graded": 1,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 0,
            "text": "0/1",
            "withheld": 0
          },
          "vvdex.knowledge.grounded-rag-1": {
            "graded": 1,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 1,
            "text": "1/1",
            "withheld": 0
          },
          "vvdex.knowledge.memory-fact-update-1": {
            "graded": 1,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 1,
            "text": "1/1",
            "withheld": 0
          }
        },
        "certifiedPasses": 3,
        "lane": "cli/codex"
      },
      {
        "cells": {
          "public.swe.martinblech-xmltodict-issue-257": {
            "graded": 0,
            "invalid": 0,
            "laneErrors": 1,
            "passes": 0,
            "text": "0/0 + 1 lane error",
            "withheld": 0
          },
          "public.swe.pytoolz-toolz-issue-602": {
            "graded": 0,
            "invalid": 0,
            "laneErrors": 1,
            "passes": 0,
            "text": "0/0 + 1 lane error",
            "withheld": 0
          },
          "public.swe.r1chardj0n3s-parse-issue-102": {
            "graded": 0,
            "invalid": 0,
            "laneErrors": 1,
            "passes": 0,
            "text": "0/0 + 1 lane error",
            "withheld": 0
          },
          "vvdex.knowledge.grounded-rag-1": {
            "graded": 0,
            "invalid": 0,
            "laneErrors": 1,
            "passes": 0,
            "text": "0/0 + 1 lane error",
            "withheld": 0
          },
          "vvdex.knowledge.memory-fact-update-1": {
            "graded": 1,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 1,
            "text": "1/1",
            "withheld": 0
          }
        },
        "certifiedPasses": 1,
        "lane": "openai/gpt-oss-120b"
      },
      {
        "cells": {
          "public.swe.martinblech-xmltodict-issue-257": {
            "graded": 1,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 0,
            "text": "0/1",
            "withheld": 0
          },
          "public.swe.pytoolz-toolz-issue-602": {
            "graded": 1,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 0,
            "text": "0/1",
            "withheld": 0
          },
          "public.swe.r1chardj0n3s-parse-issue-102": {
            "graded": 1,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 0,
            "text": "0/1",
            "withheld": 0
          },
          "vvdex.knowledge.grounded-rag-1": {
            "graded": 1,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 0,
            "text": "0/1",
            "withheld": 0
          },
          "vvdex.knowledge.memory-fact-update-1": {
            "graded": 1,
            "invalid": 0,
            "laneErrors": 0,
            "passes": 0,
            "text": "0/1",
            "withheld": 0
          }
        },
        "certifiedPasses": 0,
        "lane": "codestral-latest"
      }
    ]
  },
  "recordCount": 5,
  "records": [
    {
      "canonicalBytesPublished": false,
      "certificationReceiptSha256": "87ba9833637f004939e1a7b2d3a69b2d0e0e37b1040ef5d403eb9c392695bee0",
      "disclosureClass": "public_source",
      "environmentId": "public.swe.martinblech-xmltodict-issue-257",
      "evalId": "fr-20260901-a27316a3",
      "examFingerprint": "aa41977120e6ecdddb5c4fc603e8ab23cb272726a86750119969ff4dfc90e297",
      "recordDigest": "96f8b631c6d65f03b21e4a9a94e86c82874e1506fac38c9a093d2f8c618dd2dc",
      "retired_for_future_evaluation": null
    },
    {
      "canonicalBytesPublished": false,
      "certificationReceiptSha256": null,
      "disclosureClass": "answer_key",
      "environmentId": "public.swe.pytoolz-toolz-issue-602",
      "evalId": "fr-20260901-88c3cdf1",
      "examFingerprint": "e8312ce98b82558f9a1c5593e7d431f409fb1024b3013b7dbd934f1398a8937f",
      "recordDigest": "f49a31336e4865c1dcefccf5141e2f6926110c72f03d6cb3d645a1761fa09ac8",
      "retired_for_future_evaluation": null
    },
    {
      "canonicalBytesPublished": false,
      "certificationReceiptSha256": null,
      "disclosureClass": "answer_key",
      "environmentId": "public.swe.r1chardj0n3s-parse-issue-102",
      "evalId": "fr-20260901-4b1e3251",
      "examFingerprint": "483b68b383c851060781196403325de0e00e5f6327fd52d24eb581005be38812",
      "recordDigest": "d94c313144f8b36db4448440b873462bc9372e1783b1164451695b7b8497816a",
      "retired_for_future_evaluation": null
    },
    {
      "canonicalBytesPublished": false,
      "certificationReceiptSha256": "95deabd2469ddea08d716e0cc5c2c966b874be73743798020aeedaa2fd99d70f",
      "disclosureClass": "answer_key",
      "environmentId": "vvdex.knowledge.grounded-rag-1",
      "evalId": "fr-20260901-718db345",
      "examFingerprint": "4880a527ef97ca2f3fe6b57be026c3dc1be51e9e104d79fc13a3e558331dda92",
      "recordDigest": "a6426b0b7932b6c0f679f230c44c4b75a30df10a17f7e145b567ac9849d63a36",
      "retired_for_future_evaluation": {
        "environmentId": "vvdex.knowledge.grounded-rag-1",
        "historicalResults": "Historical sealed results on this revision are unaffected: the disclosure happened after those runs, so it could not have influenced them.",
        "reason": "post-run public disclosure of answer-bearing submission content",
        "retiredOn": "2026-09-02",
        "version": "0.1.0"
      }
    },
    {
      "canonicalBytesPublished": false,
      "certificationReceiptSha256": "0e21831617f6d3b3680450adc93ec0aa0367a79dfc6ed07538d8a69a05a104ee",
      "disclosureClass": "answer_key",
      "environmentId": "vvdex.knowledge.memory-fact-update-1",
      "evalId": "fr-20260901-109cfa3c",
      "examFingerprint": "2052eec0ff2d3044dc269e3599ae1bfaa45d0d8cd5df9554796105538d226c21",
      "recordDigest": "6354cff6fc47eca13b596d285f805a9f3994d048d69b446166acfeb7f8659929",
      "retired_for_future_evaluation": {
        "environmentId": "vvdex.knowledge.memory-fact-update-1",
        "historicalResults": "Historical sealed results on this revision are unaffected: the disclosure happened after those runs, so it could not have influenced them.",
        "reason": "post-run public disclosure of answer-bearing submission content",
        "retiredOn": "2026-09-02",
        "version": "0.1.0"
      }
    }
  ],
  "refused": 0,
  "retired_for_future_evaluation": {
    "vvdex.knowledge.grounded-rag-1": {
      "environmentId": "vvdex.knowledge.grounded-rag-1",
      "historicalResults": "Historical sealed results on this revision are unaffected: the disclosure happened after those runs, so it could not have influenced them.",
      "reason": "post-run public disclosure of answer-bearing submission content",
      "retiredOn": "2026-09-02",
      "version": "0.1.0"
    },
    "vvdex.knowledge.memory-fact-update-1": {
      "environmentId": "vvdex.knowledge.memory-fact-update-1",
      "historicalResults": "Historical sealed results on this revision are unaffected: the disclosure happened after those runs, so it could not have influenced them.",
      "reason": "post-run public disclosure of answer-bearing submission content",
      "retiredOn": "2026-09-02",
      "version": "0.1.0"
    }
  },
  "schema": "vvdex.forge.campaign-summary/v1",
  "suspectRecords": [],
  "withheldByHarness": 0
}