HISTORICAL RECORD · 2026-10-08 · invoice-06

Cloud storage license $250

Cloud storage license $250. All required fields are present.

Synthetic case. References were set by ClefBench and have not received an independent second review. These are recorded results, not a new model call.

urgent

Does this invoice need review because any required field is missing, supplier or banking details are unverified, or fraud is confirmed?

Reference: false

Model	Run 1	Run 2	Run 3	Changed answer?
Clef	false	false	false	No
Clef-flash	false	false	false	No
Jev 1.13	false	false	false	No

Yes/no uses a 50% threshold. Score values are weighted scale indices; error flags and stability use the nearest level. Use the rule and full distribution to judge whether a difference matters for your task.

Model	Run 1	Run 2	Run 3	Changed answer?
Clef	software	software	software	No
Clef-flash	software	software	software	No
Jev 1.13	software	software	software	No

severity

Rate invoice risk from the stated evidence.

Reference: 0

0: Complete invoice with no reported issue
1: A required invoice field is missing
2: Supplier identity or banking details are unverified
3: Fraud, forgery or fabrication is confirmed

Model	Run 1	Run 2	Run 3	Changed answer?
Clef	0.154	0.154	0.154	No
Clef-flash	0.1398	0.1398	0.1398	No
Jev 1.13	0.6Differs from reference	0.58Differs from reference	0.48	Yes

Time and provider cost for this case

Clef · run 1: 514 ms · $0.00009984
Clef-flash · run 1: 439 ms · $0.00003744
Jev 1.13 · run 1: 348 ms · $0.00001978
Clef-flash · run 2: 408 ms · $0.00003744
Jev 1.13 · run 2: 374 ms · $0.00001978
Clef · run 2: 473 ms · $0.00009984
Jev 1.13 · run 3: 378 ms · $0.00001978
Clef · run 3: 510 ms · $0.00009984
Clef-flash · run 3: 384 ms · $0.00003744

Full task, reference answers and raw recorded responses

{
  "sample": {
    "state": "Cloud storage license $250. All required fields are present.",
    "questions": {
      "urgent": {
        "type": "noul",
        "instructions": "Does this invoice need review because any required field is missing, supplier or banking details are unverified, or fraud is confirmed?"
      },
      "category": {
        "type": "choice",
        "instructions": "Classify the purchased item.",
        "criteria": {
          "hardware": "Physical computers, printers, monitors or servers",
          "software": "Software licenses, applications or cloud storage subscriptions",
          "services": "Professional or contractor work",
          "travel": "Transport, accommodation or taxi expenses"
        }
      },
      "severity": {
        "type": "score",
        "instructions": "Rate invoice risk from the stated evidence.",
        "criteria": [
          "Complete invoice with no reported issue",
          "A required invoice field is missing",
          "Supplier identity or banking details are unverified",
          "Fraud, forgery or fabrication is confirmed"
        ]
      }
    }
  },
  "reference": {
    "urgent": false,
    "category": "software",
    "severity": 0
  },
  "runs": [
    {
      "caseId": "invoice-06",
      "repeat": 1,
      "outcome": {
        "model": "clef",
        "measuredAt": "2026-10-08T09:26:31.639Z",
        "durationMs": 514,
        "status": "success",
        "result": {
          "id": "gen-dec-1791451591-6WmAdS18c5mhqQCEtSFi",
          "model": "cloudflare/clef",
          "provider": "Cloudflare",
          "answers": {
            "urgent": {
              "type": "noul",
              "noul": 0.0265
            },
            "category": {
              "type": "choice",
              "choice": "software",
              "probabilities": {
                "hardware": 0.0025,
                "software": 0.9915,
                "services": 0.0045,
                "travel": 0.0015
              },
              "confidence": 0.9775
            },
            "severity": {
              "type": "score",
              "score": 0.154,
              "probabilities": {
                "0": 0.9202,
                "1": 0.0173,
                "2": 0.0509,
                "3": 0.0116
              },
              "confidence": 0.7996,
              "legend": {
                "0": "Complete invoice with no reported issue",
                "1": "A required invoice field is missing",
                "2": "Supplier identity or banking details are unverified",
                "3": "Fraud, forgery or fabrication is confirmed"
              }
            }
          },
          "usage": {
            "input_tokens": 416,
            "output_tokens": 0,
            "cost": 0.00009984
          }
        }
      }
    },
    {
      "caseId": "invoice-06",
      "repeat": 1,
      "outcome": {
        "model": "clef-flash",
        "measuredAt": "2026-10-08T09:26:32.162Z",
        "durationMs": 439,
        "status": "success",
        "result": {
          "id": "gen-dec-1791451592-QjiMK1htfpGCtZohKamF",
          "model": "cloudflare/clef-flash",
          "provider": "Cloudflare",
          "answers": {
            "urgent": {
              "type": "noul",
              "noul": 0.015
            },
            "category": {
              "type": "choice",
              "choice": "software",
              "probabilities": {
                "hardware": 0.0061,
                "software": 0.9703,
                "services": 0.0193,
                "travel": 0.0043
              },
              "confidence": 0.9224
            },
            "severity": {
              "type": "score",
              "score": 0.1398,
              "probabilities": {
                "0": 0.9216,
                "1": 0.0275,
                "2": 0.0405,
                "3": 0.0104
              },
              "confidence": 0.8024,
              "legend": {
                "0": "Complete invoice with no reported issue",
                "1": "A required invoice field is missing",
                "2": "Supplier identity or banking details are unverified",
                "3": "Fraud, forgery or fabrication is confirmed"
              }
            }
          },
          "usage": {
            "input_tokens": 416,
            "output_tokens": 0,
            "cost": 0.00003744
          }
        }
      }
    },
    {
      "caseId": "invoice-06",
      "repeat": 1,
      "outcome": {
        "model": "jev",
        "measuredAt": "2026-10-08T09:26:32.610Z",
        "durationMs": 348,
        "status": "success",
        "result": {
          "id": "gen-dec-1791451592-VXOVcwAXIIhRsOesBldw",
          "model": "typesafe/jev-1.13-20260917",
          "provider": "TypeSafe",
          "answers": {
            "urgent": {
              "type": "noul",
              "noul": 0.1
            },
            "category": {
              "type": "choice",
              "choice": "software",
              "probabilities": {
                "hardware": 0,
                "software": 1,
                "services": 0,
                "travel": 0
              },
              "confidence": 1
            },
            "severity": {
              "type": "score",
              "score": 0.6,
              "probabilities": {
                "0": 0.68,
                "1": 0.04,
                "2": 0.28,
                "3": 0
              },
              "confidence": 0.4,
              "legend": {
                "0": "Complete invoice with no reported issue",
                "1": "A required invoice field is missing",
                "2": "Supplier identity or banking details are unverified",
                "3": "Fraud, forgery or fabrication is confirmed"
              }
            }
          },
          "usage": {
            "input_tokens": 471,
            "output_tokens": 75,
            "cost": 0.000019782
          }
        }
      }
    },
    {
      "caseId": "invoice-06",
      "repeat": 2,
      "outcome": {
        "model": "clef-flash",
        "measuredAt": "2026-10-08T09:28:03.374Z",
        "durationMs": 408,
        "status": "success",
        "result": {
          "id": "gen-dec-1791451683-HJ3KjVZ0GG0TMvIgXEFO",
          "model": "cloudflare/clef-flash",
          "provider": "Cloudflare",
          "answers": {
            "urgent": {
              "type": "noul",
              "noul": 0.015
            },
            "category": {
              "type": "choice",
              "choice": "software",
              "probabilities": {
                "hardware": 0.0061,
                "software": 0.9703,
                "services": 0.0193,
                "travel": 0.0043
              },
              "confidence": 0.9224
            },
            "severity": {
              "type": "score",
              "score": 0.1398,
              "probabilities": {
                "0": 0.9216,
                "1": 0.0275,
                "2": 0.0405,
                "3": 0.0104
              },
              "confidence": 0.8024,
              "legend": {
                "0": "Complete invoice with no reported issue",
                "1": "A required invoice field is missing",
                "2": "Supplier identity or banking details are unverified",
                "3": "Fraud, forgery or fabrication is confirmed"
              }
            }
          },
          "usage": {
            "input_tokens": 416,
            "output_tokens": 0,
            "cost": 0.00003744
          }
        }
      }
    },
    {
      "caseId": "invoice-06",
      "repeat": 2,
      "outcome": {
        "model": "jev",
        "measuredAt": "2026-10-08T09:28:03.793Z",
        "durationMs": 374,
        "status": "success",
        "result": {
          "id": "gen-dec-1791451683-fgwMPjKodyUO7VbT7Yoh",
          "model": "typesafe/jev-1.13-20260917",
          "provider": "TypeSafe",
          "answers": {
            "urgent": {
              "type": "noul",
              "noul": 0.1
            },
            "category": {
              "type": "choice",
              "choice": "software",
              "probabilities": {
                "software": 1,
                "hardware": 0,
                "travel": 0,
                "services": 0
              },
              "confidence": 1
            },
            "severity": {
              "type": "score",
              "score": 0.58,
              "probabilities": {
                "0": 0.68,
                "1": 0.05,
                "2": 0.26,
                "3": 0
              },
              "confidence": 0.43,
              "legend": {
                "0": "Complete invoice with no reported issue",
                "1": "A required invoice field is missing",
                "2": "Supplier identity or banking details are unverified",
                "3": "Fraud, forgery or fabrication is confirmed"
              }
            }
          },
          "usage": {
            "input_tokens": 471,
            "output_tokens": 75,
            "cost": 0.000019782
          }
        }
      }
    },
    {
      "caseId": "invoice-06",
      "repeat": 2,
      "outcome": {
        "model": "clef",
        "measuredAt": "2026-10-08T09:28:04.180Z",
        "durationMs": 473,
        "status": "success",
        "result": {
          "id": "gen-dec-1791451684-AU6QiH3MYnsRrxYmdfX4",
          "model": "cloudflare/clef",
          "provider": "Cloudflare",
          "answers": {
            "urgent": {
              "type": "noul",
              "noul": 0.0265
            },
            "category": {
              "type": "choice",
              "choice": "software",
              "probabilities": {
                "hardware": 0.0025,
                "software": 0.9915,
                "services": 0.0045,
                "travel": 0.0015
              },
              "confidence": 0.9775
            },
            "severity": {
              "type": "score",
              "score": 0.154,
              "probabilities": {
                "0": 0.9202,
                "1": 0.0173,
                "2": 0.0509,
                "3": 0.0116
              },
              "confidence": 0.7996,
              "legend": {
                "0": "Complete invoice with no reported issue",
                "1": "A required invoice field is missing",
                "2": "Supplier identity or banking details are unverified",
                "3": "Fraud, forgery or fabrication is confirmed"
              }
            }
          },
          "usage": {
            "input_tokens": 416,
            "output_tokens": 0,
            "cost": 0.00009984
          }
        }
      }
    },
    {
      "caseId": "invoice-06",
      "repeat": 3,
      "outcome": {
        "model": "jev",
        "measuredAt": "2026-10-08T09:29:33.969Z",
        "durationMs": 378,
        "status": "success",
        "result": {
          "id": "gen-dec-1791451773-RBRK2sKhkOLqOD9N5eE7",
          "model": "typesafe/jev-1.13-20260917",
          "provider": "TypeSafe",
          "answers": {
            "urgent": {
              "type": "noul",
              "noul": 0.09
            },
            "category": {
              "type": "choice",
              "choice": "software",
              "probabilities": {
                "travel": 0,
                "software": 1,
                "hardware": 0,
                "services": 0
              },
              "confidence": 1
            },
            "severity": {
              "type": "score",
              "score": 0.48,
              "probabilities": {
                "0": 0.75,
                "1": 0.03,
                "2": 0.22,
                "3": 0
              },
              "confidence": 0.53,
              "legend": {
                "0": "Complete invoice with no reported issue",
                "1": "A required invoice field is missing",
                "2": "Supplier identity or banking details are unverified",
                "3": "Fraud, forgery or fabrication is confirmed"
              }
            }
          },
          "usage": {
            "input_tokens": 471,
            "output_tokens": 75,
            "cost": 0.000019782
          }
        }
      }
    },
    {
      "caseId": "invoice-06",
      "repeat": 3,
      "outcome": {
        "model": "clef",
        "measuredAt": "2026-10-08T09:29:34.362Z",
        "durationMs": 510,
        "status": "success",
        "result": {
          "id": "gen-dec-1791451774-EmoPMT52ydXqvpVbq9yn",
          "model": "cloudflare/clef",
          "provider": "Cloudflare",
          "answers": {
            "urgent": {
              "type": "noul",
              "noul": 0.0265
            },
            "category": {
              "type": "choice",
              "choice": "software",
              "probabilities": {
                "hardware": 0.0025,
                "software": 0.9915,
                "services": 0.0045,
                "travel": 0.0015
              },
              "confidence": 0.9775
            },
            "severity": {
              "type": "score",
              "score": 0.154,
              "probabilities": {
                "0": 0.9202,
                "1": 0.0173,
                "2": 0.0509,
                "3": 0.0116
              },
              "confidence": 0.7996,
              "legend": {
                "0": "Complete invoice with no reported issue",
                "1": "A required invoice field is missing",
                "2": "Supplier identity or banking details are unverified",
                "3": "Fraud, forgery or fabrication is confirmed"
              }
            }
          },
          "usage": {
            "input_tokens": 416,
            "output_tokens": 0,
            "cost": 0.00009984
          }
        }
      }
    },
    {
      "caseId": "invoice-06",
      "repeat": 3,
      "outcome": {
        "model": "clef-flash",
        "measuredAt": "2026-10-08T09:29:34.889Z",
        "durationMs": 384,
        "status": "success",
        "result": {
          "id": "gen-dec-1791451774-4kDRlznRbYUBz9qfxtzr",
          "model": "cloudflare/clef-flash",
          "provider": "Cloudflare",
          "answers": {
            "urgent": {
              "type": "noul",
              "noul": 0.015
            },
            "category": {
              "type": "choice",
              "choice": "software",
              "probabilities": {
                "hardware": 0.0061,
                "software": 0.9703,
                "services": 0.0193,
                "travel": 0.0043
              },
              "confidence": 0.9224
            },
            "severity": {
              "type": "score",
              "score": 0.1398,
              "probabilities": {
                "0": 0.9216,
                "1": 0.0275,
                "2": 0.0405,
                "3": 0.0104
              },
              "confidence": 0.8024,
              "legend": {
                "0": "Complete invoice with no reported issue",
                "1": "A required invoice field is missing",
                "2": "Supplier identity or banking details are unverified",
                "3": "Fraud, forgery or fabrication is confirmed"
              }
            }
          },
          "usage": {
            "input_tokens": 416,
            "output_tokens": 0,
            "cost": 0.00003744
          }
        }
      }
    }
  ]
}

Download the complete original results