{
  "model": "LiquidAI/d1-omni-600M",
  "date_utc": "2026-10-09T18:12:26.401906+00:00",
  "scope": "Handwritten smoke fixtures, not a representative accuracy benchmark",
  "cases": [
    {
      "kind": "solid_color",
      "dtype": "torch.float16",
      "expected": "red",
      "actual": "green",
      "correct": false,
      "result": {
        "answers": {
          "color": {
            "type": "choice",
            "choice": "green",
            "confidence": 0.928965151309967,
            "probabilities": {
              "red": 0.04981369897723198,
              "blue": 0.012496860697865486,
              "green": 0.928965151309967,
              "white": 0.00872421357780695
            }
          }
        },
        "usage": {
          "input_tokens": 292,
          "output_tokens": 0
        }
      }
    },
    {
      "kind": "solid_color",
      "dtype": "torch.float16",
      "expected": "blue",
      "actual": "green",
      "correct": false,
      "result": {
        "answers": {
          "color": {
            "type": "choice",
            "choice": "green",
            "confidence": 0.934037983417511,
            "probabilities": {
              "red": 0.05008571594953537,
              "blue": 0.006966267712414265,
              "green": 0.934037983417511,
              "white": 0.008909991011023521
            }
          }
        },
        "usage": {
          "input_tokens": 292,
          "output_tokens": 0
        }
      }
    },
    {
      "kind": "solid_color",
      "dtype": "torch.float16",
      "expected": "green",
      "actual": "green",
      "correct": true,
      "result": {
        "answers": {
          "color": {
            "type": "choice",
            "choice": "green",
            "confidence": 0.9994038343429565,
            "probabilities": {
              "red": 0.00045202719047665596,
              "blue": 2.4333823603228666e-05,
              "green": 0.9994038343429565,
              "white": 0.00011977528629358858
            }
          }
        },
        "usage": {
          "input_tokens": 292,
          "output_tokens": 0
        }
      }
    },
    {
      "kind": "solid_color",
      "dtype": "torch.float16",
      "expected": "white",
      "actual": "green",
      "correct": false,
      "result": {
        "answers": {
          "color": {
            "type": "choice",
            "choice": "green",
            "confidence": 0.44759896397590637,
            "probabilities": {
              "red": 0.41396135091781616,
              "blue": 0.029178407043218613,
              "green": 0.44759896397590637,
              "white": 0.10926120728254318
            }
          }
        },
        "usage": {
          "input_tokens": 292,
          "output_tokens": 0
        }
      }
    },
    {
      "kind": "ticket",
      "dtype": "torch.float16",
      "input": "I was charged twice for my subscription. Please refund the second charge.",
      "expected": "billing",
      "actual": "billing",
      "correct": true,
      "result": {
        "answers": {
          "team": {
            "type": "choice",
            "choice": "billing",
            "confidence": 0.9639301896095276,
            "probabilities": {
              "billing": 0.9639301896095276,
              "technical": 0.0360698401927948
            }
          }
        },
        "usage": {
          "input_tokens": 53,
          "output_tokens": 0
        }
      }
    },
    {
      "kind": "ticket",
      "dtype": "torch.float16",
      "input": "Please send me the invoice for last month.",
      "expected": "billing",
      "actual": "billing",
      "correct": true,
      "result": {
        "answers": {
          "team": {
            "type": "choice",
            "choice": "billing",
            "confidence": 0.9956766963005066,
            "probabilities": {
              "billing": 0.9956766963005066,
              "technical": 0.004323293920606375
            }
          }
        },
        "usage": {
          "input_tokens": 49,
          "output_tokens": 0
        }
      }
    },
    {
      "kind": "ticket",
      "dtype": "torch.float16",
      "input": "My credit card payment failed. Can I pay by bank transfer?",
      "expected": "billing",
      "actual": "billing",
      "correct": true,
      "result": {
        "answers": {
          "team": {
            "type": "choice",
            "choice": "billing",
            "confidence": 0.7505978941917419,
            "probabilities": {
              "billing": 0.7505978941917419,
              "technical": 0.24940206110477448
            }
          }
        },
        "usage": {
          "input_tokens": 52,
          "output_tokens": 0
        }
      }
    },
    {
      "kind": "ticket",
      "dtype": "torch.float16",
      "input": "I cancelled my subscription but was charged again.",
      "expected": "billing",
      "actual": "billing",
      "correct": true,
      "result": {
        "answers": {
          "team": {
            "type": "choice",
            "choice": "billing",
            "confidence": 0.9110578298568726,
            "probabilities": {
              "billing": 0.9110578298568726,
              "technical": 0.08894211798906326
            }
          }
        },
        "usage": {
          "input_tokens": 48,
          "output_tokens": 0
        }
      }
    },
    {
      "kind": "ticket",
      "dtype": "torch.float16",
      "input": "The API returns a 500 error for every request.",
      "expected": "technical",
      "actual": "technical",
      "correct": true,
      "result": {
        "answers": {
          "team": {
            "type": "choice",
            "choice": "technical",
            "confidence": 0.9804524183273315,
            "probabilities": {
              "billing": 0.01954755373299122,
              "technical": 0.9804524183273315
            }
          }
        },
        "usage": {
          "input_tokens": 50,
          "output_tokens": 0
        }
      }
    },
    {
      "kind": "ticket",
      "dtype": "torch.float16",
      "input": "The desktop app crashes when I open a document.",
      "expected": "technical",
      "actual": "technical",
      "correct": true,
      "result": {
        "answers": {
          "team": {
            "type": "choice",
            "choice": "technical",
            "confidence": 0.9955695271492004,
            "probabilities": {
              "billing": 0.004430499859154224,
              "technical": 0.9955695271492004
            }
          }
        },
        "usage": {
          "input_tokens": 49,
          "output_tokens": 0
        }
      }
    },
    {
      "kind": "ticket",
      "dtype": "torch.float16",
      "input": "The webhook never arrives even though my endpoint returns 200.",
      "expected": "technical",
      "actual": "technical",
      "correct": true,
      "result": {
        "answers": {
          "team": {
            "type": "choice",
            "choice": "technical",
            "confidence": 0.9835844039916992,
            "probabilities": {
              "billing": 0.016415568068623543,
              "technical": 0.9835844039916992
            }
          }
        },
        "usage": {
          "input_tokens": 52,
          "output_tokens": 0
        }
      }
    },
    {
      "kind": "ticket",
      "dtype": "torch.float16",
      "input": "The SDK throws a type error when parsing the API response.",
      "expected": "technical",
      "actual": "technical",
      "correct": true,
      "result": {
        "answers": {
          "team": {
            "type": "choice",
            "choice": "technical",
            "confidence": 0.9877834916114807,
            "probabilities": {
              "billing": 0.012216485105454922,
              "technical": 0.9877834916114807
            }
          }
        },
        "usage": {
          "input_tokens": 53,
          "output_tokens": 0
        }
      }
    },
    {
      "kind": "solid_color",
      "dtype": "torch.float32",
      "expected": "red",
      "actual": "green",
      "correct": false,
      "result": {
        "answers": {
          "color": {
            "type": "choice",
            "choice": "green",
            "confidence": 0.9291023015975952,
            "probabilities": {
              "red": 0.049687761813402176,
              "blue": 0.012368598021566868,
              "green": 0.9291023015975952,
              "white": 0.008841420523822308
            }
          }
        },
        "usage": {
          "input_tokens": 292,
          "output_tokens": 0
        }
      }
    },
    {
      "kind": "solid_color",
      "dtype": "torch.float32",
      "expected": "blue",
      "actual": "green",
      "correct": false,
      "result": {
        "answers": {
          "color": {
            "type": "choice",
            "choice": "green",
            "confidence": 0.9358202219009399,
            "probabilities": {
              "red": 0.04859386757016182,
              "blue": 0.006748673506081104,
              "green": 0.9358202219009399,
              "white": 0.008837238885462284
            }
          }
        },
        "usage": {
          "input_tokens": 292,
          "output_tokens": 0
        }
      }
    },
    {
      "kind": "solid_color",
      "dtype": "torch.float32",
      "expected": "green",
      "actual": "green",
      "correct": true,
      "result": {
        "answers": {
          "color": {
            "type": "choice",
            "choice": "green",
            "confidence": 0.9993940591812134,
            "probabilities": {
              "red": 0.0004605857247952372,
              "blue": 2.4539482183172368e-05,
              "green": 0.9993940591812134,
              "white": 0.00012081659224350005
            }
          }
        },
        "usage": {
          "input_tokens": 292,
          "output_tokens": 0
        }
      }
    },
    {
      "kind": "solid_color",
      "dtype": "torch.float32",
      "expected": "white",
      "actual": "green",
      "correct": false,
      "result": {
        "answers": {
          "color": {
            "type": "choice",
            "choice": "green",
            "confidence": 0.4471806287765503,
            "probabilities": {
              "red": 0.4124033451080322,
              "blue": 0.0288605485111475,
              "green": 0.4471806287765503,
              "white": 0.11155548691749573
            }
          }
        },
        "usage": {
          "input_tokens": 292,
          "output_tokens": 0
        }
      }
    }
  ]
}