{
  "tool_id": "art-657-forecast-accuracy-scorer",
  "note": "Empty/default sample, five explicit mixed-category forecasts with a custom reference_probability, a small (n=3) sample with a high-confidence wrong call, and a perfect two-record calibration case.",
  "vectors": [
    {
      "name": "empty_input",
      "policy_parameters": {},
      "output_payload": {
        "n": 5,
        "brier_score": 0.08916,
        "log_score": 0.332187,
        "base_rate": 0.6,
        "reference_probability": 0.5,
        "brier_reference": 0.25,
        "brier_skill_score": 0.64336,
        "accuracy_class": "GOOD",
        "category_breakdown": [
          {
            "category": "economic_indicator",
            "n": 1,
            "brier_score": 0.0324,
            "log_score": 0.198451
          },
          {
            "category": "election_political",
            "n": 1,
            "brier_score": 0.0784,
            "log_score": 0.328504
          },
          {
            "category": "sports_competition",
            "n": 1,
            "brier_score": 0.2025,
            "log_score": 0.597837
          },
          {
            "category": "gaming_style_event",
            "n": 1,
            "brier_score": 0.1225,
            "log_score": 0.430783
          },
          {
            "category": "weather_climate",
            "n": 1,
            "brier_score": 0.01,
            "log_score": 0.105361
          },
          {
            "category": "other",
            "n": 0,
            "brier_score": null,
            "log_score": null
          }
        ],
        "warnings": [
          "DEFAULT_SAMPLE_USED"
        ],
        "category_note": "Category labels are an informational grouping for reporting purposes only. They do not determine, and are not derived from, contract eligibility, legality, or regulatory status under any authority -- supply your own category per forecast.",
        "scoring_note": "Brier score and logarithmic score are proper scoring rules: a forecaster cannot improve their expected score by reporting anything other than their true belief. Lower is better for both; 0 is a perfect score.",
        "disclaimer": "Not financial or legal advice. This tool scores forecast calibration only -- it does not determine contract eligibility, legality, or regulatory status under any authority. For informational purposes only."
      },
      "golden_hash": "46eb0b86af839e42770ebd2ccc96f169b0e69245aef02030a946e6268fab8771"
    },
    {
      "name": "five_mixed_categories",
      "policy_parameters": {
        "forecasts": [
          {
            "probability": 0.8,
            "outcome": 1,
            "category": "economic_indicator"
          },
          {
            "probability": 0.3,
            "outcome": 0,
            "category": "election_political"
          },
          {
            "probability": 0.6,
            "outcome": 1,
            "category": "sports_competition"
          },
          {
            "probability": 0.4,
            "outcome": 0,
            "category": "weather_climate"
          },
          {
            "probability": 0.7,
            "outcome": 1,
            "category": "gaming_style_event"
          }
        ],
        "reference_probability": 0.5
      },
      "output_payload": {
        "n": 5,
        "brier_score": 0.108,
        "log_score": 0.391629,
        "base_rate": 0.6,
        "reference_probability": 0.5,
        "brier_reference": 0.25,
        "brier_skill_score": 0.568,
        "accuracy_class": "GOOD",
        "category_breakdown": [
          {
            "category": "economic_indicator",
            "n": 1,
            "brier_score": 0.04,
            "log_score": 0.223144
          },
          {
            "category": "election_political",
            "n": 1,
            "brier_score": 0.09,
            "log_score": 0.356675
          },
          {
            "category": "sports_competition",
            "n": 1,
            "brier_score": 0.16,
            "log_score": 0.510826
          },
          {
            "category": "gaming_style_event",
            "n": 1,
            "brier_score": 0.09,
            "log_score": 0.356675
          },
          {
            "category": "weather_climate",
            "n": 1,
            "brier_score": 0.16,
            "log_score": 0.510826
          },
          {
            "category": "other",
            "n": 0,
            "brier_score": null,
            "log_score": null
          }
        ],
        "warnings": [],
        "category_note": "Category labels are an informational grouping for reporting purposes only. They do not determine, and are not derived from, contract eligibility, legality, or regulatory status under any authority -- supply your own category per forecast.",
        "scoring_note": "Brier score and logarithmic score are proper scoring rules: a forecaster cannot improve their expected score by reporting anything other than their true belief. Lower is better for both; 0 is a perfect score.",
        "disclaimer": "Not financial or legal advice. This tool scores forecast calibration only -- it does not determine contract eligibility, legality, or regulatory status under any authority. For informational purposes only."
      },
      "golden_hash": "f6f0b7fd42adff0a49c2e62223e6ad958b79266e177bb79259b295a9f17a4263"
    },
    {
      "name": "small_sample_high_confidence_miss",
      "policy_parameters": {
        "forecasts": [
          {
            "probability": 0.999,
            "outcome": 0,
            "category": "economic_indicator"
          },
          {
            "probability": 0.5,
            "outcome": 1,
            "category": "other"
          },
          {
            "probability": 0.4,
            "outcome": 0
          }
        ]
      },
      "output_payload": {
        "n": 3,
        "brier_score": 0.469334,
        "log_score": 2.703909,
        "base_rate": 0.333333,
        "reference_probability": 0.5,
        "brier_reference": 0.25,
        "brier_skill_score": -0.877335,
        "accuracy_class": "POOR",
        "category_breakdown": [
          {
            "category": "economic_indicator",
            "n": 1,
            "brier_score": 0.998001,
            "log_score": 6.907755
          },
          {
            "category": "election_political",
            "n": 0,
            "brier_score": null,
            "log_score": null
          },
          {
            "category": "sports_competition",
            "n": 0,
            "brier_score": null,
            "log_score": null
          },
          {
            "category": "gaming_style_event",
            "n": 0,
            "brier_score": null,
            "log_score": null
          },
          {
            "category": "weather_climate",
            "n": 0,
            "brier_score": null,
            "log_score": null
          },
          {
            "category": "other",
            "n": 2,
            "brier_score": 0.205,
            "log_score": 0.601986
          }
        ],
        "warnings": [
          "INSUFFICIENT_SAMPLE_SIZE",
          "HIGH_CONFIDENCE_MISS"
        ],
        "category_note": "Category labels are an informational grouping for reporting purposes only. They do not determine, and are not derived from, contract eligibility, legality, or regulatory status under any authority -- supply your own category per forecast.",
        "scoring_note": "Brier score and logarithmic score are proper scoring rules: a forecaster cannot improve their expected score by reporting anything other than their true belief. Lower is better for both; 0 is a perfect score.",
        "disclaimer": "Not financial or legal advice. This tool scores forecast calibration only -- it does not determine contract eligibility, legality, or regulatory status under any authority. For informational purposes only."
      },
      "golden_hash": "c4041bc5967e72b665f7d0d2674ae6ad4345f41c5c465989612252cf902da173"
    },
    {
      "name": "perfect_calibration_two_records",
      "policy_parameters": {
        "forecasts": [
          {
            "probability": 1,
            "outcome": 1,
            "category": "weather_climate"
          },
          {
            "probability": 0,
            "outcome": 0,
            "category": "weather_climate"
          }
        ],
        "reference_probability": 0.5
      },
      "output_payload": {
        "n": 2,
        "brier_score": 0,
        "log_score": 0,
        "base_rate": 0.5,
        "reference_probability": 0.5,
        "brier_reference": 0.25,
        "brier_skill_score": 1,
        "accuracy_class": "EXCELLENT",
        "category_breakdown": [
          {
            "category": "economic_indicator",
            "n": 0,
            "brier_score": null,
            "log_score": null
          },
          {
            "category": "election_political",
            "n": 0,
            "brier_score": null,
            "log_score": null
          },
          {
            "category": "sports_competition",
            "n": 0,
            "brier_score": null,
            "log_score": null
          },
          {
            "category": "gaming_style_event",
            "n": 0,
            "brier_score": null,
            "log_score": null
          },
          {
            "category": "weather_climate",
            "n": 2,
            "brier_score": 0,
            "log_score": 0
          },
          {
            "category": "other",
            "n": 0,
            "brier_score": null,
            "log_score": null
          }
        ],
        "warnings": [
          "INSUFFICIENT_SAMPLE_SIZE",
          "ZERO_VARIANCE_SUSPICIOUS"
        ],
        "category_note": "Category labels are an informational grouping for reporting purposes only. They do not determine, and are not derived from, contract eligibility, legality, or regulatory status under any authority -- supply your own category per forecast.",
        "scoring_note": "Brier score and logarithmic score are proper scoring rules: a forecaster cannot improve their expected score by reporting anything other than their true belief. Lower is better for both; 0 is a perfect score.",
        "disclaimer": "Not financial or legal advice. This tool scores forecast calibration only -- it does not determine contract eligibility, legality, or regulatory status under any authority. For informational purposes only."
      },
      "golden_hash": "c552d33d29c724187d3671598753d1d7b3cbb5d20fa29035a307a373595efedb"
    }
  ]
}