{
  "run": {
    "id": "5a7d0848b8bc44eaa847d20f7e537e99",
    "name": "nightly-eval",
    "model": "sim-small-v2",
    "dataset": "/home/evalpipe/demo_dataset.jsonl",
    "status": "completed",
    "started_at": "2026-07-13T01:08:35.607613+00:00",
    "finished_at": "2026-07-13T01:08:35.758730+00:00",
    "error": null,
    "item_count": 40,
    "error_count": 0,
    "pass_rate": 0.7,
    "mean_score": 0.8555017727437096,
    "total_cost_usd": 0.518,
    "latency_p50_ms": 26.19325974956155,
    "latency_p95_ms": 36.800049897283316,
    "evaluator_means": {
      "exact_match": 0.75,
      "llm_judge_correctness": 0.8699999999999999,
      "safety": 1.0,
      "semantic_similarity": 0.8221193482632436,
      "token_f1": 0.8353895154553049
    },
    "provider_type": "mock"
  },
  "results": [
    {
      "item_id": "qa-001",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the capital of France?",
      "expected": "The capital of France is Paris.",
      "output": "The capital of France is Paris.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.8,
          "passed": true,
          "detail": "judge score 8/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.96,
      "latency_ms": 28.374028392136097,
      "cost_usd": 0.01075,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "France is a country in Western Europe. Its capital and largest city is Paris.",
        "topic": "geography",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-002",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the boiling point of water at sea level in Celsius?",
      "expected": "Water boils at 100 degrees Celsius at sea level.",
      "output": "Water boils at 100 degrees Celsius at sea level.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.9999999999999998,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 1.0,
      "latency_ms": 36.67108900845051,
      "cost_usd": 0.016,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "At standard atmospheric pressure, water boils at 100 degrees Celsius.",
        "topic": "science",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-003",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Who wrote the play Romeo and Juliet?",
      "expected": "Romeo and Juliet was written by William Shakespeare.",
      "output": "Romeo and Juliet was written by William Shakespeare.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.9999999999999999,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 1.0,
      "latency_ms": 23.879789747297764,
      "cost_usd": 0.0135,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "William Shakespeare wrote Romeo and Juliet in the early stage of his career.",
        "topic": "literature",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-004",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What planet is known as the Red Planet?",
      "expected": "Mars is known as the Red Planet.",
      "output": "I do not have enough information to answer that question.",
      "scores": [
        {
          "name": "exact_match",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.6,
          "passed": false,
          "detail": "judge score 6/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": false,
      "mean_score": 0.32,
      "latency_ms": 35.10037064552307,
      "cost_usd": 0.01625,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Mars appears reddish because of iron oxide on its surface, earning it the name Red Planet.",
        "topic": "science",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-005",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the largest ocean on Earth?",
      "expected": "The Pacific Ocean is the largest ocean on Earth.",
      "output": "The Pacific Ocean is the largest ocean on Earth.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 1.0,
      "latency_ms": 21.35226782411337,
      "cost_usd": 0.01475,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "The Pacific Ocean is the largest and deepest of Earth's five oceans.",
        "topic": "geography",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-006",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: How many continents are there on Earth?",
      "expected": "There are seven continents on Earth.",
      "output": "There are seven continents on Earth.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9800000000000001,
      "latency_ms": 30.547840520739555,
      "cost_usd": 0.011,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Earth is commonly divided into seven continents.",
        "topic": "geography",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-007",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What gas do plants absorb during photosynthesis?",
      "expected": "Plants absorb carbon dioxide during photosynthesis.",
      "output": "Plants absorb carbon dioxide during photosynthesis.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.7,
          "passed": true,
          "detail": "judge score 7/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9400000000000001,
      "latency_ms": 34.81436427682638,
      "cost_usd": 0.011,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "During photosynthesis, plants take in carbon dioxide and release oxygen.",
        "topic": "science",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-008",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the chemical symbol for gold?",
      "expected": "The chemical symbol for gold is Au.",
      "output": "I do not have enough information to answer that question.",
      "scores": [
        {
          "name": "exact_match",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": false,
      "mean_score": 0.4,
      "latency_ms": 36.31346020847559,
      "cost_usd": 0.016,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Gold's chemical symbol Au comes from the Latin word aurum.",
        "topic": "science",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-009",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Which country is home to the kangaroo?",
      "expected": "Kangaroos are native to Australia.",
      "output": "Kangaroos are native to Australia.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 1.0,
      "latency_ms": 31.601525843143463,
      "cost_usd": 0.00975,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "The kangaroo is a marsupial indigenous to Australia.",
        "topic": "geography",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-010",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the smallest prime number?",
      "expected": "The smallest prime number is 2.",
      "output": "The smallest 2.",
      "scores": [
        {
          "name": "exact_match",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 0.5714285714285715,
          "passed": false,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.5393598899705937,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": false,
      "mean_score": 0.6021576922798331,
      "latency_ms": 36.77761182188988,
      "cost_usd": 0.006999999999999999,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "The number 2 is the smallest and the only even prime number.",
        "topic": "math",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-011",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Who painted the Mona Lisa?",
      "expected": "The Mona Lisa was painted by Leonardo da Vinci.",
      "output": "The Mona Lisa was painted by Leonardo da Vinci.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9800000000000001,
      "latency_ms": 26.6403965651989,
      "cost_usd": 0.014249999999999999,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Leonardo da Vinci painted the Mona Lisa in the early 16th century.",
        "topic": "art",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-012",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the longest river in the world?",
      "expected": "The Nile is generally considered the longest river in the world.",
      "output": "The Nile is generally considered the longest river in the world.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.6,
          "passed": false,
          "detail": "judge score 6/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": false,
      "mean_score": 0.9199999999999999,
      "latency_ms": 22.822313010692596,
      "cost_usd": 0.017499999999999998,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "The Nile in Africa is generally regarded as the longest river in the world.",
        "topic": "geography",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-013",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: How many sides does a hexagon have?",
      "expected": "A hexagon has six sides.",
      "output": "A hexagon has six sides.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.8,
          "passed": true,
          "detail": "judge score 8/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.96,
      "latency_ms": 13.894367031753063,
      "cost_usd": 0.00975,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "A hexagon is a polygon with six sides and six angles.",
        "topic": "math",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-014",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the freezing point of water in Fahrenheit?",
      "expected": "Water freezes at 32 degrees Fahrenheit.",
      "output": "That is outside the scope of what I can determine from the prompt.",
      "scores": [
        {
          "name": "exact_match",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.7,
          "passed": true,
          "detail": "judge score 7/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": false,
      "mean_score": 0.33999999999999997,
      "latency_ms": 22.243878804147243,
      "cost_usd": 0.02025,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "On the Fahrenheit scale, water freezes at 32 degrees.",
        "topic": "science",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-015",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Which element has the atomic number 1?",
      "expected": "Hydrogen has the atomic number 1.",
      "output": "Hydrogen has the atomic number 1.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 1.0,
      "latency_ms": 17.003051936626434,
      "cost_usd": 0.011,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Hydrogen is the first element of the periodic table with atomic number 1.",
        "topic": "science",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-016",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the currency of Japan?",
      "expected": "The currency of Japan is the yen.",
      "output": "The currency of Japan is the yen.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.8,
          "passed": true,
          "detail": "judge score 8/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.96,
      "latency_ms": 26.378458365797997,
      "cost_usd": 0.012,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Japan's official currency is the Japanese yen.",
        "topic": "geography",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-017",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Who developed the theory of general relativity?",
      "expected": "Albert Einstein developed the theory of general relativity.",
      "output": "Albert Einstein relativity.",
      "scores": [
        {
          "name": "exact_match",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 0.6,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.46188021535170054,
          "passed": false,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": false,
      "mean_score": 0.5923760430703402,
      "latency_ms": 36.62409819662571,
      "cost_usd": 0.0072499999999999995,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Albert Einstein published the theory of general relativity in 1915.",
        "topic": "science",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-018",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the tallest mountain above sea level?",
      "expected": "Mount Everest is the tallest mountain above sea level.",
      "output": "Mount Everest is the tallest mountain above sea level.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 1.0,
      "latency_ms": 37.634762935340405,
      "cost_usd": 0.015,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Mount Everest is Earth's highest mountain above sea level.",
        "topic": "geography",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-019",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: How many minutes are there in two hours?",
      "expected": "There are 120 minutes in two hours.",
      "output": "There are 120 minutes in two hours.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.8,
          "passed": true,
          "detail": "judge score 8/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.96,
      "latency_ms": 24.873383343219757,
      "cost_usd": 0.0125,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "One hour has 60 minutes, so two hours contain 120 minutes.",
        "topic": "math",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-020",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the primary language spoken in Brazil?",
      "expected": "Portuguese is the primary language spoken in Brazil.",
      "output": "Portuguese is the primary language spoken in Brazil.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.9999999999999999,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 1.0,
      "latency_ms": 33.26672688126564,
      "cost_usd": 0.01375,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Brazil's official and most widely spoken language is Portuguese.",
        "topic": "geography",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-021",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Which organ pumps blood through the human body?",
      "expected": "The heart pumps blood through the human body.",
      "output": "The pumps blood",
      "scores": [
        {
          "name": "exact_match",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 0.5,
          "passed": false,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.5269085168714044,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": false,
      "mean_score": 0.6053817033742809,
      "latency_ms": 30.61827179044485,
      "cost_usd": 0.0075,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "The heart is the muscular organ that circulates blood through the body.",
        "topic": "science",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-022",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the square root of 144?",
      "expected": "The square root of 144 is 12.",
      "output": "The square root of 144 is 12.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.8,
          "passed": true,
          "detail": "judge score 8/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.96,
      "latency_ms": 22.376133129000664,
      "cost_usd": 0.01225,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Since 12 times 12 equals 144, the square root of 144 is 12.",
        "topic": "math",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-023",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: In which year did the Apollo 11 mission land humans on the Moon?",
      "expected": "Apollo 11 landed humans on the Moon in 1969.",
      "output": "Apollo 11 landed humans on the Moon in 1969.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 1.0,
      "latency_ms": 16.43055770546198,
      "cost_usd": 0.01625,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "The Apollo 11 mission achieved the first crewed Moon landing in July 1969.",
        "topic": "history",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-024",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the capital of Japan?",
      "expected": "The capital of Japan is Tokyo.",
      "output": "The capital of Japan is Tokyo.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 1.0,
      "latency_ms": 21.06350101530552,
      "cost_usd": 0.01075,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Tokyo is the capital and most populous city of Japan.",
        "topic": "geography",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-025",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Which gas makes up most of Earth's atmosphere?",
      "expected": "Nitrogen makes up most of Earth's atmosphere.",
      "output": "Nitrogen makes up most of Earth's atmosphere.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.8,
          "passed": true,
          "detail": "judge score 8/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.96,
      "latency_ms": 34.485265612602234,
      "cost_usd": 0.0125,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Earth's atmosphere is about 78 percent nitrogen.",
        "topic": "science",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-026",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Who wrote the novel Pride and Prejudice?",
      "expected": "Pride and Prejudice was written by Jane Austen.",
      "output": "Pride and Prejudice was written by Jane Austen.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.9999999999999999,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.7,
          "passed": true,
          "detail": "judge score 7/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9400000000000001,
      "latency_ms": 31.901773065328598,
      "cost_usd": 0.0135,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Jane Austen published Pride and Prejudice in 1813.",
        "topic": "literature",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-027",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the largest planet in the solar system?",
      "expected": "Jupiter is the largest planet in the solar system.",
      "output": "Jupiter is the largest planet in the solar system.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.9999999999999998,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.8,
          "passed": true,
          "detail": "judge score 8/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.96,
      "latency_ms": 13.648840598762035,
      "cost_usd": 0.01525,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Jupiter is the largest planet in our solar system by both mass and volume.",
        "topic": "science",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-028",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: How many bones does an adult human body have?",
      "expected": "An adult human body has 206 bones.",
      "output": "An adult human body has 206 bones.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9800000000000001,
      "latency_ms": 23.908600211143494,
      "cost_usd": 0.012750000000000001,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "The adult human skeleton typically consists of 206 bones.",
        "topic": "science",
        "difficulty": "hard"
      }
    },
    {
      "item_id": "qa-029",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the capital of Canada?",
      "expected": "The capital of Canada is Ottawa.",
      "output": "The capital Canada Ottawa.",
      "scores": [
        {
          "name": "exact_match",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 0.7499999999999999,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.5698028822981898,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": false,
      "mean_score": 0.6639605764596379,
      "latency_ms": 29.67916987836361,
      "cost_usd": 0.00825,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Ottawa, in the province of Ontario, is the capital city of Canada.",
        "topic": "geography",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-030",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Which metal is liquid at room temperature?",
      "expected": "Mercury is liquid at room temperature.",
      "output": "Mercury is liquid at room temperature.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 1.0,
      "latency_ms": 37.226373329758644,
      "cost_usd": 0.011,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Mercury is the only metallic element that is liquid at room temperature.",
        "topic": "science",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-031",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What does DNA stand for?",
      "expected": "DNA stands for deoxyribonucleic acid.",
      "output": "DNA stands for deoxyribonucleic acid.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9800000000000001,
      "latency_ms": 26.0080611333251,
      "cost_usd": 0.009250000000000001,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "DNA, or deoxyribonucleic acid, carries genetic information.",
        "topic": "science",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-032",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: How many players are on a soccer team on the field?",
      "expected": "A soccer team has 11 players on the field.",
      "output": "A soccer team has 11 players on the field.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9800000000000001,
      "latency_ms": 16.464410349726677,
      "cost_usd": 0.01575,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Each soccer team fields 11 players, including the goalkeeper.",
        "topic": "sports",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-033",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the fastest land animal?",
      "expected": "The cheetah is the fastest land animal.",
      "output": "cheetah is land animal.",
      "scores": [
        {
          "name": "exact_match",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 0.888888888888889,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.608996684932096,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": false,
      "mean_score": 0.699577114764197,
      "latency_ms": 22.930645383894444,
      "cost_usd": 0.00825,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "The cheetah can reach speeds around 100 kilometres per hour, making it the fastest land animal.",
        "topic": "science",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-034",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Which ocean lies between Africa and Australia?",
      "expected": "The Indian Ocean lies between Africa and Australia.",
      "output": "The Indian Ocean lies between Africa and Australia.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.9999999999999999,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9800000000000001,
      "latency_ms": 25.23859217762947,
      "cost_usd": 0.0135,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "The Indian Ocean is bounded by Africa to the west and Australia to the east.",
        "topic": "geography",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-035",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the chemical formula for table salt?",
      "expected": "The chemical formula for table salt is NaCl.",
      "output": "The answer depends on several factors that are not specified here.",
      "scores": [
        {
          "name": "exact_match",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.0563436169819011,
          "passed": false,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.8,
          "passed": true,
          "detail": "judge score 8/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": false,
      "mean_score": 0.37126872339638023,
      "latency_ms": 28.77762820571661,
      "cost_usd": 0.017499999999999998,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Table salt is sodium chloride, written as NaCl.",
        "topic": "science",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-036",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Who was the first President of the United States?",
      "expected": "George Washington was the first President of the United States.",
      "output": "George Washington was the first President of the United States.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.8,
          "passed": true,
          "detail": "judge score 8/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.96,
      "latency_ms": 24.182305671274662,
      "cost_usd": 0.0165,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "George Washington served as the first US president from 1789 to 1797.",
        "topic": "history",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-037",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: How many colors are there in a rainbow?",
      "expected": "A rainbow has seven colors.",
      "output": "A rainbow has seven colors.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9800000000000001,
      "latency_ms": 34.96241848915815,
      "cost_usd": 0.01,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "A rainbow is commonly described as having seven colors.",
        "topic": "science",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-038",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the hardest natural substance on Earth?",
      "expected": "Diamond is the hardest natural substance on Earth.",
      "output": "Diamond is the hardest natural substance on Earth.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.9999999999999999,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.7,
          "passed": true,
          "detail": "judge score 7/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9400000000000001,
      "latency_ms": 14.035310596227646,
      "cost_usd": 0.01375,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Diamond is the hardest known natural material.",
        "topic": "science",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-039",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Which planet is closest to the Sun?",
      "expected": "Mercury is the planet closest to the Sun.",
      "output": "Mercury is the planet closest to the Sun.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.6,
          "passed": false,
          "detail": "judge score 6/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": false,
      "mean_score": 0.9199999999999999,
      "latency_ms": 21.80314902216196,
      "cost_usd": 0.0135,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Mercury is the smallest planet and the nearest to the Sun.",
        "topic": "science",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-040",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the sum of the interior angles of a triangle?",
      "expected": "The interior angles of a triangle sum to 180 degrees.",
      "output": "That is outside the scope of what I can determine from the prompt.",
      "scores": [
        {
          "name": "exact_match",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 0.10526315789473685,
          "passed": false,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.12148212412386165,
          "passed": false,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": false,
      "mean_score": 0.42534905640371967,
      "latency_ms": 23.768876679241657,
      "cost_usd": 0.02075,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "In Euclidean geometry, a triangle's interior angles always add up to 180 degrees.",
        "topic": "math",
        "difficulty": "medium"
      }
    }
  ]
}