{
  "run": {
    "id": "da16be5aa117443ea91611c509d390d0",
    "name": "ab-candidate",
    "model": "sim-large-v1",
    "dataset": "/home/evalpipe/demo_dataset.jsonl",
    "status": "completed",
    "started_at": "2026-07-13T01:08:36.231407+00:00",
    "finished_at": "2026-07-13T01:08:36.373275+00:00",
    "error": null,
    "item_count": 40,
    "error_count": 0,
    "pass_rate": 0.825,
    "mean_score": 0.9183330082205929,
    "total_cost_usd": 0.5105,
    "latency_p50_ms": 24.161935318261385,
    "latency_p95_ms": 35.75004148297012,
    "evaluator_means": {
      "exact_match": 0.85,
      "llm_judge_correctness": 0.875,
      "safety": 1.0,
      "semantic_similarity": 0.925885036793548,
      "token_f1": 0.9407800043094161
    },
    "provider_type": "mock"
  },
  "results": [
    {
      "item_id": "qa-001",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the capital of France?",
      "expected": "The capital of France is Paris.",
      "output": "The capital of France is Paris.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.8,
          "passed": true,
          "detail": "judge score 8/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.96,
      "latency_ms": 28.301346115767956,
      "cost_usd": 0.01075,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "France is a country in Western Europe. Its capital and largest city is Paris.",
        "topic": "geography",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-002",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the boiling point of water at sea level in Celsius?",
      "expected": "Water boils at 100 degrees Celsius at sea level.",
      "output": "Water boils at 100 degrees Celsius at sea level.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.9999999999999998,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 1.0,
      "latency_ms": 14.327307231724262,
      "cost_usd": 0.016,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "At standard atmospheric pressure, water boils at 100 degrees Celsius.",
        "topic": "science",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-003",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Who wrote the play Romeo and Juliet?",
      "expected": "Romeo and Juliet was written by William Shakespeare.",
      "output": "Romeo Juliet was written William Shakespeare.",
      "scores": [
        {
          "name": "exact_match",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 0.8571428571428571,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.7006490497453707,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": false,
      "mean_score": 0.6915583813776456,
      "latency_ms": 14.57427442073822,
      "cost_usd": 0.011,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "William Shakespeare wrote Romeo and Juliet in the early stage of his career.",
        "topic": "literature",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-004",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What planet is known as the Red Planet?",
      "expected": "Mars is known as the Red Planet.",
      "output": "Mars is known as the Red Planet.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9800000000000001,
      "latency_ms": 19.60943453013897,
      "cost_usd": 0.0125,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Mars appears reddish because of iron oxide on its surface, earning it the name Red Planet.",
        "topic": "science",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-005",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the largest ocean on Earth?",
      "expected": "The Pacific Ocean is the largest ocean on Earth.",
      "output": "The answer depends on several factors that are not specified here.",
      "scores": [
        {
          "name": "exact_match",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 0.11764705882352941,
          "passed": false,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.13578163673201857,
          "passed": false,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": false,
      "mean_score": 0.45068573911110954,
      "latency_ms": 18.238911405205727,
      "cost_usd": 0.017249999999999998,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "The Pacific Ocean is the largest and deepest of Earth's five oceans.",
        "topic": "geography",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-006",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: How many continents are there on Earth?",
      "expected": "There are seven continents on Earth.",
      "output": "There are seven continents on Earth.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9800000000000001,
      "latency_ms": 15.90158510953188,
      "cost_usd": 0.011,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Earth is commonly divided into seven continents.",
        "topic": "geography",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-007",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What gas do plants absorb during photosynthesis?",
      "expected": "Plants absorb carbon dioxide during photosynthesis.",
      "output": "Plants absorb carbon dioxide during photosynthesis.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.7,
          "passed": true,
          "detail": "judge score 7/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9400000000000001,
      "latency_ms": 29.47391476482153,
      "cost_usd": 0.011,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "During photosynthesis, plants take in carbon dioxide and release oxygen.",
        "topic": "science",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-008",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the chemical symbol for gold?",
      "expected": "The chemical symbol for gold is Au.",
      "output": "I do not have enough information to answer that question.",
      "scores": [
        {
          "name": "exact_match",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": false,
      "mean_score": 0.4,
      "latency_ms": 27.90557686239481,
      "cost_usd": 0.016,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Gold's chemical symbol Au comes from the Latin word aurum.",
        "topic": "science",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-009",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Which country is home to the kangaroo?",
      "expected": "Kangaroos are native to Australia.",
      "output": "Kangaroos are native to Australia.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 1.0,
      "latency_ms": 15.033784322440624,
      "cost_usd": 0.00975,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "The kangaroo is a marsupial indigenous to Australia.",
        "topic": "geography",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-010",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the smallest prime number?",
      "expected": "The smallest prime number is 2.",
      "output": "The smallest prime number is 2.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.7,
          "passed": true,
          "detail": "judge score 7/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9400000000000001,
      "latency_ms": 16.33228547871113,
      "cost_usd": 0.01075,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "The number 2 is the smallest and the only even prime number.",
        "topic": "math",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-011",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Who painted the Mona Lisa?",
      "expected": "The Mona Lisa was painted by Leonardo da Vinci.",
      "output": "The Mona Lisa was painted by Leonardo da Vinci.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9800000000000001,
      "latency_ms": 35.15011165291071,
      "cost_usd": 0.014249999999999999,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Leonardo da Vinci painted the Mona Lisa in the early 16th century.",
        "topic": "art",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-012",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the longest river in the world?",
      "expected": "The Nile is generally considered the longest river in the world.",
      "output": "The Nile is generally considered the longest river in the world.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.6,
          "passed": false,
          "detail": "judge score 6/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": false,
      "mean_score": 0.9199999999999999,
      "latency_ms": 28.369677253067493,
      "cost_usd": 0.017499999999999998,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "The Nile in Africa is generally regarded as the longest river in the world.",
        "topic": "geography",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-013",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: How many sides does a hexagon have?",
      "expected": "A hexagon has six sides.",
      "output": "A hexagon has six sides.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.8,
          "passed": true,
          "detail": "judge score 8/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.96,
      "latency_ms": 34.91075709462166,
      "cost_usd": 0.00975,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "A hexagon is a polygon with six sides and six angles.",
        "topic": "math",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-014",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the freezing point of water in Fahrenheit?",
      "expected": "Water freezes at 32 degrees Fahrenheit.",
      "output": "Water freezes at 32 degrees Fahrenheit.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9800000000000001,
      "latency_ms": 17.10208412259817,
      "cost_usd": 0.0115,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "On the Fahrenheit scale, water freezes at 32 degrees.",
        "topic": "science",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-015",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Which element has the atomic number 1?",
      "expected": "Hydrogen has the atomic number 1.",
      "output": "Hydrogen has the atomic number 1.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 1.0,
      "latency_ms": 29.733721166849136,
      "cost_usd": 0.011,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Hydrogen is the first element of the periodic table with atomic number 1.",
        "topic": "science",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-016",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the currency of Japan?",
      "expected": "The currency of Japan is the yen.",
      "output": "The currency of Japan is the yen.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.8,
          "passed": true,
          "detail": "judge score 8/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.96,
      "latency_ms": 35.742866806685925,
      "cost_usd": 0.012,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Japan's official currency is the Japanese yen.",
        "topic": "geography",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-017",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Who developed the theory of general relativity?",
      "expected": "Albert Einstein developed the theory of general relativity.",
      "output": "Albert Einstein developed the theory of general relativity.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.9999999999999999,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 1.0,
      "latency_ms": 37.82774694263935,
      "cost_usd": 0.0135,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Albert Einstein published the theory of general relativity in 1915.",
        "topic": "science",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-018",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the tallest mountain above sea level?",
      "expected": "Mount Everest is the tallest mountain above sea level.",
      "output": "Mount Everest is the tallest mountain above sea level.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 1.0,
      "latency_ms": 29.309910722076893,
      "cost_usd": 0.015,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Mount Everest is Earth's highest mountain above sea level.",
        "topic": "geography",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-019",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: How many minutes are there in two hours?",
      "expected": "There are 120 minutes in two hours.",
      "output": "There are 120 minutes in two hours.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.8,
          "passed": true,
          "detail": "judge score 8/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.96,
      "latency_ms": 16.380516812205315,
      "cost_usd": 0.0125,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "One hour has 60 minutes, so two hours contain 120 minutes.",
        "topic": "math",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-020",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the primary language spoken in Brazil?",
      "expected": "Portuguese is the primary language spoken in Brazil.",
      "output": "Portuguese is language spoken in Brazil.",
      "scores": [
        {
          "name": "exact_match",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 0.923076923076923,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.778498944161523,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.7,
          "passed": true,
          "detail": "judge score 7/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": false,
      "mean_score": 0.6803151734476891,
      "latency_ms": 29.37997132539749,
      "cost_usd": 0.01125,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Brazil's official and most widely spoken language is Portuguese.",
        "topic": "geography",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-021",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Which organ pumps blood through the human body?",
      "expected": "The heart pumps blood through the human body.",
      "output": "The heart pumps blood through the human body.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 1.0,
      "latency_ms": 28.27178593724966,
      "cost_usd": 0.01375,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "The heart is the muscular organ that circulates blood through the body.",
        "topic": "science",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-022",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the square root of 144?",
      "expected": "The square root of 144 is 12.",
      "output": "The square root of 144 is 12.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.8,
          "passed": true,
          "detail": "judge score 8/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.96,
      "latency_ms": 30.644122511148453,
      "cost_usd": 0.01225,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Since 12 times 12 equals 144, the square root of 144 is 12.",
        "topic": "math",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-023",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: In which year did the Apollo 11 mission land humans on the Moon?",
      "expected": "Apollo 11 landed humans on the Moon in 1969.",
      "output": "Apollo 11 landed humans on the Moon in 1969.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 1.0,
      "latency_ms": 35.080560483038425,
      "cost_usd": 0.01625,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "The Apollo 11 mission achieved the first crewed Moon landing in July 1969.",
        "topic": "history",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-024",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the capital of Japan?",
      "expected": "The capital of Japan is Tokyo.",
      "output": "The capital of Japan is Tokyo.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 1.0,
      "latency_ms": 19.358069635927677,
      "cost_usd": 0.01075,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Tokyo is the capital and most populous city of Japan.",
        "topic": "geography",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-025",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Which gas makes up most of Earth's atmosphere?",
      "expected": "Nitrogen makes up most of Earth's atmosphere.",
      "output": "Nitrogen makes up most of Earth's atmosphere.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.8,
          "passed": true,
          "detail": "judge score 8/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.96,
      "latency_ms": 17.732620239257812,
      "cost_usd": 0.0125,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Earth's atmosphere is about 78 percent nitrogen.",
        "topic": "science",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-026",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Who wrote the novel Pride and Prejudice?",
      "expected": "Pride and Prejudice was written by Jane Austen.",
      "output": "Pride and Prejudice was written by Jane Austen.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.9999999999999999,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.7,
          "passed": true,
          "detail": "judge score 7/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9400000000000001,
      "latency_ms": 25.281542912125587,
      "cost_usd": 0.0135,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Jane Austen published Pride and Prejudice in 1813.",
        "topic": "literature",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-027",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the largest planet in the solar system?",
      "expected": "Jupiter is the largest planet in the solar system.",
      "output": "Jupiter is the largest planet in the solar system.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.9999999999999998,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.8,
          "passed": true,
          "detail": "judge score 8/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.96,
      "latency_ms": 23.042327724397182,
      "cost_usd": 0.01525,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Jupiter is the largest planet in our solar system by both mass and volume.",
        "topic": "science",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-028",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: How many bones does an adult human body have?",
      "expected": "An adult human body has 206 bones.",
      "output": "An adult human body has 206 bones.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9800000000000001,
      "latency_ms": 31.55913483351469,
      "cost_usd": 0.012750000000000001,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "The adult human skeleton typically consists of 206 bones.",
        "topic": "science",
        "difficulty": "hard"
      }
    },
    {
      "item_id": "qa-029",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the capital of Canada?",
      "expected": "The capital of Canada is Ottawa.",
      "output": "The capital of Canada is Ottawa.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.8,
          "passed": true,
          "detail": "judge score 8/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.96,
      "latency_ms": 30.386166647076607,
      "cost_usd": 0.01075,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Ottawa, in the province of Ontario, is the capital city of Canada.",
        "topic": "geography",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-030",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Which metal is liquid at room temperature?",
      "expected": "Mercury is liquid at room temperature.",
      "output": "Mercury is liquid at room temperature.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 1.0,
      "latency_ms": 17.605376429855824,
      "cost_usd": 0.011,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Mercury is the only metallic element that is liquid at room temperature.",
        "topic": "science",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-031",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What does DNA stand for?",
      "expected": "DNA stands for deoxyribonucleic acid.",
      "output": "DNA stands for deoxyribonucleic acid.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9800000000000001,
      "latency_ms": 21.706026047468185,
      "cost_usd": 0.009250000000000001,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "DNA, or deoxyribonucleic acid, carries genetic information.",
        "topic": "science",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-032",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: How many players are on a soccer team on the field?",
      "expected": "A soccer team has 11 players on the field.",
      "output": "A soccer team has 11 players on the field.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9800000000000001,
      "latency_ms": 19.73496936261654,
      "cost_usd": 0.01575,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Each soccer team fields 11 players, including the goalkeeper.",
        "topic": "sports",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-033",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the fastest land animal?",
      "expected": "The cheetah is the fastest land animal.",
      "output": "The cheetah is the fastest land animal.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9800000000000001,
      "latency_ms": 33.51647313684225,
      "cost_usd": 0.012,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "The cheetah can reach speeds around 100 kilometres per hour, making it the fastest land animal.",
        "topic": "science",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-034",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Which ocean lies between Africa and Australia?",
      "expected": "The Indian Ocean lies between Africa and Australia.",
      "output": "The Indian Ocean lies between Africa and Australia.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.9999999999999999,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9800000000000001,
      "latency_ms": 16.681184992194176,
      "cost_usd": 0.0135,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "The Indian Ocean is bounded by Africa to the west and Australia to the east.",
        "topic": "geography",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-035",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the chemical formula for table salt?",
      "expected": "The chemical formula for table salt is NaCl.",
      "output": "The chemical formula for table salt is NaCl.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.9999999999999999,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.8,
          "passed": true,
          "detail": "judge score 8/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.96,
      "latency_ms": 14.149093069136143,
      "cost_usd": 0.01375,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Table salt is sodium chloride, written as NaCl.",
        "topic": "science",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-036",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Who was the first President of the United States?",
      "expected": "George Washington was the first President of the United States.",
      "output": "Washington was the first President of United States.",
      "scores": [
        {
          "name": "exact_match",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 0.9333333333333333,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.8511494340864495,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 1.0,
          "passed": true,
          "detail": "judge score 10/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": false,
      "mean_score": 0.7568965534839566,
      "latency_ms": 31.606536358594894,
      "cost_usd": 0.014,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "George Washington served as the first US president from 1789 to 1797.",
        "topic": "history",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-037",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: How many colors are there in a rainbow?",
      "expected": "A rainbow has seven colors.",
      "output": "A rainbow has seven colors.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9800000000000001,
      "latency_ms": 35.886360332369804,
      "cost_usd": 0.01,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "A rainbow is commonly described as having seven colors.",
        "topic": "science",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-038",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the hardest natural substance on Earth?",
      "expected": "Diamond is the hardest natural substance on Earth.",
      "output": "Diamond is the hardest natural substance on Earth.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.9999999999999999,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.7,
          "passed": true,
          "detail": "judge score 7/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9400000000000001,
      "latency_ms": 19.429711624979973,
      "cost_usd": 0.01375,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Diamond is the hardest known natural material.",
        "topic": "science",
        "difficulty": "medium"
      }
    },
    {
      "item_id": "qa-039",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: Which planet is closest to the Sun?",
      "expected": "Mercury is the planet closest to the Sun.",
      "output": "planet closest to Sun.",
      "scores": [
        {
          "name": "exact_match",
          "score": 0.0,
          "passed": false,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 0.8,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.5693224070165573,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": false,
      "mean_score": 0.6538644814033114,
      "latency_ms": 22.51958567649126,
      "cost_usd": 0.0085,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "Mercury is the smallest planet and the nearest to the Sun.",
        "topic": "science",
        "difficulty": "easy"
      }
    },
    {
      "item_id": "qa-040",
      "prompt": "Answer the question accurately and concisely.\n\nQuestion: What is the sum of the interior angles of a triangle?",
      "expected": "The interior angles of a triangle sum to 180 degrees.",
      "output": "The interior angles of a triangle sum to 180 degrees.",
      "scores": [
        {
          "name": "exact_match",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "token_f1",
          "score": 1.0,
          "passed": true,
          "detail": ""
        },
        {
          "name": "semantic_similarity",
          "score": 0.9999999999999998,
          "passed": true,
          "detail": ""
        },
        {
          "name": "llm_judge_correctness",
          "score": 0.9,
          "passed": true,
          "detail": "judge score 9/10"
        },
        {
          "name": "safety",
          "score": 1.0,
          "passed": true,
          "detail": "no blocked terms present"
        }
      ],
      "passed": true,
      "mean_score": 0.9799999999999999,
      "latency_ms": 16.86787884682417,
      "cost_usd": 0.017,
      "attempts": 1,
      "error": null,
      "metadata": {
        "context": "In Euclidean geometry, a triangle's interior angles always add up to 180 degrees.",
        "topic": "math",
        "difficulty": "medium"
      }
    }
  ]
}