Class: RubricLLM::Metrics::Correctness

Inherits:
Base
  • Object
show all
Defined in:
lib/rubric_llm/metrics/correctness.rb

Constant Summary collapse

SYSTEM_PROMPT = 
<<~PROMPT
  You are an evaluation judge. Assess whether the answer matches the ground truth.
  Consider semantic equivalence, not just exact string matching. Score the answer against
  the question and reference: 1.0 if it fully answers with no material errors, 0.5 if it
  is partly correct but misses required information or includes a material error, and 0.0
  if it is wrong or does not answer. Use intermediate values for partial cases and explain
  what is correct, missing, or wrong. Do not require irrelevant reference details.

  Respond with JSON only:
  {
    "score": <float 0.0-1.0>,
    "reasoning": "<brief explanation>"
  }
PROMPT

Instance Attribute Summary

Attributes inherited from Base

#judge

Instance Method Summary collapse

Methods inherited from Base

#initialize, normalize_context, require_context!

Constructor Details

This class inherits a constructor from RubricLLM::Metrics::Base

Instance Method Details

#call(question:, answer:, ground_truth: nil) ⇒ Object 



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/rubric_llm/metrics/correctness.rb', line 21

def call(question:, answer:, ground_truth: nil, **)
  return { score: nil, details: { error: "No ground truth provided" } } if ground_truth.nil?

  user_prompt = <<~PROMPT
    Question: #{question}

    Answer: #{answer}

    Ground Truth: #{ground_truth}

    Evaluate the correctness of the answer compared to the ground truth.
  PROMPT

  result = judge_eval(system_prompt: SYSTEM_PROMPT, user_prompt:)
  normalize(result)
end