Class: RubricLLM::Metrics::ContextRecall

Inherits:
Base
  • Object
show all
Defined in:
lib/rubric_llm/metrics/context_recall.rb

Constant Summary collapse

SYSTEM_PROMPT = 
<<~PROMPT
  You are an evaluation judge. Assess whether the provided contexts cover the information in the ground truth.
  Context recall is the fraction of facts in the ground truth supported by the contexts.
  List every distinct factual claim in the ground truth. Mark it covered only if a numbered
  context supports it, and use that context's 1-based index as source_context.
  Use null for source_context when a fact is not covered.

  Respond with JSON only:
  {
    "score": <float 0.0-1.0>,
    "covered_facts": [{"fact": "<from ground truth>", "covered": <true/false>, "source_context": <int or null>}],
    "reasoning": "<brief explanation>"
  }
PROMPT

Instance Attribute Summary

Attributes inherited from Base

#judge

Instance Method Summary collapse

Methods inherited from Base

#initialize, normalize_context, require_context!

Constructor Details

This class inherits a constructor from RubricLLM::Metrics::Base

Instance Method Details

#call(context: [], ground_truth: nil) ⇒ Object 



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/rubric_llm/metrics/context_recall.rb', line 21

def call(context: [], ground_truth: nil, **)
  return { score: nil, details: { error: "No ground truth provided" } } if ground_truth.nil?

  context_chunks = Base.normalize_context(context)
  return { score: nil, details: { error: "No context provided" } } if context_chunks.empty?

  user_prompt = <<~PROMPT
    Contexts:
    #{context_chunks.each_with_index.map { |c, i| "#{i + 1}. #{c}" }.join("\n")}

    Ground Truth: #{ground_truth}

    Evaluate how well the contexts cover the facts in the ground truth.
  PROMPT

  result = judge_eval(system_prompt: SYSTEM_PROMPT, user_prompt:)
  normalize(result, context_chunks.size)
end