Class: RubricLLM::RetrievalResult

Inherits:
Object
  • Object
show all
Defined in:
lib/rubric_llm/retrieval_result.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(retrieved:, relevant:) ⇒ RetrievalResult

Returns a new instance of RetrievalResult.



7
8
9
10
 
# File 'lib/rubric_llm/retrieval_result.rb', line 7

def initialize(retrieved:, relevant:)
  @retrieved = Array(retrieved)
  @relevant = Set.new(Array(relevant))
end

Instance Attribute Details

#relevant ⇒ Object (readonly)

Returns the value of attribute relevant.



5
6
7
 
# File 'lib/rubric_llm/retrieval_result.rb', line 5

def relevant
  @relevant
end

#retrieved ⇒ Object (readonly)

Returns the value of attribute retrieved.



5
6
7
 
# File 'lib/rubric_llm/retrieval_result.rb', line 5

def retrieved
  @retrieved
end

Instance Method Details

#hit_rate ⇒ Object 



53
54
55
 
# File 'lib/rubric_llm/retrieval_result.rb', line 53

def hit_rate
  retrieved.any? { |doc| relevant.include?(doc) } ? 1.0 : 0.0
end

#mrr ⇒ Object

Mean Reciprocal Rank — reciprocal of the rank of the first relevant document.



29
30
31
32
33
34
 
# File 'lib/rubric_llm/retrieval_result.rb', line 29

def mrr
  retrieved.each_with_index do |doc, i|
    return 1.0 / (i + 1) if relevant.include?(doc)
  end
  0.0
end

#ndcg(k: retrieved.size) ⇒ Object

Normalized Discounted Cumulative Gain.



37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/rubric_llm/retrieval_result.rb', line 37

def ndcg(k: retrieved.size)
  return 0.0 if relevant.empty?

  dcg = retrieved.first(k).each_with_index.sum do |doc, i|
    gain = relevant.include?(doc) ? 1.0 : 0.0
    gain / Math.log2(i + 2)
  end

  ideal_count = [relevant.size, k].min
  idcg = (0...ideal_count).sum { |i| 1.0 / Math.log2(i + 2) }

  return 0.0 if idcg.zero?

  dcg / idcg
end

#precision_at_k(k) ⇒ Object 



12
13
14
15
16
17
18
 
# File 'lib/rubric_llm/retrieval_result.rb', line 12

def precision_at_k(k)
  top_k = retrieved.first(k)
  return 0.0 if top_k.empty?

  hits = top_k.count { |doc| relevant.include?(doc) }
  hits / top_k.size.to_f
end

#recall_at_k(k) ⇒ Object 



20
21
22
23
24
25
26
 
# File 'lib/rubric_llm/retrieval_result.rb', line 20

def recall_at_k(k)
  return 0.0 if relevant.empty?

  top_k = retrieved.first(k)
  hits = top_k.count { |doc| relevant.include?(doc) }
  hits / relevant.size.to_f
end

#to_h ⇒ Object 



57
58
59
60
61
62
63
64
65
66
 
# File 'lib/rubric_llm/retrieval_result.rb', line 57

def to_h
  k = retrieved.size
  {
    precision_at_k: precision_at_k(k),
    recall_at_k: recall_at_k(k),
    mrr: mrr,
    ndcg: ndcg,
    hit_rate: hit_rate
  }
end