Class: Langfuse::Evaluators::LLMEvaluator

Inherits:
BaseEvaluator show all
Defined in:
lib/langfuse/evaluation.rb

Instance Method Summary collapse

Constructor Details

#initialize(client:, name: 'llm_evaluator', description: 'LLM-based evaluator', model: 'gpt-3.5-turbo', prompt_template: nil) ⇒ LLMEvaluator

Returns a new instance of LLMEvaluator.



222
223
224
225
226
227
228
 
# File 'lib/langfuse/evaluation.rb', line 222

def initialize(client:, name: 'llm_evaluator', description: 'LLM-based evaluator', model: 'gpt-3.5-turbo',
               prompt_template: nil)
  super(name: name, description: description)
  @client = client
  @model = model
  @prompt_template = prompt_template || default_prompt_template
end

Instance Method Details

#evaluate(input, output, expected: nil, context: nil) ⇒ Object 



230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
 
# File 'lib/langfuse/evaluation.rb', line 230

def evaluate(input, output, expected: nil, context: nil)
  # This is a placeholder for LLM-based evaluation
  # In a real implementation, you would call an LLM API here
  prompt = @prompt_template.gsub('{input}', input.to_s)
                           .gsub('{output}', output.to_s)
                           .gsub('{expected}', expected.to_s)
                           .gsub('{context}', context.to_s)

  # Simulate LLM response (in real implementation, call actual LLM)
  score = rand(0.0..1.0).round(2)

  create_score(
    value: score,
    data_type: 'NUMERIC',
    comment: "LLM evaluation score: #{score}"
  )
end