Class: Leva::BaseRun Abstract

Inherits:
Object
  • Object
show all
Defined in:
lib/leva.rb

Overview

This class is abstract.

Subclass and override #execute to implement custom run logic.

Base class for all run implementations in Leva.

Instance Method Summary collapse

Instance Method Details

#execute(record) ⇒ Object

Executes the run on a given record.

Parameters:

Returns:

  • (Object) —

    The output of the model execution.

Raises:

  • (NotImplementedError) —

    if the method is not implemented in a subclass.



54
55
56
 
# File 'lib/leva.rb', line 54

def execute(record)
  raise NotImplementedError, "#{self.class} must implement #execute"
end

#execute_and_store(experiment, dataset_record, prompt) ⇒ Leva::RunnerResult

Executes the run on a given dataset record and stores the result.

Parameters:

Returns:



64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/leva.rb', line 64

def execute_and_store(experiment, dataset_record, prompt)
  # Expose these to the subclass execution
  @experiment = experiment
  @prompt = prompt

  result = execute(dataset_record.recordable)
  RunnerResult.create!(
    experiment: experiment,
    dataset_record: dataset_record,
    prompt: prompt,
    prediction: result,
    runner_class: self.class.name
  )
end

#extract_regex_pattern(runner_result) ⇒ Regexp?

Returns The regex pattern to use for parsing predictions.

Parameters:

Returns:

  • (Regexp, nil) —

    The regex pattern to use for parsing predictions



91
92
93
 
# File 'lib/leva.rb', line 91

def extract_regex_pattern(runner_result)
  runner_result.dataset_record.recordable.extract_regex_pattern if runner_result.dataset_record.recordable.respond_to?(:extract_regex_pattern)
end

#ground_truth(runner_result) ⇒ String

Returns The ground truth for the runner result.

Parameters:

Returns:

  • (String) —

    The ground truth for the runner result



97
98
99
 
# File 'lib/leva.rb', line 97

def ground_truth(runner_result)
  runner_result.dataset_record.ground_truth
end

#parsed_predictions(runner_result) ⇒ Array<String>

Returns The parsed predictions.

Parameters:

Returns:

  • (Array<String>) —

    The parsed predictions



81
82
83
84
85
86
87
 
# File 'lib/leva.rb', line 81

def parsed_predictions(runner_result)
  if extract_regex_pattern(runner_result)
    runner_result.prediction.scan(extract_regex_pattern(runner_result)).map { |match| match.first&.strip }.compact
  else
    [runner_result.prediction]
  end
end