Module: Leva

Defined in:
app/models/leva/prompt.rb,
lib/leva.rb,
lib/leva/engine.rb,
lib/leva/version.rb,
app/models/leva/dataset.rb,
app/jobs/leva/run_eval_job.rb,
app/models/leva/experiment.rb,
app/jobs/leva/experiment_job.rb,
app/jobs/leva/application_job.rb,
app/models/leva/runner_result.rb,
app/models/leva/dataset_record.rb,
app/models/leva/evaluation_result.rb,
app/models/leva/application_record.rb,
lib/generators/leva/eval_generator.rb,
app/helpers/leva/application_helper.rb,
app/mailers/leva/application_mailer.rb,
app/models/concerns/leva/recordable.rb,
lib/generators/leva/runner_generator.rb,
app/controllers/leva/datasets_controller.rb,
app/controllers/leva/workbench_controller.rb,
app/controllers/leva/application_controller.rb,
app/controllers/leva/experiments_controller.rb,
app/controllers/leva/runner_results_controller.rb,
app/controllers/leva/dataset_records_controller.rb

Overview

Schema Information

Table name: leva_evaluation_results

id                :integer          not null, primary key
evaluator_class   :string           not null
score             :float
created_at        :datetime         not null
updated_at        :datetime         not null
dataset_record_id :integer          not null
experiment_id     :integer
runner_result_id  :integer          not null

Indexes

index_leva_evaluation_results_on_dataset_record_id  (dataset_record_id)
index_leva_evaluation_results_on_experiment_id      (experiment_id)
index_leva_evaluation_results_on_runner_result_id   (runner_result_id)

Foreign Keys

dataset_record_id  (dataset_record_id => leva_dataset_records.id)
experiment_id      (experiment_id => leva_experiments.id)
runner_result_id   (runner_result_id => leva_runner_results.id)

Defined Under Namespace

Modules: ApplicationHelper, Generators, Recordable Classes: ApplicationController, ApplicationJob, ApplicationMailer, ApplicationRecord, BaseEval, BaseRun, Dataset, DatasetRecord, DatasetRecordsController, DatasetsController, Engine, EvaluationResult, Experiment, ExperimentJob, ExperimentsController, Prompt, RunEvalJob, RunnerResult, RunnerResultsController, WorkbenchController

Constant Summary collapse

VERSION = 
"0.1.7"

Class Method Summary collapse

Class Method Details

.run_evaluation(experiment:, run:, evals:) ⇒ void

This method returns an undefined value.

Runs an evaluation experiment with the given run and evals.

Parameters:



12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
 
# File 'lib/leva.rb', line 12

def self.run_evaluation(experiment:, run:, evals:)
  experiment.update(status: :running)

  experiment.dataset.dataset_records.find_each do |dataset_record|
    runner_result = run.execute_and_store(experiment, dataset_record, experiment.prompt)

    evals.each do |eval|
      eval.evaluate_and_store(experiment, runner_result)
    end
  end

  experiment.update(status: :completed)
rescue StandardError => e
  experiment.update(status: :failed)
  Rails.logger.error "Error in experiment #{experiment.name}: #{e.message}"
end

.run_single_evaluation(experiment:, run:, evals:, dataset_record:) ⇒ void

This method returns an undefined value.

Runs a single evaluation for a dataset record

Parameters:



36
37
38
39
40
41
42
 
# File 'lib/leva.rb', line 36

def self.run_single_evaluation(experiment:, run:, evals:, dataset_record:)
  runner_result = run.execute_and_store(experiment, dataset_record, experiment.prompt)

  evals.each do |eval|
    eval.evaluate_and_store(experiment, runner_result)
  end
end