Class: Ast::Merge::Git::LocalBenchmarkRunner

Inherits:
Object
  • Object
show all
Defined in:
lib/ast/merge/git/local_benchmark.rb,
sig/ast/merge/git.rbs

Overview

Executes the same authored merge bytes through Git and the installed driver.

Constant Summary collapse

DEFAULT_TIMEOUT =

Returns:

30
MAX_WORKERS =

Returns:

32
ADAPTER_DESCRIPTOR_SCHEMA = 
'structuredmerge.benchmark.adapter-descriptor/v1'
MERGIRAF_EXTENSIONS = 
{
  'bash' => 'sh',
  'html' => 'html',
  'json' => 'json',
  'markdown' => 'md',
  'ruby' => 'rb',
  'toml' => 'toml',
  'typescript' => 'ts',
  'yaml' => 'yaml'
}.freeze

Instance Method Summary collapse

Constructor Details

#initialize(benchmark:, driver_path:, tmp_root:, timeout: DEFAULT_TIMEOUT, competitor_paths: {}, workers: 1, adapter_descriptor: nil) ⇒ LocalBenchmarkRunner

rubocop:disable Metrics/ParameterLists -- runner dependencies and resource limits remain explicit



521
522
523
524
525
526
527
528
529
530
531
 
# File 'lib/ast/merge/git/local_benchmark.rb', line 521

def initialize(benchmark:, driver_path:, tmp_root:, timeout: DEFAULT_TIMEOUT, competitor_paths: {}, workers: 1,
               adapter_descriptor: nil)
  @benchmark = benchmark
  @driver_path = Pathname(driver_path).expand_path
  @tmp_root = Pathname(tmp_root).expand_path
  @timeout = Integer(timeout)
  @competitor_paths = competitor_paths.transform_values { |path| Pathname(path).expand_path }
  @workers = Integer(workers)
  error!("workers must be between 1 and #{MAX_WORKERS}") unless @workers.between?(1, MAX_WORKERS)
  @adapter_descriptor = load_adapter_descriptor(adapter_descriptor)
end

Instance Method Details

#performance(profile:, changed_paths: [], iterations: 1) ⇒ Object 



573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
 
# File 'lib/ast/merge/git/local_benchmark.rb', line 573

def performance(profile:, changed_paths: [], iterations: 1)
  verify_environment!
  count = Integer(iterations)
  error!('iterations must be positive') unless count.positive?

  selection = @benchmark.select(profile: profile, changed_paths: changed_paths)
  execution = {
    'kind' => 'performance',
    'adapter_mode' => 'persistent-jsonl',
    'iterations' => count,
    'timing_components' => %w[spawn adapter_execution harness_overhead round_trip]
  }
  manifest = run_manifest(selection, execution: execution)
  FileUtils.mkdir_p(@tmp_root)
  session = BenchmarkAdapterSession.new(
    driver_path: @driver_path,
    chdir: @tmp_root,
    timeout: @timeout,
    env: oracle_free_env
  ).start
  samples = selection.fetch('selected_case_ids').flat_map do |id|
    item = @benchmark.case_by_id(id)
    Array.new(count) do |index|
      performance_sample(session, item, index + 1)
    end
  end
  {
    'schema_version' => 'structuredmerge.benchmark.performance-run/v1',
    'kind' => 'performance_only',
    'quality_classification_performed' => false,
    'selection' => selection,
    'run_manifest' => manifest,
    'cache_identity' => cache_identity(selection, manifest),
    'session' => {
      'adapter_mode' => 'persistent-jsonl',
      'spawn_duration_ns' => session.spawn_duration_ns,
      'process_ids' => samples.map { |sample| sample['process_id'] }.uniq
    },
    'timing' => performance_timing(samples, session.spawn_duration_ns),
    'samples' => samples
  }
ensure
  session&.close
end

#run(profile:, changed_paths: []) ⇒ Hash[String, untyped]

rubocop:enable Metrics/ParameterLists

Parameters:

  • (defaults to: [])

Returns:



534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
 
# File 'lib/ast/merge/git/local_benchmark.rb', line 534

def run(profile:, changed_paths: [])
  verify_environment!
  selection = @benchmark.select(profile: profile, changed_paths: changed_paths)
  if @competitor_paths.any? && selection.fetch('competitor_policy') != 'configured'
    error!('competitor adapters require the competitive profile')
  end
  execution = {
    'kind' => 'correctness',
    'adapter_mode' => 'cold-process',
    'workers' => @workers
  }
  manifest = run_manifest(selection, execution: execution)
  original_cwd = Dir.pwd
  results, worker_process_ids = execute_selected_cases(
    selection.fetch('selected_case_ids'),
    competitors: selection.fetch('competitor_policy') == 'configured'
  )
  raise LocalBenchmark::Error, 'runner changed its working directory' unless Dir.pwd == original_cwd

  {
    'schema_version' => 'structuredmerge.benchmark.run/v1',
    'kind' => 'paired_local_run',
    'corpus_id' => @benchmark.document['id'],
    'corpus_digest' => @benchmark.corpus_digest,
    'selection' => selection,
    'run_manifest' => manifest,
    'cache_identity' => cache_identity(selection, manifest),
    'competitors' => competitor_provenance,
    'execution' => {
      'workers_requested' => @workers,
      'workers_used' => worker_process_ids.length,
      'worker_process_ids' => worker_process_ids
    },
    'results' => results
  }
ensure
  Dir.chdir(original_cwd) if original_cwd && Dir.pwd != original_cwd
end