Class: OodCore::Job::Adapters::Slurm

Inherits:
OodCore::Job::Adapter show all
Defined in:
lib/ood_core/job/adapters/slurm.rb

Overview

An adapter object that describes the communication with a Slurm resource manager for job management.

Defined Under Namespace

Classes: Batch

Constant Summary collapse

STATE_MAP =

Mapping of state codes for Slurm

{
  'BF' => :completed,  # BOOT_FAIL
  'CA' => :completed,  # CANCELLED
  'CD' => :completed,  # COMPLETED
  'CF' => :queued,     # CONFIGURING
  'CG' => :running,    # COMPLETING
  'F'  => :completed,  # FAILED
  'NF' => :completed,  # NODE_FAIL
  'PD' => :queued,     # PENDING
  'PR' => :suspended,  # PREEMPTED
  'RV' => :completed,  # REVOKED
  'R'  => :running,    # RUNNING
  'SE' => :completed,  # SPECIAL_EXIT
  'ST' => :running,    # STOPPED
  'S'  => :suspended,  # SUSPENDED
  'TO' => :completed,   # TIMEOUT
  'OOM' => :completed   # OUT_OF_MEMORY
}

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from OodCore::Job::Adapter

#info_all_each, #info_where_owner_each, #job_name_illegal_chars, #sanitize_job_name, #supports_job_arrays?

Constructor Details

#initialize(opts = {}) ⇒ Slurm

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future.

Returns a new instance of Slurm.

Parameters:

  • opts (#to_h) (defaults to: {}) —

    the options defining this adapter

Options Hash (opts):

  • :slurm (Batch) —

    The Slurm batch object

See Also:



473
474
475
476
477
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 473

def initialize(opts = {})
  o = opts.to_h.symbolize_keys

  @slurm = o.fetch(:slurm) { raise ArgumentError, "No slurm object specified. Missing argument: slurm" }
end

Class Method Details

.gpus_from_gres(gres) ⇒ Integer

Get integer representing the number of gpus used by a node or job, calculated from gres string

Returns:

  • (Integer) —

    the number of gpus in gres



43
44
45
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 43

def self.gpus_from_gres(gres)
  gres.to_s.scan(/gpu[^(,]*[:=](\d+)/).flatten.map(&:to_i).sum
end

Instance Method Details

#accounts ⇒ Array<String>

Retrieve the accounts available to use for the current user.

Returns:

  • (Array<String>) —

    the accounts available to the user.



569
570
571
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 569

def accounts
  @slurm.accounts
end

#cluster_info ⇒ Hash

Retrieve info about active and total cpus, gpus, and nodes

Returns:

  • (Hash) —

    information about cluster usage



562
563
564
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 562

def cluster_info
  @slurm.get_cluster_info
end

#delete(id) ⇒ void

This method returns an undefined value.

Delete the submitted job

Parameters:

  • id (#to_s) —

    the id of the job

Raises:

See Also:



684
685
686
687
688
689
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 684

def delete(id)
  @slurm.delete_job(id.to_s)
rescue Batch::Error => e
  # assume successful job deletion if can't find job id
  raise JobAdapterError, e.message unless /Invalid job id specified/ =~ e.message
end

#directive_prefix ⇒ Object 



691
692
693
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 691

def directive_prefix
  '#SBATCH'
end

#hold(id) ⇒ void

This method returns an undefined value.

Put the submitted job on hold

Parameters:

  • id (#to_s) —

    the id of the job

Raises:

See Also:



660
661
662
663
664
665
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 660

def hold(id)
  @slurm.hold_job(id.to_s)
rescue Batch::Error => e
  # assume successful job hold if can't find job id
  raise JobAdapterError, e.message unless /Invalid job id specified/ =~ e.message
end

#info(id) ⇒ Info

Retrieve job info from the resource manager

Parameters:

  • id (#to_s) —

    the id of the job

Returns:

  • (Info) —

    information describing submitted job

Raises:

See Also:



590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 590

def info(id)
  id = id.to_s
  info_ary = @slurm.get_jobs(id: id).map do |v|
    parse_job_info(v)
  end

  # If no job was found we assume that it has completed
  info_ary.empty? ? Info.new(id: id, status: :completed) : handle_job_array(info_ary, id)
rescue Batch::Error => e
  # set completed status if can't find job id
  if /Invalid job id specified/ =~ e.message
    Info.new(
      id: id,
      status: :completed
    )
  else
    raise JobAdapterError, e.message
  end
end

#info_all(attrs: nil) ⇒ Array<Info>

Retrieve info for all jobs from the resource manager

Returns:

  • (Array<Info>) —

    information describing submitted jobs

Raises:

See Also:



577
578
579
580
581
582
583
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 577

def info_all(attrs: nil)
  @slurm.get_jobs(attrs: attrs).map do |v|
    parse_job_info(v)
  end
rescue Batch::Error => e
  raise JobAdapterError, e.message
end

#info_where_owner(owner, attrs: nil) ⇒ Array<Info>

Retrieve info for all jobs for a given owner or owners from the resource manager

Parameters:

  • owner (#to_s, Array<#to_s>) —

    the owner(s) of the jobs

Returns:

  • (Array<Info>) —

    information describing submitted jobs

Raises:



615
616
617
618
619
620
621
622
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 615

def info_where_owner(owner, attrs: nil)
  owner = Array.wrap(owner).map(&:to_s).join(',')
  @slurm.get_jobs(owner: owner).map do |v|
    parse_job_info(v)
  end
rescue Batch::Error => e
  raise JobAdapterError, e.message
end

#nodes ⇒ Object 



699
700
701
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 699

def nodes
  @slurm.nodes
end

#queues ⇒ Object 



695
696
697
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 695

def queues
  @slurm.queues
end

#release(id) ⇒ void

This method returns an undefined value.

Release the job that is on hold

Parameters:

  • id (#to_s) —

    the id of the job

Raises:

See Also:



672
673
674
675
676
677
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 672

def release(id)
  @slurm.release_job(id.to_s)
rescue Batch::Error => e
  # assume successful job release if can't find job id
  raise JobAdapterError, e.message unless /Invalid job id specified/ =~ e.message
end

#status(id) ⇒ Status

Retrieve job status from resource manager

Parameters:

  • id (#to_s) —

    the id of the job

Returns:

Raises:

See Also:



629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 629

def status(id)
  id = id.to_s
  jobs = @slurm.get_jobs(
    id: id,
    attrs: [:job_id, :array_job_task_id, :state_compact]
  )
  # A job id can return multiple jobs if it corresponds to a job array
  # id, so we need to find the job that corresponds to the given job id
  # (if we can't find it, we assume it has completed)
  #
  # Match against the job id or the formatted job & task id "1234_0"
  if job = jobs.detect { |j| j[:job_id] == id || j[:array_job_task_id] == id }
    Status.new(state: get_state(job[:state_compact]))
  else
    # set completed status if can't find job id
    Status.new(state: :completed)
  end
rescue Batch::Error => e
  # set completed status if can't find job id
  if /Invalid job id specified/ =~ e.message
    Status.new(state: :completed)
  else
    raise JobAdapterError, e.message
  end
end

#submit(script, after: [], afterok: [], afternotok: [], afterany: []) ⇒ String

Submit a job with the attributes defined in the job template instance

Parameters:

  • script (Script) —

    script object that describes the script and attributes for the submitted job

  • after (#to_s, Array<#to_s>) (defaults to: []) —

    this job may be scheduled for execution at any point after dependent jobs have started execution

  • afterok (#to_s, Array<#to_s>) (defaults to: []) —

    this job may be scheduled for execution only after dependent jobs have terminated with no errors

  • afternotok (#to_s, Array<#to_s>) (defaults to: []) —

    this job may be scheduled for execution only after dependent jobs have terminated with errors

  • afterany (#to_s, Array<#to_s>) (defaults to: []) —

    this job may be scheduled for execution after dependent jobs have terminated

Returns:

  • (String) —

    the job id returned after successfully submitting a job

Raises:

See Also:



494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
 
# File 'lib/ood_core/job/adapters/slurm.rb', line 494

def submit(script, after: [], afterok: [], afternotok: [], afterany: [])
  after      = Array(after).map(&:to_s)
  afterok    = Array(afterok).map(&:to_s)
  afternotok = Array(afternotok).map(&:to_s)
  afterany   = Array(afterany).map(&:to_s)

  # Set sbatch options
  args = []
  # ignore args, don't know how to do this for slurm
  args.concat ["-H"] if script.submit_as_hold
  args.concat (script.rerunnable ? ["--requeue"] : ["--no-requeue"]) unless script.rerunnable.nil?
  args.concat ["-D", script.workdir.to_s] unless script.workdir.nil?
  args.concat ["--mail-user", script.email.join(",")] unless script.email.nil?
  if script.email_on_started && script.email_on_terminated
    args.concat ["--mail-type", "ALL"]
  elsif script.email_on_started
    args.concat ["--mail-type", "BEGIN"]
  elsif script.email_on_terminated
    args.concat ["--mail-type", "END"]
  elsif script.email_on_started == false && script.email_on_terminated == false
    args.concat ["--mail-type", "NONE"]
  end
  args.concat ["-J", script.job_name] unless script.job_name.nil?
  args.concat ["-i", script.input_path] unless script.input_path.nil?
  args.concat ["-o", script.output_path] unless script.output_path.nil?
  args.concat ["-e", script.error_path] unless script.error_path.nil?
  args.concat ["--reservation", script.reservation_id] unless script.reservation_id.nil?
  args.concat ["-p", script.queue_name] unless script.queue_name.nil?
  args.concat ["--priority", script.priority] unless script.priority.nil?
  args.concat ["--begin", script.start_time.localtime.strftime("%C%y-%m-%dT%H:%M:%S")] unless script.start_time.nil?
  args.concat ["-A", script.accounting_id] unless script.accounting_id.nil?
  args.concat ["-t", seconds_to_duration(script.wall_time)] unless script.wall_time.nil?
  args.concat ['-a', script.job_array_request] unless script.job_array_request.nil?
  args.concat ['--qos', script.qos] unless script.qos.nil?
  args.concat ['--gpus-per-node', script.gpus_per_node] unless script.gpus_per_node.nil?
  args.concat ['-n', script.cores] unless script.cores.nil?
  # ignore nodes, don't know how to do this for slurm

  # Set dependencies
  depend = []
  depend << "after:#{after.join(":")}"           unless after.empty?
  depend << "afterok:#{afterok.join(":")}"       unless afterok.empty?
  depend << "afternotok:#{afternotok.join(":")}" unless afternotok.empty?
  depend << "afterany:#{afterany.join(":")}"     unless afterany.empty?
  args.concat ["-d", depend.join(",")]               unless depend.empty?

  # Set environment variables
  env = script.job_environment || {}
  args.concat ["--export", export_arg(env, script.copy_environment?)]

  # Set native options
  args.concat script.native if script.native

  # Set content
  content = if script.shell_path.nil?
              script.content
            else
              "#!#{script.shell_path}\n#{script.content}"
            end

  # Submit job
  @slurm.submit_string(content, args: args, env: env)
rescue Batch::Error => e
  raise JobAdapterError, e.message
end