Module: HPC::SLURM
Class Method Summary
collapse
batch_options, batch_system_variables, cleanup_environment, coda, exec_cmd, execute, follow_job, header, job_template, load_modules, meta_data, prepare_environment, prepare_submision, rbbt_job_exec_cmd, run_job, sync_environment, wait_for_job
get_chains, get_job_dependencies, get_recursive_job_dependencies, job_rules, orchestrate_job, piggyback, workload
Class Method Details
.batch_system_variables ⇒ Object
9
10
11
12
13
14
15
|
# File 'lib/rbbt/hpc/slurm.rb', line 9
def self.batch_system_variables
<<-EOF
let "MAX_MEMORY=$SLURM_MEM_PER_CPU * $SLURM_CPUS_PER_TASK" || let MAX_MEMORY="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / 1024"
BATCH_JOB_ID=$SLURM_JOB_ID
BATCH_SYSTEM=SLURM
EOF
end
|
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
|
# File 'lib/rbbt/hpc/slurm.rb', line 17
def self.(options = {})
options = options.dup
queue = Misc.process_options options, :queue
task_cpus = Misc.process_options options, :task_cpus
time = Misc.process_options options, :time
nodes = Misc.process_options options, :nodes
workdir = Misc.process_options options, :workdir
exclusive = Misc.process_options options, :exclusive
batch_dir = Misc.process_options options, :batch_dir
batch_name = Misc.process_options options, :batch_name
fout = File.join(batch_dir, 'std.out')
ferr = File.join(batch_dir, 'std.err')
time = Misc.format_seconds Misc.timespan(time) unless time.include? ":"
=<<-EOF
#!/bin/bash
#SBATCH --job-name="#{batch_name}"
#SBATCH --workdir="#{workdir}"
#SBATCH --output="#{fout}"
#SBATCH --error="#{ferr}"
#SBATCH --qos="#{queue}"
#SBATCH --cpus-per-task="#{task_cpus}"
#SBATCH --time="#{time}"
#SBATCH --nodes="#{nodes}"
EOF
<< "#SBATCH --exclusive" << "\n" if exclusive
end
|
.job_status(job = nil) ⇒ Object
106
107
108
109
110
111
112
|
# File 'lib/rbbt/hpc/slurm.rb', line 106
def self.job_status(job = nil)
if job.nil?
CMD.cmd("squeue").read
else
CMD.cmd("squeue --job #{job}").read
end
end
|
.run_template(batch_dir, dry_run) ⇒ Object
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
|
# File 'lib/rbbt/hpc/slurm.rb', line 52
def self.run_template(batch_dir, dry_run)
fout = File.join(batch_dir, 'std.out')
ferr = File.join(batch_dir, 'std.err')
fjob = File.join(batch_dir, 'job.id')
fdep = File.join(batch_dir, 'dependencies.list')
fcfdep = File.join(batch_dir, 'canfail_dependencies.list')
fexit = File.join(batch_dir, 'exit.status')
fsync = File.join(batch_dir, 'sync.log')
fcmd = File.join(batch_dir, 'command.batch')
return if Open.exists?(fexit)
STDERR.puts Log.color(:magenta, "Issuing SLURM file: #{fcmd}")
STDERR.puts Open.read(fcmd)
if File.exists?(fjob)
job = Open.read(fjob).to_i
else
dependencies = Open.read(fdep).split("\n") if File.exists? fdep
canfail_dependencies = Open.read(fcfdep).split("\n") if File.exists? fcfdep
normal_dep_str = dependencies && dependencies.any? ? "afterok:" + dependencies * ":" : nil
canfail_dep_str = canfail_dependencies && canfail_dependencies.any? ? "afterany:" + canfail_dependencies * ":" : nil
if normal_dep_str.nil? && canfail_dep_str.nil?
dep_str = ""
else
dep_str = '--dependency=' + [normal_dep_str, canfail_dep_str].compact * ","
end
cmd = "sbatch #{dep_str} '#{fcmd}'"
if File.exists?(fout)
return
elsif dry_run
STDERR.puts Log.color(:magenta, "To execute run: ") + Log.color(:blue, "sbatch '#{fcmd}'")
STDERR.puts Log.color(:magenta, "To monitor progress run (needs local rbbt): ") + Log.color(:blue, "rbbt slurm tail '#{batch_dir}'")
raise HPC::SBATCH, batch_dir
else
Open.rm fsync
Open.rm fexit
Open.rm fout
Open.rm ferr
job = CMD.cmd(cmd).read.scan(/\d+/).first.to_i
Log.debug "SBATCH job id: #{job}"
Open.write(fjob, job.to_s)
job
end
end
end
|