Class: WorkFlow

Inherits:
Object
  • Object
show all
Includes:
DataMapper::Resource
Defined in:
lib/manband/workflow.rb

Overview

This class orchestrator the workflow status and the workflow file analysis. It determines if workflow is over, what are the next jobs to execute, …

Constant Summary collapse

@@log = 
Logger.new(STDOUT)

Instance Method Summary collapse

Instance Method Details

#clean ⇒ Object

Clean a workflow directory



268
269
270
271
272
273
274
275
 
# File 'lib/manband/workflow.rb', line 268

def clean
  if self.workdir == nil
    return
  end
  if File.directory? self.workdir
    FileUtils.rm_rf(self.workdir)
  end
end

#delete(deletesub = true) ⇒ Object

Delete workflow, its directory and all its components (jobs, messages,…) deletesub: delete sub workflows too



279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
 
# File 'lib/manband/workflow.rb', line 279

def delete(deletesub=true)
  self.clean
  jobs = Job.all(:wid => self.id)
  if jobs!=nil
    jobs.destroy
  end
  links = JobLink.all(:wid => self.id)
  if links!=nil
    links.destroy
  end
  messages = BandMessage.all(:wid => self.id)
  if messages != nil
    messages.destroy
  end
  if deletesub
    subworkflows = WorkFlow.all(:parent => self.id)
    if subworkflows != nil
      subworkflows.each do |sub|
        sub.delete(false)
      end
    end
  end
  self.destroy
end

#getnextjobs(curnode) ⇒ Object

Get the list of jobs to be run after current node

Returns:

  • an array of node names



63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
 
# File 'lib/manband/workflow.rb', line 63

def getnextjobs(curnode)
  #fworkflow = YAML.load_file(@file)
  fworkflow = BandManager.load(@file)
  if fworkflow==nil
    return nil
  end
  if fworkflow["workflow"][curnode]["next"] == nil
    @@log.debug "no next node, this branch is over"
    #isover?
    return nil
  end
  nexts = fworkflow["workflow"][curnode]["next"].split(',')
  if nexts[0].empty?
    @@log.debug "no next node, this branch is over"
    #isover?
    return nil
  end
  return nexts
end

#getnodecommand(curnode) ⇒ Object

Return a list of commands for the node in the workflow There is one command per input file matching regular expresssions, if any. Default is 1 command.



161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
 
# File 'lib/manband/workflow.rb', line 161

def getnodecommand(curnode)
  #fworkflow = YAML.load_file(@file)
  fworkflow = BandManager.load(@file)
  if fworkflow==nil
    return nil
  end
  maincommand = fworkflow["workflow"][curnode]["command"]
  # Manage node regexp
  exprs = maincommand.scan(/#(node|var)\.(.*?)#/)
  if exprs.length == 0
    return [ fworkflow["workflow"][curnode]["command"] ]
  end
  subnodefilelist = Hash.new
  multinode=nil
  for reg in 0..exprs.length-1
    @@log.debug "Expr "+": "+exprs[reg][0]+" "+exprs[reg][1]
    if exprs[reg][0] == "var"
      maincommand = setruntimevars(exprs[reg][1],maincommand)
      if maincommand == nil
        @@log.error "Runtime var "+exprs[reg][1]+" is not defined for  workflow "+@id.to_s
        return nil
      end
    end
    if exprs[reg][0] == "node"
      # Get regexp for this node
      subnode = exprs[reg][1]
      # If regexp is empty, we jsut want the directory
      if fworkflow["workflow"][curnode][subnode]==nil
        return nil
      end
      if fworkflow["workflow"][curnode][subnode]['regexp'].strip == ''
        subnoderegexp = nil
      else
        subnoderegexp = Regexp.new(fworkflow["workflow"][curnode][subnode]['regexp'])
      end
      # List all files for this node regexp
      if subnode.match(/local/)
        # Local files reference
        nodepath = fworkflow["workflow"][curnode][subnode]['url']
      else
        # An other node reference
        if subnode == 'root'
          nodepath = self.workdir+'/root';
        else
          subjob = Job.first(:wid => @id, :node => subnode)
          nodepath = subjob.workdir
        end
      end
      
      filelist = Array.new
      if subnoderegexp == nil
        filelist.push(nodepath+"/")
      else
        if !File.exists?(nodepath)
          @@log.error("path does not exists!")
          return nil
        end
        Dir.new(nodepath).entries.each do |n|
          if subnoderegexp.match(n)
            filelist.push(nodepath+"/"+n)
          end
        end
      end
      @@log.debug "File list "+subnode+": "+filelist.to_s
      # update file list per node regexp in the command
      subnodefilelist[subnode]=filelist 
      if filelist.length>1
        if multinode!=nil
          # We do not support multiple lists in same command (N*N*N*....)
          return nil
        else
          multinode = subnode
        end
      end
    end
  end

  # TODO manage local URI regexp (or remote)
  # If remote: list then download

  # Now create an array of command with file lists substitution
  commands = Array.new
  subnodefilelist.each do |key,slist|
    if slist[0] == nil
      return nil
    end
    if key != multinode
      # replace, one match only allowed here
      maincommand["#node."+key+"#"]= slist[0]
    end
  end
  if multinode!=nil
    @@log.debug "Multinode: "+multinode
    subnodefilelist[multinode].each do |file|
      newcommand = String.new(maincommand)
      @@log.debug "Command: "+newcommand
      newcommand["#node."+multinode+"#"]= file
      commands.push(newcommand)
    end
  else
    commands.push(maincommand)
  end
  @@log.debug "Commands: "+commands.to_s
  return commands
end

#isover? ⇒ Boolean

Checks if a workflow is over, e.g. we have reached all the terminal nodes (leafs).

Returns:

  • (Boolean) 


41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/manband/workflow.rb', line 41

def isover?
  # decrement terminals
  @terminals = @terminals - 1
  curw = nil
  # Use lock if MYSQL
  if ENV['MYSQL_URL'].include?("mysql")
    DataMapper.repository(:default).adapter.execute("UPDATE work_flows SET terminals = terminals - 1 WHERE id="+@id.to_s);
    curw = WorkFlow.get(@id)
  else
    curw = WorkFlow.get(@id)
    curw.update(:terminals => @terminals)
  end
  if curw.terminals <=0
    @@log.info "Workflow "+@id.to_s+" is over"
    curw.update(:terminated_at => Time.now, :status => STATUS_OVER)
    return true
  end
  return false
end

#parse(curnode, id = nil) ⇒ Object

Parse workflow file and create jobs and links in the database curnode: current node id: id of the node as link originator



86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
 
# File 'lib/manband/workflow.rb', line 86

def parse(curnode, id = nil)
  #fworkflow = YAML.load_file(@file)
  fworkflow = BandManager.load(@file)
  if fworkflow==nil
    return nil
  end
  jobs = getnextjobs(curnode)
  if jobs == nil
    return
  end
  jobs.each do |job|
    if Job.count(:wid => @id, :node => job) == 0
      queue = ""
      if fworkflow["workflow"][job]["queue"]!=nil
        queue = fworkflow["workflow"][job]["queue"]
      end
      status = STATUS_NEW
      if fworkflow["workflow"][job]["breakpoint"]!=nil
        @@log.debug "Node "+job+" has a breakpoint set"
        status = STATUS_SUSPEND
      end
      store = STORE_NO
      if (!fworkflow["options"].nil? && fworkflow["options"]["store"] == "all") || fworkflow["workflow"][job]["store"] == true
        @@log.debug "Add store option for job "+job
        store = STORE_DO
      end
      type = ACTOR
      if fworkflow["workflow"][job]["type"]=='if'
        type = IF_ACTOR
      end
      if fworkflow["workflow"][job]["type"]=='interactive'
        type = INTERACTIVE_ACTOR
      end
      workdir = FlowConfig.getjobdir(@workdir)
      if curnode == "root"
        workdir = self.workdir + "/root";
      end
      newjob = Job.new(:wid => @id, :node => job, :command => "", :status => status, :instances => 0, :maxinstances => 0, :queue => queue, :workdir => FlowConfig.getjobdir(@workdir), :store => store, :type => type)
      newjob.save
      if id != nil
        @@log.debug "Add link "+id.to_s+"->"+newjob.id.to_s+","+newjob.node
        link = JobLink.new(:wid => @id, :from => id, :to => newjob.id)
        link.save
      end
      parse(job,newjob.id)
    else
      if id!=nil
        # Already declared, just add link
        linkedjob = Job.first(:wid => @id, :node => job)
        @@log.debug "Add link "+id.to_s+"->"+linkedjob.id.to_s+","+linkedjob.node
        link = JobLink.new(:wid => @id, :from => id, :to => linkedjob.id)
        link.save
      end
    end
  end
end

#setruntimevars(varexpr, command) ⇒ Object

Update in command the runtime vars return modified command



145
146
147
148
149
150
151
152
153
154
155
156
 
# File 'lib/manband/workflow.rb', line 145

def setruntimevars(varexpr,command)
  newcommand = String.new(command)
  runtimevars = JSON.parse(self.vars)
  if runtimevars[varexpr]!=nil
    @@log.debug "Replace in command #var."+varexpr+"# by "+runtimevars[varexpr]
    newcommand["#var."+varexpr+"#"]= runtimevars[varexpr]
  else
    # runtime var is node defined
    return nil
  end
  return newcommand
end