Class: JobMaster

Inherits:
Object
  • Object
show all
Defined in:
lib/job_master.rb,
lib/job_master/stealth.rb,
lib/job_master/gen_methods.rb,
lib/job_master/job_methods.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(database) ⇒ JobMaster

Returns a new instance of JobMaster.



9
10
11
12
13
14
 
# File 'lib/job_master.rb', line 9

def initialize (database)
  @myself = ENV['USER']
  @database = database
  @programs_db = database[:program_constraints]
  @jobs_db = database[:job_log]
end

Instance Attribute Details

#database ⇒ Object (readonly)

Returns the value of attribute database.



7
8
9
 
# File 'lib/job_master.rb', line 7

def database
  @database
end

#jobs_db ⇒ Object (readonly)

Returns the value of attribute jobs_db.



7
8
9
 
# File 'lib/job_master.rb', line 7

def jobs_db
  @jobs_db
end

#programs_db ⇒ Object (readonly)

Returns the value of attribute programs_db.



7
8
9
 
# File 'lib/job_master.rb', line 7

def programs_db
  @programs_db
end

#websites ⇒ Object (readonly)

Returns the value of attribute websites.



7
8
9
 
# File 'lib/job_master.rb', line 7

def websites
  @websites
end

Instance Method Details

#add_websites(*websites) ⇒ Object 



16
17
18
 
# File 'lib/job_master.rb', line 16

def add_websites (*websites)
  @websites = websites
end

#date_gen(args) ⇒ Object 



18
19
20
21
22
 
# File 'lib/job_master/gen_methods.rb', line 18

def date_gen (args)
  min = Date.parse(args[:earliest]).jd
  max = args[:latest] == 'none' ? Date.today.jd : Date.parse(args[:latest]).jd
  units = (min..max).to_a.shuffle
end

#from_csv(file, *labels) ⇒ Object

Raises:

  • (ArgumentError) 


53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/job_master/gen_methods.rb', line 53

def from_csv (file, *labels)

  file = File.open(file, 'r').read.gsub(/\r/, '')
  rows = CSV.parse(file)
  headers = rows.shift.map(&:downcase.strip)
  end_index = headers.rindex(nil)
  headers.slice!(0..end_index) unless end_index == -1
  raise ArgumentError.new('Header cannot contain empty cells') if headers.include?(nil)
  raise ArgumentError.new('Header labels must be unique.') if headers.size != headers.uniq.size
  indexes = labels.map { |label|
    headers.index(label)
  }
  rows.map{ |row|
    new_row = row.values_at(*indexes)
    hash = {}
    new_row.each_with_index { |cell, index|
      hash.store!(labels[index], cell)
    }
  }
  rows
end

#from_table(args) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/job_master/gen_methods.rb', line 24

def from_table (args)
  table_name = args[:table_name]
  columns = args[:columns]
  filters = args[:filters]

  table_name = table_name.to_sym if table_name.is_a?(String)
  columns = eval(columns) if columns.is_a?(String)
  filters = eval(filters) if filters.is_a?(String)

  output = database[table_name].all
  output.map! { |row| row.keep(*columns) } unless columns.nil?

  unless filters.nil?
    output.reject! { |row|
      rejected = false
      filters.each { |key, value|
        if row[key] != value
          rejected = true
          break
        end
      }
      rejected
    }
  end

  output.map!(&:inspect)
  output.to_a.shuffle!
end

#insert_job_details(program, units, machines) ⇒ Object 



83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
 
# File 'lib/job_master/job_methods.rb', line 83

def insert_job_details (program, units, machines)

  time_share = machines.inject({}) { |acc, m|
    acc.store(m, 0)
    acc
  }

  details = {
      program:       program,
      created:       Time.now,
      constraints:   @program_id,
      units:         Sequel.pg_array(units),
      size:          units.size,
      total_minutes: 0.0,
      time_share:    Sequel.hstore(time_share),
      unit_checkout: Sequel.hstore({:apples => :bananas})
  }
  jobs_db.insert(details)
end

#master_cycle(sleep) ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
 
# File 'lib/job_master/job_methods.rb', line 3

def master_cycle (sleep)

  @schedule = database[:job_log].where(status: 'active').order(:frequency).reverse
  @time_log = {}

  stop = false

  until stop
    @schedule.each { |job|
    program = job[:program]
      if job[:frequency] == 0
        step_single(program)
      else
        @time_log[program] ||= Time.now
        difference = (Time.now - @time_log[program]) / 60
        if difference > job[:frequency]
          step_single(program)
          @time_log[program] = Time.now
        end
      end
      sleep(sleep)
    }
  end

end

#next_unit ⇒ Object 



103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
 
# File 'lib/job_master/job_methods.rb', line 103

def next_unit
  checkout = jobs_db[id: @job_id][:unit_checkout].to_hash

  if checkout.keys.include?(@myself)
    checkout[@myself]
  else
    units = jobs_db[id: @job_id][:units]
    return false if units.empty?
    next_unit = units.pop
    jobs_db.where(id: @job_id).update(
        :units => units,
        :unit_checkout => Sequel.hstore(Sequel.expr(:unit_checkout)).merge(@myself => next_unit))
    next_unit
  end
end

#set_up_job(program) ⇒ Object 



69
70
71
72
73
74
75
76
77
78
79
80
81
 
# File 'lib/job_master/job_methods.rb', line 69

def set_up_job (program)
  last_matching = programs_db.where(program: program, @myself => Sequel.pg_array_op(:machines).any).order(:id).last

  @program_id = last_matching[:id]
  the_job = jobs_db.where(program: program, constraints: @program_id).order(:id).last

  if the_job.nil?
    units = self.send(last_matching[:method].to_sym, last_matching[:constraints].merge(program: program))
    insert_job_details(program, units, last_matching[:machines])
  end

  @job_id = the_job[:id]
end

#single_mapping(args) ⇒ Object 



11
12
13
14
15
16
 
# File 'lib/job_master/gen_methods.rb', line 11

def single_mapping (args)
  args = args.first
  eval(args[1]).map { |item|
    {args[0] => item}
  }
end

#start_single(prog_name, minutes = 10, sleep = 3) ⇒ Object 



29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'lib/job_master/job_methods.rb', line 29

def start_single (prog_name , minutes = 10, sleep = 3)

  @start_time = Time.now
  set_up_job(prog_name)

  unit_time = 0.0

  while (Time.now - @start_time) / 60 < minutes

    unless unit_time == 0.0
      update_job(unit_time)
    end

    unit_start = Time.now
    unit = next_unit

    break if unit.nil?

    @websites.first.send(prog_name.to_sym, unit)

    puts unit_time = (Time.now - unit_start) / 60
    sleep(sleep)
    unit_time += sleep / 60
  end

end

#stealth(max_requests, args, *websites) ⇒ Object 



4
5
6
7
8
9
 
# File 'lib/job_master/stealth.rb', line 4

def stealth(max_requests, args, *websites)
  @tunnel = TunnelBlick.new(database)
  @tunnel.connect_smart(max_requests, args, *websites)
  @tunnel.add_websites(*websites)
  @tunnel
end

#step_single(prog_name) ⇒ Object 



56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/job_master/job_methods.rb', line 56

def step_single (prog_name)

  @start_time = Time.now
  set_up_job(prog_name)
  unit = next_unit
  return false if unit.nil?

  @websites.first.send(prog_name.to_sym, unit)
  puts unit_time = (Time.now - @start_time) / 60
  update_job(unit_time)

end

#update_job(unit_time) ⇒ Object 



119
120
121
122
123
124
125
126
127
128
129
130
131
132
 
# File 'lib/job_master/job_methods.rb', line 119

def update_job (unit_time)

  time_share = jobs_db[id: @job_id][:time_share].to_hash
  time_share.find_add!(@myself, unit_time)

  unit_checkout = jobs_db[id: @job_id][:unit_checkout].to_hash
  unit_checkout.delete!(@myself)

  jobs_db.where(id: @job_id).update(
      :total_minutes => Sequel.expr(:total_minutes) + unit_time,
      :time_share => Sequel.hstore(time_share),
      :unit_checkout => Sequel.hstore(unit_checkout)
  )
end