Class: Datahen::Client::Job

Inherits:
Base
  • Object
show all
Defined in:
lib/datahen/client/job.rb

Constant Summary

Constants inherited from Base

Base::CHECK_EMPTY_BODY, Base::CHECK_NIL, Base::DEFAULT_RETRY_LIMIT

Instance Method Summary collapse

Methods inherited from Base

#auth_token, #auth_token=, #default_retry_limit, #env_api_url, env_auth_token, env_ignore_ssl, #ignore_ssl, #initialize, #left_merge, random_delay, #retry

Constructor Details

This class inherits a constructor from Datahen::Client::Base

Instance Method Details

#all(opts = {}) ⇒ Object 



4
5
6
7
 
# File 'lib/datahen/client/job.rb', line 4

def all(opts={})
  params = @options.merge(opts)
  self.class.get("/jobs", params)
end

#cancel(job_id, opts = {}) ⇒ Object 



38
39
40
41
 
# File 'lib/datahen/client/job.rb', line 38

def cancel(job_id, opts={})
  opts[:status] = 'cancelled'
  update(job_id, opts)
end

#delete(job_id, opts = {}) ⇒ Object 



97
98
99
100
 
# File 'lib/datahen/client/job.rb', line 97

def delete(job_id, opts={})
  params = @options.merge(opts)
  self.class.delete("/jobs/#{job_id}", params)
end

#find(job_id, opts = {}) ⇒ Object 



9
10
11
12
13
14
15
 
# File 'lib/datahen/client/job.rb', line 9

def find(job_id, opts={})
  if opts[:live]
    self.class.get("/jobs/#{job_id}", @options)
  else
    self.class.get("/cached/jobs/#{job_id}", @options)
  end
end

#finisher_update(job_id, opts = {}) ⇒ Object 



73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
 
# File 'lib/datahen/client/job.rb', line 73

def finisher_update(job_id, opts={})
  body = {}
  body[:outputs] = opts.fetch(:outputs) {[]}
  body[:finisher_status] = opts.fetch(:finisher_status){ nil }
  body[:log_error] = opts[:log_error] if opts[:log_error]

  params = @options.merge({body: body.to_json})

  limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:finisher]
  self.retry(limit, 5, "Error while updating the finisher.", false, CHECK_EMPTY_BODY) do
    response = self.class.put("/jobs/#{job_id}/finisher_update", params)
    if response.code == 422 && response.body.to_s =~ /pq:\s*deadlock/
      raise CustomRetryError.new(self.class.random_delay(5), response.body.to_s)
    end
    response
  end
end

#pause(job_id, opts = {}) ⇒ Object 



48
49
50
51
 
# File 'lib/datahen/client/job.rb', line 48

def pause(job_id, opts={})
  opts[:status] = 'paused'
  update(job_id, opts)
end

#profile(job_id, opts = {}) ⇒ Object 



91
92
93
94
95
 
# File 'lib/datahen/client/job.rb', line 91

def profile(job_id, opts={})
  params = @options.merge(opts)

  self.class.get("/jobs/#{job_id}/profile", params)
end

#resume(job_id, opts = {}) ⇒ Object 



43
44
45
46
 
# File 'lib/datahen/client/job.rb', line 43

def resume(job_id, opts={})
  opts[:status] = 'active'
  update(job_id, opts)
end

#seeding_update(job_id, opts = {}) ⇒ Object 



53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/datahen/client/job.rb', line 53

def seeding_update(job_id, opts={})
  body = {}
  body[:outputs] = opts.fetch(:outputs) {[]}
  body[:pages] = opts.fetch(:pages) {[]}
  body[:seeding_status] = opts.fetch(:seeding_status){ nil }
  body[:log_error] = opts[:log_error] if opts[:log_error]
  body[:keep_outputs] = !!opts[:keep_outputs] if opts.has_key?(:keep_outputs)

  params = @options.merge({body: body.to_json})

  limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:seeder]
  self.retry(limit, 5, "Error while updating the seeder.", false, CHECK_EMPTY_BODY) do
    response = self.class.put("/jobs/#{job_id}/seeding_update", params)
    if response.code == 422 && response.body.to_s =~ /pq:\s*deadlock/i
      raise CustomRetryError.new(self.class.random_delay(5), response.body.to_s)
    end
    response
  end
end

#sync_schema(job_id, opts = {}) ⇒ Object 



102
103
104
105
106
 
# File 'lib/datahen/client/job.rb', line 102

def sync_schema(job_id, opts={})
  params = @options.merge(opts)

  self.class.put("/jobs/#{job_id}/sync/schema", params)
end

#update(job_id, opts = {}) ⇒ Object 



17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/datahen/client/job.rb', line 17

def update(job_id, opts={})
  body = {}
  body[:status] = opts[:status] if opts[:status]
  body[:parser_worker_count] = opts[:parsers] if opts[:parsers]
  body[:fetcher_worker_count] = opts[:fetchers] if opts[:fetchers]
  body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
  body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
  body[:profile] = opts[:profile] if opts[:profile]
  body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
  body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
  body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
  body[:soft_fetching_try_limit] = opts[:soft_fetching_try_limit] if opts[:soft_fetching_try_limit]
  body[:soft_refetch_limit] = opts[:soft_refetch_limit] if opts[:soft_refetch_limit]
  body[:parsing_try_limit] = opts[:parsing_try_limit] if opts[:parsing_try_limit]
  body[:prevent_kb_autoscaler] = opts[:prevent_kb_autoscaler] if opts.has_key?("prevent_kb_autoscaler") || opts.has_key?(:prevent_kb_autoscaler)
  body[:deletion_protected] = opts[:deletion_protected] if opts.has_key?("deletion_protected") || opts.has_key?(:deletion_protected)
  params = @options.merge({body: body.to_json})

  self.class.put("/jobs/#{job_id}", params)
end