Class: Datahen::CLI::Scraper

Inherits:
Thor
  • Object
show all
Defined in:
lib/datahen/cli/scraper.rb

Instance Method Summary collapse

Instance Method Details

#create(scraper_name, git_repository) ⇒ Object 



44
45
46
47
48
 
# File 'lib/datahen/cli/scraper.rb', line 44

def create(scraper_name, git_repository)
  # puts "options #{options}"
  client = Client::Scraper.new(options)
  puts "#{client.create(scraper_name, git_repository, options)}"
end

#delete(scraper_name) ⇒ Object 



90
91
92
93
 
# File 'lib/datahen/cli/scraper.rb', line 90

def delete(scraper_name)
  client = Client::Scraper.new(options)
  puts "#{client.delete(scraper_name)}"
end

#deploy(scraper_name) ⇒ Object 



100
101
102
103
104
 
# File 'lib/datahen/cli/scraper.rb', line 100

def deploy(scraper_name)
  client = Client::ScraperDeployment.new()
  puts "Deploying scraper. This may take a while..."
  puts "#{client.deploy(scraper_name)}"
end

#history(scraper_name) ⇒ Object 



200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
 
# File 'lib/datahen/cli/scraper.rb', line 200

def history(scraper_name)
  client = Client::JobStat.new(options)
  if options[:job]
    json = JSON.parse(client.job_stats_history(options[:job], options).body)
    if json['error'] == ""
      puts "#{JSON.pretty_generate(json['data'])}"
    else 
      puts "#{JSON.pretty_generate(json['error'])}"
    end
  else
    json = JSON.parse(client.scraper_job_stats_history(scraper_name, options).body)
    if json['error'] == ""
      puts "#{JSON.pretty_generate(json['data'])}"
    else 
      puts "#{JSON.pretty_generate(json['error'])}"
    end
  end
end

#list ⇒ Object 



12
13
14
15
 
# File 'lib/datahen/cli/scraper.rb', line 12

def list
  client = Client::Scraper.new(options)
  puts "#{client.all}"
end

#log(scraper_name) ⇒ Object 



139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
 
# File 'lib/datahen/cli/scraper.rb', line 139

def log(scraper_name)
  client = Client::JobLog.new(options)

  query = {}
  query["order"] = options.delete(:head) if options[:head]
  query["job_type"] = "parsing" if options[:parsing]
  query["job_type"] = "seeding" if options[:seeding]
  query["job_type"] = "finisher executing" if options[:finisher]
  query["page_token"] = options.delete(:more) if options[:more]
  query["per_page"] = options.delete(:per_page) if options[:per_page]

  puts "Fetching logs..."

  if options[:job]
    result = client.all_job_log(options[:job], {query: query})
  else
    result = client.scraper_all_job_log(scraper_name, {query: query})
  end

  if result['entries'].nil? || result["entries"].length == 0
    puts "No logs yet, please try again later."
  else
    more_token = result["more_token"]

    result["entries"].each do |entry|
      puts "#{entry["timestamp"]} #{entry["severity"]}: #{entry["payload"]}" if entry.is_a?(Hash)
    end

    unless more_token.nil?
      puts "-----------"
      puts "To see more entries, add: \"--more #{more_token}\""
    end
  end
end

#profile(scraper_name) ⇒ Object 



223
224
225
226
 
# File 'lib/datahen/cli/scraper.rb', line 223

def profile(scraper_name)
  client = Client::Scraper.new(options)
  puts "#{client.profile(scraper_name)}"
end

#show(scraper_name) ⇒ Object 



84
85
86
87
 
# File 'lib/datahen/cli/scraper.rb', line 84

def show(scraper_name)
  client = Client::Scraper.new(options)
  puts "#{client.find(scraper_name)}"
end

#start(scraper_name) ⇒ Object 



121
122
123
124
125
 
# File 'lib/datahen/cli/scraper.rb', line 121

def start(scraper_name)
  client = Client::ScraperJob.new(options)
  puts "Starting a scrape job..."
  puts "#{client.create(scraper_name, options)}"
end

#stats(scraper_name) ⇒ Object 



180
181
182
183
184
185
186
187
 
# File 'lib/datahen/cli/scraper.rb', line 180

def stats(scraper_name)
  client = Client::JobStat.new(options)
  if options[:job]
    puts "#{client.job_current_stats(options[:job], options)}"
  else
    puts "#{client.scraper_job_current_stats(scraper_name, options)}"
  end
end

#update(scraper_name) ⇒ Object 



77
78
79
80
 
# File 'lib/datahen/cli/scraper.rb', line 77

def update(scraper_name)
  client = Client::Scraper.new(options)
  puts "#{client.update(scraper_name, options)}"
end