Class: AnswersEngine::CLI::Scraper

Inherits:
Thor
  • Object
show all
Defined in:
lib/answersengine/cli/scraper.rb

Instance Method Summary collapse

Instance Method Details

#create(scraper_name, git_repository) ⇒ Object 



32
33
34
35
36
 
# File 'lib/answersengine/cli/scraper.rb', line 32

def create(scraper_name, git_repository)
  # puts "options #{options}"
  client = Client::Scraper.new(options)
  puts "#{client.create(scraper_name, git_repository, options)}"
end

#delete(scraper_name) ⇒ Object 



67
68
69
70
 
# File 'lib/answersengine/cli/scraper.rb', line 67

def delete(scraper_name)
  client = Client::Scraper.new(options)
  puts "#{client.delete(scraper_name)}"
end

#deploy(scraper_name) ⇒ Object 



77
78
79
80
81
 
# File 'lib/answersengine/cli/scraper.rb', line 77

def deploy(scraper_name)
  client = Client::ScraperDeployment.new()
  puts "Deploying scraper. This may take a while..."
  puts "#{client.deploy(scraper_name)}"
end

#list ⇒ Object 



11
12
13
14
 
# File 'lib/answersengine/cli/scraper.rb', line 11

def list
  client = Client::Scraper.new(options)
  puts "#{client.all}"
end

#log(scraper_name) ⇒ Object 



107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
 
# File 'lib/answersengine/cli/scraper.rb', line 107

def log(scraper_name)
  client = Client::JobLog.new(options)

  query = {}
  query["order"] = options.delete(:head) if options[:head]
  query["job_type"] = "parsing" if options[:parsing]
  query["job_type"] = "seeding" if options[:seeding]
  query["page_token"] = options.delete(:more) if options[:more]
  query["per_page"] = options.delete(:per_page) if options[:per_page]

  puts "Fetching logs..."

  if options[:job]
    result = client.all_job_log(options[:job], {query: query})
  else
    result = client.scraper_all_job_log(scraper_name, {query: query})
  end

  if result['entries'].nil? || result["entries"].length == 0
    puts "No logs yet, please try again later."
  else
    more_token = result["more_token"]

    result["entries"].each do |entry|
      puts "#{entry["timestamp"]} #{entry["severity"]}: #{entry["payload"]}" if entry.is_a?(Hash)
    end

    unless more_token.nil?
      puts "-----------"
      puts "To see more entries, add: \"--more #{more_token}\""
    end
  end
end

#show(scraper_name) ⇒ Object 



61
62
63
64
 
# File 'lib/answersengine/cli/scraper.rb', line 61

def show(scraper_name)
  client = Client::Scraper.new(options)
  puts "#{client.find(scraper_name)}"
end

#start(scraper_name) ⇒ Object 



90
91
92
93
94
 
# File 'lib/answersengine/cli/scraper.rb', line 90

def start(scraper_name)
  client = Client::ScraperJob.new(options)
  puts "Starting a scrape job..."
  puts "#{client.create(scraper_name, options)}"
end

#stats(scraper_name) ⇒ Object 



146
147
148
149
150
151
152
153
154
 
# File 'lib/answersengine/cli/scraper.rb', line 146

def stats(scraper_name)
  client = Client::JobStat.new(options)
  if options[:job]
    puts "#{client.job_current_stats(options[:job])}"
  else
    puts "#{client.scraper_job_current_stats(scraper_name)}"
  end

end

#update(scraper_name) ⇒ Object 



54
55
56
57
 
# File 'lib/answersengine/cli/scraper.rb', line 54

def update(scraper_name)
  client = Client::Scraper.new(options)
  puts "#{client.update(scraper_name, options)}"
end