Class: PromptCloudApi

Inherits:
Object
  • Object
show all
Defined in:
lib/promptcloud_data_api.rb

Constant Summary collapse

@@promptcloudhome =
"#{ENV["HOME"]}/promptcloud/"

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(args_hash = {}) ⇒ PromptCloudApi

Returns a new instance of PromptCloudApi.


18
19
20
21
22
23
24
25
# File 'lib/promptcloud_data_api.rb', line 18

def initialize(args_hash = {})
	super()
	@download_dir = nil
	@client_id = nil
	@password = nil
	@client_auth_key = nil
	perform_initial_setup(args_hash)
end

Instance Attribute Details

#api_downtimeObject

Returns the value of attribute api_downtime


16
17
18
# File 'lib/promptcloud_data_api.rb', line 16

def api_downtime
  @api_downtime
end

Instance Method Details

#display_info(args_hash) ⇒ Object


27
28
29
30
31
32
33
34
35
36
37
38
# File 'lib/promptcloud_data_api.rb', line 27

def display_info(args_hash)
	apiconf = "#{@@promptcloudhome}/configs/config.yml"
	apiconf = args_hash[:apiconf] if args_hash[:apiconf]
	if File.file?(apiconf)
		conf_hash = YAML::load_file(apiconf)
		conf_hash.each_pair do |key, val|
			puts "#{key} : #{val}"
		end
	else
		$stderr.puts "Config file #{apiconf} doesn't exist, use -i to create config file"
	end
end

#download_files(args_hash) ⇒ Object


98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
# File 'lib/promptcloud_data_api.rb', line 98

def download_files(args_hash)
	new_feed_exists = false
	ts = ("%10.9f" % (Time.now).to_f).to_s.gsub(/\./, "").to_i
	fetch_log = "#{args_hash[:log_dir]}/fetched_urls-#{ts}.log"
	fetch_log_file = File.open(fetch_log, "w")
	ts_urls_map, url_md5_map = get_file_urls(args_hash)
	unless ts_urls_map
		$stderr.puts "Could not obtain file urls to download."
		return new_feed_exists
	end
	if ts_urls_map.keys.empty?
		$stderr.puts "No new files to download"
		return new_feed_exists
	end
	sorted_ts = ts_urls_map.keys.sort
	sorted_ts.each do |ts|
		urls = ts_urls_map[ts]
		next if not urls
		urls.each do |url|
			md5sum = url_md5_map[url]
			filename = File.basename(url)
			md5_filename = filename.gsub(/\.gz/, ".md5sum")
			md5_filepath = args_hash[:md5_dir]+ "/#{md5_filename}"
			if File.file?(md5_filepath) and File.open(md5_filepath).read.chomp.strip == md5sum
				$stderr.puts "Skipping file #{url}, it has been downloaded earlier."
				next
			end
			new_feed_exists = true
			begin
				$stderr.puts "Fetching file #{url}"
				req = RestClient::Request.new({:method => "get", :user => @client_id, :password => @password, :url =>url}) if args_hash[:api_version] == "v1"
				req = RestClient::Request.new({:method=>"get", :url =>url}) if args_hash[:api_version] == "v2"
				outfile = File.join(@download_dir, File.basename(url))
				File.open(outfile, "wb") do |file|
					file.write req.execute
				end
				content = ""
				Zlib::GzipReader.open(outfile) {|gz|
					content = gz.read
				}
				downloaded_md5 = Digest::MD5.hexdigest(content)
				if md5sum == downloaded_md5
					File.open(md5_filepath, "w"){|file| file.puts md5sum}
					fetch_log_file << "Fetched: #{url}"
				else
					$stderr.puts "Url : #{url} was not downloaded completely, hence deleting the downloaded file"
					fetch_log_file.puts "Failed: #{url}"
					File.delete(outfile)
				end
			rescue Exception => e
				$stderr.puts "Failed to fetch url: #{url}, Exception: #{e.class}, #{e.message}"
				fetch_log_file.puts "Failed: #{url}"
			end
		end
	end
	fetch_log_file.close
	$stderr.puts "\nLog file : #{fetch_log}"
	$stderr.puts "Downloaded files are available at : #{@download_dir}\n\n"
	return new_feed_exists
end

#perform_initial_setup(args_hash = {}) ⇒ Object

optional argument args_hash=:apiconf=>.…, :queried_timestamp_file=>


41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
# File 'lib/promptcloud_data_api.rb', line 41

def perform_initial_setup(args_hash={})
	@@promptcloudhome = args_hash[:promptcloudhome] if args_hash[:promptcloudhome]
	FileUtils.mkdir_p(@@promptcloudhome) unless File.directory?(@@promptcloudhome)
	
	args_hash[:apiconf] = "#{@@promptcloudhome}/configs/config.yml" unless args_hash[:apiconf]
	FileUtils.mkdir_p(File.dirname(args_hash[:apiconf])) unless File.directory?(File.dirname(args_hash[:apiconf]))
	
	args_hash[:log_dir]="#{@@promptcloudhome}/log" unless args_hash[:log_dir]
	FileUtils.mkdir_p(args_hash[:log_dir]) unless File.directory?(args_hash[:log_dir])
	
	args_hash[:md5_dir]="#{@@promptcloudhome}/md5sums" unless args_hash[:md5_dir]
	FileUtils.mkdir_p(args_hash[:md5_dir]) unless File.directory?(args_hash[:md5_dir])
	
	args_hash[:queried_timestamp_file]="#{@@promptcloudhome}/last_queried_ts" unless args_hash[:queried_timestamp_file]
	
	args_hash["download_dir"] = File.join(@@promptcloudhome, "downloads") unless args_hash["download_dir"]
	@download_dir = args_hash["download_dir"]
	FileUtils.mkdir_p(@download_dir) unless File.directory?(@download_dir)

	@conf_hash = {}
	if File.file?(args_hash[:apiconf])
		conf_hash = YAML::load_file(args_hash[:apiconf])
		@conf_hash = conf_hash if conf_hash and conf_hash.is_a?Hash
		@client_id = @conf_hash["client_id"]
		if args_hash[:api_version] == "v2"
			@client_auth_key = @conf_hash["client_auth_key"]
		else
			@password = @conf_hash["password"]
		end
	end
	@client_id = args_hash[:user] if args_hash[:user]
	@client_auth_key = args_hash[:client_auth_key] if args_hash[:client_auth_key]
	@password = args_hash[:pass] if args_hash[:pass]
	unless @client_id	
		$stdout.print "\nPlease enter the user id(for example if you use url http://api.promptcloud.com/data/info?id=demo, then your user id is demo)\n:"
		@client_id = STDIN.gets.chomp.strip
	end
	if args_hash[:api_version] == "v2"
		unless @client_auth_key
			$stdout.print "\nPlease enter the auth key(Provided by PromptCloud)\n:"
			@client_auth_key = STDIN.gets.chomp.strip
		end
	else
		unless @password
			$stdout.print "\nPlease enter the password(Provided by PromptCloud)\n:"
			@password = STDIN.gets.chomp.strip
		end
	end	
	@conf_hash["client_id"] = @client_id
	@conf_hash["client_auth_key"] = @client_auth_key if args_hash[:api_version] == "v2"
	@conf_hash["password"] = @password if args_hash[:api_version] == "v1"
	@conf_hash["download_dir"] = @download_dir
	File.open(args_hash[:apiconf], "w") do |file|
		file << @conf_hash.to_yaml
	end
end