23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
|
# File 'lib/argos/download.rb', line 23
def self.download(username, password, archive, log, days=20)
log.debug "Starting download of Argos XML to #{archive}"
soap = Argos::Soap.new({username: username, password: password})
soap.log = log
programs = soap.programs
log.debug "Programs for #{username}: #{programs.to_json}"
year = DateTime.now.year
soap.getPlatformList["data"]["program"].each do |program|
programNumber = program["programNumber"]
soap.programNumber = programNumber
platforms = soap.platforms
active = program["platform"].select {|platform|
lastCollectDate = DateTime.parse(platform["lastCollectDate"])
lastLocationDate = DateTime.parse(platform["lastLocationDate"])
twentydays = DateTime.parse((Date.today-20).to_s)
(lastCollectDate > twentydays or lastLocationDate > twentydays)
}
inactive = program["platform"] - active
active.each_with_index do |a,m|
log.debug "Active [#{m+1}/#{active.size}]: #{a.reject{|k,v| k =~ /location/i }.to_json}"
end
inactive.each_with_index do |i,n|
log.debug "Inactive [#{n+1}/#{inactive.size}]: #{i.reject{|k,v| k =~ /location/i }.to_json}"
end
active.each_with_index do | platform, idx |
platformId = platform["platformId"]
soap.platformId = platformId
log.debug "About to download program: #{programNumber}, platform: #{platformId} [#{idx+1}/#{active.size}], lastCollectDate: #{platform["lastCollectDate"]}"
20.downto(1) do |daysago|
date = Date.today-daysago
destination = "#{archive}/#{year}/program-#{programNumber}/platform-#{platformId}"
begin
soap.period = {startDate: "#{date}T00:00:00Z", endDate: "#{date}T23:59:59.999Z"}
soap.getXml
FileUtils.mkdir_p(destination)
filename = destination+"/argos-#{date}-platform-#{platformId}.xml"
if File.exists? filename
existing_sha1 = Digest::SHA1.file(filename).hexdigest
existing_errors = Argos::Soap.new.validate(File.read(filename))
if existing_errors.any?
log.error "Validation error for existing data #{filename} (#{File.size(filename)} bytes): #{existing_errors.uniq.to_json}"
end
end
new_xml_ng = Nokogiri::XML(soap.xml, nil, nil, Nokogiri::XML::ParseOptions::NOBLANKS | Nokogiri::XML::ParseOptions::NOCDATA | Nokogiri::XML::ParseOptions::STRICT)
new_xml = new_xml_ng.canonicalize
new_sha1 = Digest::SHA1.hexdigest(new_xml)
if existing_sha1.nil? or existing_errors.any? or existing_sha1 != new_sha1
if errors = Argos::Soap.new.validate(new_xml_ng).any?
raise "Failed XML schema validation"
else
File.open(filename, "wb") { |file| file.write(new_xml)}
log.debug "Validated and saved new data: #{filename}"
end
end
log.debug "Day -#{daysago}: #{date}: #{new_xml_ng.xpath("//message").size} message(s) (program #{programNumber}, platform #{platformId})"
rescue Argos::NodataException
log.debug "Day -#{daysago}: #{date}: No data for (program #{programNumber}, platform #{platformId})"
rescue => e
log.error e
end
end
log.debug "Completed download of #{platformId}"
end
log.debug "Completed download for #{username}"
end
end
|