Class: ObservationMover

Inherits:
Object
  • Object
show all
Defined in:
lib/logbox/observation_mover.rb

Constant Summary collapse

DEFAULT_KEY_FILE = 
'/etc/s3_key.yml'
DEFAULT_BUCKET = 
'rwdata-logs'
TIME_FORMAT = 
'-%Y-%m-%d-%H-%M-%S'

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(path, pid_file = '/var/run/nginx.pid') ⇒ ObservationMover

Returns a new instance of ObservationMover.



14
15
16
17
18
 
# File 'lib/logbox/observation_mover.rb', line 14

def initialize (path, pid_file = '/var/run/nginx.pid')
  @dir, @file_name = File.split(path)
  @basename = File.basename(@file_name, File.extname(@file_name))
  @pid_file = pid_file
end

Instance Attribute Details

#bucket ⇒ Object 



98
99
100
 
# File 'lib/logbox/observation_mover.rb', line 98

def bucket
  @bucket ||= DEFAULT_BUCKET
end

#key_file ⇒ Object 



93
94
95
 
# File 'lib/logbox/observation_mover.rb', line 93

def key_file
  @key_file ||= DEFAULT_KEY_FILE
end

Class Method Details

.act_as_fake_nginx ⇒ Object 



132
133
134
135
136
137
138
139
140
 
# File 'lib/logbox/observation_mover.rb', line 132

def self.act_as_fake_nginx
  Dir.mkdir 'testlogs' unless File.exists?('testlogs')
  File.open('testlogs/test.log', 'a') { |f| f.puts "loggy" }
  File.open('test.pid', 'w') { |f| f.print $$ }
  Signal.trap("USR1") { puts "USR1 @ #{Time.now}" }
  puts "Listening for USR1s"
  10.times { sleep 10}
  puts "Done listening for USR1s"
end

Instance Method Details

#aws_keys(file = key_file) ⇒ Object 



84
85
86
87
 
# File 'lib/logbox/observation_mover.rb', line 84

def aws_keys (file = key_file)
  hash = YAML.load_file(file)
  [hash[:access_key_id], hash[:secret_access_key]]
end

#create_new ⇒ Object 



111
112
113
 
# File 'lib/logbox/observation_mover.rb', line 111

def create_new
  File.new(file_path, "a", 0644).close
end

#delete_file(file) ⇒ Object 



52
53
54
 
# File 'lib/logbox/observation_mover.rb', line 52

def delete_file (file)
  File.unlink(file)
end

#file_path ⇒ Object 



20
21
22
 
# File 'lib/logbox/observation_mover.rb', line 20

def file_path
  File.join(@dir, @file_name)
end

#md5(path) ⇒ Object 



89
90
91
 
# File 'lib/logbox/observation_mover.rb', line 89

def md5 (path)
  [Digest::MD5.file(path).digest].pack('m').chomp
end

#move_prepared_to_archive ⇒ Object 



32
33
34
35
36
 
# File 'lib/logbox/observation_mover.rb', line 32

def move_prepared_to_archive
  prepared_for_archiving.each do |file|
    store_file_on_s3(file) && delete_file(file)
  end
end

#notify_logger ⇒ Object 



74
75
76
 
# File 'lib/logbox/observation_mover.rb', line 74

def notify_logger
  Process.kill 'USR1', File.read(@pid_file).to_i if File.exists?(@pid_file)
end

#prepare_rotated_for_archiving ⇒ Object 



56
57
58
59
60
61
62
 
# File 'lib/logbox/observation_mover.rb', line 56

def prepare_rotated_for_archiving
  Dir[rotated_glob].each do |file|
    signed_name = "#{file}-#{short_sha(file)}"
    File.rename file, signed_name
    system "gzip", signed_name
  end
end

#prepared_for_archiving ⇒ Object 



127
128
129
130
 
# File 'lib/logbox/observation_mover.rb', line 127

def prepared_for_archiving
  glob = rotated_glob + '-' + ('[0-9a-f]' * 16) + ".gz"
  Dir[glob]
end

#rename_current ⇒ Object 



107
108
109
 
# File 'lib/logbox/observation_mover.rb', line 107

def rename_current
  File.rename file_path, rotated_path if File.exists?(file_path)
end

#rotate_current ⇒ Object 



64
65
66
67
68
69
70
71
72
 
# File 'lib/logbox/observation_mover.rb', line 64

def rotate_current
  if File.size?(file_path)
    rename_current
    create_new
    true
  else
    false
  end
end

#rotated_glob ⇒ Object 



121
122
123
124
125
 
# File 'lib/logbox/observation_mover.rb', line 121

def rotated_glob
  file_part = @file_name.gsub('.','-')
  time_part = Time.now.strftime(TIME_FORMAT).gsub(/\d/,'[0-9]')
  File.join(@dir, file_part + time_part)
end

#rotated_path ⇒ Object 



117
118
119
 
# File 'lib/logbox/observation_mover.rb', line 117

def rotated_path
  File.join(@dir, @file_name.gsub('.','-') + Time.now.strftime(TIME_FORMAT))
end

#run ⇒ Object 



24
25
26
27
28
29
30
 
# File 'lib/logbox/observation_mover.rb', line 24

def run
  SingleInstance.exclusive_non_blocking(:move_observations) do
    prepare_rotated_for_archiving
    move_prepared_to_archive
    rotate_current && notify_logger
  end
end

#s3 ⇒ Object 



80
81
82
 
# File 'lib/logbox/observation_mover.rb', line 80

def s3
  @s3 ||= RightAws::S3.new(*aws_keys).interface
end

#short_sha(file) ⇒ Object 



103
104
105
 
# File 'lib/logbox/observation_mover.rb', line 103

def short_sha (file)
  Digest::SHA1.file(file).hexdigest[0,16]
end

#store_file_on_s3(file) ⇒ Object

TODO: report transmission errors: W, [2010-04-23T09:10:49.396732 #30275] WARN – : ##### RightAws::S3Interface returned an error: 400 Bad Request <?xml version=“1.0” encoding=“UTF-8”?> <Error>BadDigest<Message>The Content-MD5 you specified did not match what we received.</Message><ExpectedDigest>5f7d35353204db919b646cd3eeeedca2</ExpectedDigest><CalculatedDigest>X301NTIE25GbZGzT7u7cog==</CalculatedDigest><RequestId>CA99C7EA27C5563D</RequestId><HostId>n125lcdYZDbNTV1Wy8OSfL9W5itYbB7wJtLHCo0Uzq/gKfXbu6hzUzwOP9dgkKXy</HostId></Error> ##### W, [2010-04-23T09:10:49.396788 #30275] WARN – : ##### RightAws::S3Interface request: rwdata-logs.s3.amazonaws.com:443/observer-log-2010-04-17-23-20-01-4ef6cbdef683b156 #### /usr/lib/ruby/gems/1.8/gems/right_aws-1.10.0/lib/awsbase/right_awsbase.rb:359:in ‘request_info_impl’: BadDigest: The Content-MD5 you specified did not match what we received. (RightAws::AwsError)



45
46
47
48
49
50
 
# File 'lib/logbox/observation_mover.rb', line 45

def store_file_on_s3 (file)
  File.open file do |f|
    s3.put(bucket, File.basename(file), f, "Content-MD5" => md5(file), 'Content-Type' => 'text/plain')
  end
  true
end