Class: Nearline::Models::Manifest

Inherits:
ActiveRecord::Base
  • Object
show all
Defined in:
lib/nearline/manifest.rb

Overview

A Manifest represents the corpus of ArchivedFiles and set of Log messages resulting from a backup attempt

Constant Summary collapse

@@max_files_cached =

Maximum number of files to stat and process in a batch

10000

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#backup_exclusions ⇒ Object

Just needed when you create a manifest



39
40
41
 
# File 'lib/nearline/manifest.rb', line 39

def backup_exclusions
  @backup_exclusions
end

#backup_paths ⇒ Object

Just needed when you create a manifest



37
38
39
 
# File 'lib/nearline/manifest.rb', line 37

def backup_paths
  @backup_paths
end

Class Method Details

.backup(system, backup_paths, backup_exclusions) ⇒ Object 



52
53
54
55
56
 
# File 'lib/nearline/manifest.rb', line 52

def self.backup(system, backup_paths, backup_exclusions)
  manifest = self.new(:system => system)
  manifest.save!
  manifest.backup(backup_paths, backup_exclusions)
end

.incomplete_manifests ⇒ Object

Find all Manifest entries (across all Systems) which have never finished.

They are:

  • Currently under-way

  • Have failed in some untimely way



119
120
121
 
# File 'lib/nearline/manifest.rb', line 119

def self.incomplete_manifests
  self.find_all_by_completed_at(nil)
end

.new_for_name(system_name) ⇒ Object 



45
46
47
48
49
50
 
# File 'lib/nearline/manifest.rb', line 45

def self.new_for_name(system_name)
  system = System.for_name(system_name)
  system.manifests << m = Nearline::Models::Manifest.new
  system.save!
  m        
end

.restore_all_missing(system, latest_date_time = Time.now) ⇒ Object 



123
124
125
126
127
128
 
# File 'lib/nearline/manifest.rb', line 123

def self.restore_all_missing(system, latest_date_time = Time.now)
  manifest = system.latest_manifest_as_of(latest_date_time)
  manifest.iterate_all_missing do |af|
    af.restore
  end 
end

.what_would_restore(system, latest_date_time = Time.now) ⇒ Object 



130
131
132
133
 
# File 'lib/nearline/manifest.rb', line 130

def self.what_would_restore(system, latest_date_time = Time.now)
  manifest = system.latest_manifest_as_of(latest_date_time)        
  manifest.iterate_all_missing {}
end

Instance Method Details

#add_log(message) ⇒ Object 



149
150
151
152
153
 
# File 'lib/nearline/manifest.rb', line 149

def add_log(message)
  puts message
  log = Nearline::Models::Log.new({:message => message, :manifest_id => self.id})
  log.save!
end

#backup(backup_paths, backup_exclusions) ⇒ Object 



58
59
60
61
62
63
64
65
66
67
 
# File 'lib/nearline/manifest.rb', line 58

def backup(backup_paths, backup_exclusions)
  FileFinder.recurse(backup_paths, backup_exclusions) do |file_path|
    handle_file_path(file_path)
  end
  finish_remaining_file_infos
  
  self.completed_at = Time.now
  self.save!
  self        
end

#before_destroy ⇒ Object 



155
156
157
158
159
160
161
 
# File 'lib/nearline/manifest.rb', line 155

def before_destroy
  destroy_archived_files_with_content
  destroy_archived_files_without_content
  destroy_archived_files_manifests
  destroy_logs
  self.destroy_without_habtm_shim_for_archived_files
end

#existing_archived_file_lookup ⇒ Object 



104
105
106
107
108
109
110
111
112
 
# File 'lib/nearline/manifest.rb', line 104

def existing_archived_file_lookup
  return {} if @file_infos.size == 0
  path_hashes = @file_infos.collect {|e| "'#{e.path_hash}'"}.join(", ")
  conditions = "path_hash in (#{path_hashes})"
  hits = ArchivedFile.find(:all, :conditions => conditions)
  existing_files = {}
  hits.each { |e| existing_files[e.path_hash] = e }
  existing_files
end

#finish_remaining_file_infos ⇒ Object 



78
79
80
 
# File 'lib/nearline/manifest.rb', line 78

def finish_remaining_file_infos
  process_file_infos
end

#handle_file_path(file_path) ⇒ Object 



69
70
71
72
73
74
75
76
 
# File 'lib/nearline/manifest.rb', line 69

def handle_file_path(file_path)
  @file_infos = @file_infos || []
  @file_infos << FileInformation.new(file_path, self)
  
  if @file_infos.size > @@max_files_cached
    process_file_infos
  end
end

#iterate_all_missing ⇒ Object

Iterate all missing files from this manifest, yielding each



136
137
138
139
140
141
142
143
144
145
146
147
 
# File 'lib/nearline/manifest.rb', line 136

def iterate_all_missing
  files_iterated = []
  self.archived_files.each do |af|
    begin
      File.lstat(af.path)
    rescue
      yield af
      files_iterated << af.path
    end
  end
  return files_iterated
end

#process_file_infos ⇒ Object 



82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
 
# File 'lib/nearline/manifest.rb', line 82

def process_file_infos
  return if @file_infos.size == 0

  lookup = existing_archived_file_lookup        
  @file_infos.each do |file_info|
    $stdout.write file_info.file_path + " "
    if (af = lookup[file_info.path_hash])
      self.archived_files << af
    else
      af = ArchivedFile.create_for(file_info)            
    end
    if (!af.nil?)
      $stdout.write "#{Time.at(af.mtime).asctime}"
      if (!af.file_content.nil?)
        $stdout.write" (#{af.file_content.file_size} bytes)"
      end
      $stdout.write("\n")
    end
  end
  @file_infos = []
end

#summary ⇒ Object

A simple string reporting the performance of the manifest



244
245
246
247
248
 
# File 'lib/nearline/manifest.rb', line 244

def summary
  completed = (completed_at.nil?) ? "DNF" : completed_at
  "#{system.name} started: #{created_at}\nfinished: #{completed}\n" +
    "#{archived_files.size} files\n#{logs.size} Error#{(logs.size != 1) ? 's' : ''} reported"
end

#total_size ⇒ Object 



233
234
235
236
237
238
239
240
241
 
# File 'lib/nearline/manifest.rb', line 233

def total_size
  size = 0
  archived_files.each do |af|
    unless af.file_content.nil?
      size += af.file_content.file_size.to_i
    end
  end
  size
end