Class: Mir::Application

Inherits:
Object
  • Object
show all
Defined in:
lib/mir/application.rb

Constant Summary collapse

DEFAULT_SETTINGS_FILE_NAME = 
"mir_settings.yml"
DEFAULT_BATCH_SIZE = 
100

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Mir::Application

Creates a new Mir instance. Parses any command line arguments provided to the application



22
23
24
25
26
27
28
29
30
 
# File 'lib/mir/application.rb', line 22

def initialize
  @options = Mir::Options.parse(ARGV)
  Mir.logger = Logger.new(options.log_destination)
  Mir.logger.level = if options.debug
    Logger::DEBUG
  else
    Logger::ERROR
  end
end

Instance Attribute Details

#disk ⇒ Object (readonly)

Returns the value of attribute disk.



18
19
20
 
# File 'lib/mir/application.rb', line 18

def disk
  @disk
end

#index ⇒ Object (readonly)

Returns the value of attribute index.



18
19
20
 
# File 'lib/mir/application.rb', line 18

def index
  @index
end

#options ⇒ Object (readonly)

Returns the value of attribute options.



18
19
20
 
# File 'lib/mir/application.rb', line 18

def options
  @options
end

Class Method Details

.config ⇒ Mir::Config

Returns a global configuration instance

Returns:



61
62
63
 
# File 'lib/mir/application.rb', line 61

def self.config
  @@config
end

.start ⇒ void

This method returns an undefined value.

Creates a new Mir instance and starts the main execution thread



14
15
16
 
# File 'lib/mir/application.rb', line 14

def self.start
  new.start
end

Instance Method Details

#config ⇒ Object

Alias for +config



67
68
69
 
# File 'lib/mir/application.rb', line 67

def config
  self.class.config
end

#pull(target) ⇒ void

This method returns an undefined value.

Copy the remote disk contents into the specified directory

Parameters:

  • target (String) —

    the absolute path to the destination of the S3 disk contents



107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
 
# File 'lib/mir/application.rb', line 107

def pull(target)
  write_dir = Utils.try_create_dir(target)
  Mir.logger.info "Copying remote disk to #{write_dir.path} using #{config.max_threads} threads"
  failed_downloads = []
  
  time = Benchmark.measure do 
    queue = WorkQueue.new(config.max_threads)
    
    Models::Resource.ordered_groups(DEFAULT_BATCH_SIZE) do |resources|
      # Track the number of download attempts made per resource
      batch = resources.inject({}) { |set, resource| set[resource] = 0; set }
      
      while !batch.empty? do
        batch.each do |resource, attempts|
          dest = File.join(write_dir.path, resource.filename)
          if resource.is_directory?
            Utils.try_create_dir(dest)
            batch.delete(resource)
          elsif attempts >= config.max_download_attempts
            Mir.logger.info "Resource #{resource.abs_path} failed to download"
            failed_downloads << resource
            batch.delete(resource)
          elsif resource.synchronized? dest
            Mir.logger.debug "Skipping already downloaded file #{resource.abs_path}"
            batch.delete(resource)
          else
            batch[resource] += 1
            queue.enqueue_b do
              Mir.logger.debug "Beginning download of #{resource.abs_path}"
              disk.copy(resource.abs_path, dest)
              if resource.synchronized? dest
                FileUtils.chmod_R 0755, dest # allow binaries to execute
                puts "Pulled #{dest}"
                batch.delete(resource)
              end
            end
          end
        end
        queue.join
      end
    end
  end
  Mir.logger.info time
  puts "Completed pull operation #{time}"
  unless failed_downloads.empty?
    puts "The following files failed to download after several attempts:\n}"
    puts failed_downloads.join("\n")
  end
end

#push(target) ⇒ void

This method returns an undefined value.

Synchronize the target directory to the remote disk. Will create a file index if one does not already exist

Parameters:

  • target (String) —

    the absolute path of the folder that will be synchronized remotely



76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
 
# File 'lib/mir/application.rb', line 76

def push(target)
  Mir.logger.info "Starting push operation"
  
  if Models::AppSetting.backup_path != target
    Mir.logger.error "Target does not match directory stored in index"
    exit
  end
  
  index.update
  
  time = Benchmark.measure do
    queue = WorkQueue.new(config.max_threads)
    while Models::Resource.pending_jobs? do
      Models::Resource.pending_sync_groups(DEFAULT_BATCH_SIZE) do |resources|
        push_group(queue, resources)
        handle_push_failures(resources)
      end
    end
  end
  
  # If any assets have been deleted locally, also remove them from remote disk
  index.orphans.each { |orphan| disk.delete(orphan.abs_path) }
  index.clean! # Remove orphans from index
  puts "Completed push operation #{time}"
  Mir.logger.info time
end

#start ⇒ void

This method returns an undefined value.

Begins the synchronization operation after initializing the file index and remote storage container



36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/mir/application.rb', line 36

def start
  if options.copy && options.flush
    Mir.logger.error "Conflicting options: Cannot copy from remote source with an empty file index"
    exit
  end
  
  @@config = Config.new find_settings_file
  param_path = File.expand_path(ARGV[0])
  exit unless config.valid?
  
  # Initialize our remote disk
  @disk = Disk.fetch(config.cloud_provider)
  exit unless @disk.connected?
  
  # Initialize our local index
  @index = Index.new(param_path, config.database)
  @index.setup(:verbose => options.verbose, :force_flush => options.flush)
  
  options.copy ? pull(param_path) : push(param_path)
end