Class: Ssync::Sync

Inherits:
Object
  • Object
show all
Extended by:
Helpers
Defined in:
lib/ssync/sync.rb

Class Method Summary collapse

Methods included from Helpers

aquire_lock!, ask, config_exists?, config_path, create_homedir!, default_config_path, display, display_error, exit_with_error!, lock_path, read_config, read_default_config, ssync_filename, ssync_homedir, write_config!, write_default_config!

Class Method Details

.clear_sync_state ⇒ Object 



39
40
41
 
# File 'lib/ssync/sync.rb', line 39

def clear_sync_state
  `rm -f #{last_sync_started} #{last_sync_completed}`
end

.create_tmp_sync_state ⇒ Object 



43
44
45
 
# File 'lib/ssync/sync.rb', line 43

def create_tmp_sync_state
  `touch #{last_sync_started}`
end

.fetch_remote_manifest ⇒ Object 



78
79
80
81
 
# File 'lib/ssync/sync.rb', line 78

def fetch_remote_manifest
  @remote_objects_cache = []
  traverse_s3_for_objects(AWS::S3::Bucket.find(read_config[:aws_dest_bucket]), @remote_objects_cache)
end

.files_modified_since_last_sync ⇒ Object 



63
64
65
66
 
# File 'lib/ssync/sync.rb', line 63

def files_modified_since_last_sync
  # '! -type d' ignores directories, in local manifest directories are spit out to stderr whereas directories pop up in this query
  `find #{read_config[:local_file_path]} #{read_config[:find_options]} \! -type d -cnewer #{last_sync_completed}`.split("\n").collect { |path| { :path => path } }
end

.files_on_localhost_with_checksums ⇒ Object 



93
94
95
 
# File 'lib/ssync/sync.rb', line 93

def files_on_localhost_with_checksums
  parse_manifest(local_manifest_path)
end

.files_on_s3 ⇒ Object 



97
98
99
 
# File 'lib/ssync/sync.rb', line 97

def files_on_s3
  @remote_objects_cache
end

.finalize_sync_state ⇒ Object 



59
60
61
 
# File 'lib/ssync/sync.rb', line 59

def finalize_sync_state
  `cp #{last_sync_started} #{last_sync_completed}`
end

.generate_local_manifest ⇒ Object 



74
75
76
 
# File 'lib/ssync/sync.rb', line 74

def generate_local_manifest
  `find #{read_config[:local_file_path]} #{read_config[:find_options]} -print0 | xargs -0 openssl md5 2> /dev/null > #{local_manifest_path}`
end

.last_sync_completed ⇒ Object 



51
52
53
 
# File 'lib/ssync/sync.rb', line 51

def last_sync_completed
  "#{ssync_homedir}/#{ssync_filename}.last-sync.completed"
end

.last_sync_recorded? ⇒ Boolean

Returns:

  • (Boolean) 


55
56
57
 
# File 'lib/ssync/sync.rb', line 55

def last_sync_recorded?
  File.exist?(last_sync_completed)
end

.last_sync_started ⇒ Object 



47
48
49
 
# File 'lib/ssync/sync.rb', line 47

def last_sync_started
  "#{ssync_homedir}/#{ssync_filename}.last-sync.started"
end

.local_manifest_path ⇒ Object 



101
102
103
 
# File 'lib/ssync/sync.rb', line 101

def local_manifest_path
  "/tmp/#{ssync_filename}.manifest.local"
end

.parse_manifest(location) ⇒ Object 



105
106
107
108
109
110
111
112
113
 
# File 'lib/ssync/sync.rb', line 105

def parse_manifest(location)
  []
  open(location, "r") do |file|
    file.collect do |line|
      path, checksum = *line.chomp.match(/^MD5\((.*)\)= (.*)$/).captures
      { :path => path, :checksum => checksum }
    end
  end if File.exist?(location)
end

.push_file(file) ⇒ Object 



115
116
117
118
119
120
121
 
# File 'lib/ssync/sync.rb', line 115

def push_file(file)
  # xfer speed, logging, etc can occur in this method
  display "Pushing '#{file[:path]}' ..."
  AWS::S3::S3Object.store(file[:path], open(file[:path]), read_config[:aws_dest_bucket])
rescue
  e "Could not push '#{file[:path]}': #{$!.inspect}"
end

.run!(options) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
 
# File 'lib/ssync/sync.rb', line 6

def run!(options)
  display "Initialising Ssync, performing pre-sync checks ..."

  e! "Couldn't connect to AWS with the credentials specified in '#{config_path}'." unless Setup.aws_credentials_is_valid?
  e! "Couldn't find the S3 bucket specified in '#{config_path}'." unless Setup.bucket_exists?
  e! "The local path specified in '#{config_path}' does not exist." unless Setup.local_file_path_exists?

  if options.force?
    display "Clearing previous sync state ..."
    clear_sync_state
  end
  create_tmp_sync_state

  if last_sync_recorded?
    display "Performing time based comparison ..."
    files_modified_since_last_sync
  else
    display "Performing (potentially expensive) MD5 checksum comparison ..."
    display "Generating local manifest ..."
    generate_local_manifest
    display "Traversing S3 for remote manifest ..."
    fetch_remote_manifest
    # note that we do not remove files on s3 that no longer exist on local host.
    # this behaviour may be desirable (ala rsync --delete) but we currently don't support it.
    display "Performing checksum comparison ..."
    files_on_localhost_with_checksums - files_on_s3
  end.each { |file| push_file(file) }

  finalize_sync_state

  display "Sync complete!"
end

.traverse_s3_for_objects(bucket, collection, n = 1000, upto = 0, marker = nil) ⇒ Object 



83
84
85
86
87
88
89
90
91
 
# File 'lib/ssync/sync.rb', line 83

def traverse_s3_for_objects(bucket, collection, n = 1000, upto = 0, marker = nil)
  objects = bucket.objects(:marker => marker, :max_keys => n)
  if objects.size == 0
    return
  else
    objects.each { |object| collection << { :path => "/#{object.key}", :checksum => object.etag } }
    traverse_s3_for_objects(bucket, collection, n, upto+n, objects.last.key)
  end
end

.update_config_with_sync_state(sync_start) ⇒ Object 



68
69
70
71
72
 
# File 'lib/ssync/sync.rb', line 68

def update_config_with_sync_state(sync_start)
  config = read_config()
  config[:last_sync_at] = sync_start
  write_config!(config)
end