Class: NliPipeline::FileManager

Inherits:
SystemWrapper::CallWrapper show all
Includes:
AbstractUtil
Defined in:
lib/nli_pipeline/file_manager.rb

Overview

Manage example files

EXAMPLE:

>> NliPipeline::FileManager.new.copy_example_files()
=> Moving 1 .EXAMPLE files

Instance Attribute Summary

Attributes inherited from SystemWrapper::CallWrapper

#last_return_code

Class Method Summary collapse

Instance Method Summary collapse

Methods included from AbstractUtil

#add_attrs, #drop_forbidden_args_message, included, #init_with_attrs, #raise_unless_all, #to_s

Methods inherited from SystemWrapper::CallWrapper

#call_system

Constructor Details

#initialize(**kwargs) ⇒ FileManager

automatically set backup path init_with_attrs handles the rest

Parameters:

  • kwargs (Hash) —

    all keyword (only supported_args) will be added as vars



38
39
40
41
42
43
 
# File 'lib/nli_pipeline/file_manager.rb', line 38

def initialize(**kwargs)
  # backup_path should not be configurable
  # set it directly regardless of whether it's passed in
  kwargs[:backup_path] = "#{kwargs[:path]}/**/*.backup*"
  init_with_attrs(**kwargs)
end

Class Method Details

.required_args ⇒ Array[Symbol]

Returns:

  • (Array[Symbol])

See Also:



31
32
33
 
# File 'lib/nli_pipeline/file_manager.rb', line 31

def self.required_args
  [:path]
end

.supported_args ⇒ Hash

static methods required by NliPipeline::AbstractUtil::init_attrs

Returns:

  • (Hash)

See Also:



20
21
22
23
24
25
26
 
# File 'lib/nli_pipeline/file_manager.rb', line 20

def self.supported_args
  {
    path: '', extension: '.EXAMPLE', debug: false,
    fail_on_error: false, log_dir: '.pipeline_logs',
    created_get_log_name: 'created_files_', backup_path: ''
  }
end

Instance Method Details

#all_backups ⇒ Object

get backup dates in order from newest to oldest



91
92
93
94
95
 
# File 'lib/nli_pipeline/file_manager.rb', line 91

def all_backups
  backup_files = Dir.glob(@backup_path)
  backup_numbers = backup_files.map { |x| x.split('.backup').last }
  backup_numbers.uniq.sort
end

#all_example_files ⇒ Array[String]

Returns all example files under @path.

Returns:

  • (Array[String]) —

    all example files under @path



46
47
48
49
50
51
52
53
54
 
# File 'lib/nli_pipeline/file_manager.rb', line 46

def all_example_files
  example_file_path = "#{@path}/**/*#{@extension}"
  example_files = Dir.glob(example_file_path)
  if example_files.empty?
    raise ArgumentError, "No #{@extension} Files found at #{example_file_path}"
  end
  puts("Moving #{example_files.count} #{@extension} files".green)
  example_files
end

#backup_non_example_files(example_files, time_stamp, command = 'cp') ⇒ Object

for each example file, if a non-example version of that file exists, back it up.

Parameters:

  • example_files (Array[string]) —

    list of example files

  • time_stamp (String) —

    time of backup

  • command (String) (defaults to: 'cp') —

    command to run (cp/mv)



104
105
106
107
108
109
110
111
112
113
114
115
 
# File 'lib/nli_pipeline/file_manager.rb', line 104

def backup_non_example_files(example_files, time_stamp, command = 'cp')
  possible_files = example_files.map { |f| f.gsub(@extension, '') }
  files_to_backup = possible_files.select { |file| File.exist?(file) }
  # raise ArgumentError.new("#{files_to_backup.to_s} #{example_files.to_s}")
  return false if files_to_backup.empty?

  # preset backup time so all backups fomr same batch have the same timestamp
  puts("Backing up #{files_to_backup.count} files".green)
  files_to_backup.each do |file_to_backup|
    call_system("#{command} #{file_to_backup} #{file_to_backup}.backup#{time_stamp}")
  end
end

#copy_example_files(command = 'cp') ⇒ Object

copy all files ending in @extension under @path. if a non-exmaple version of the file exists, back it up.

Parameters:

  • command (String) (defaults to: 'cp') —

    command to run (cp/mv)



72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/nli_pipeline/file_manager.rb', line 72

def copy_example_files(command = 'cp')
  time_stamp = Time.now.strftime('%Y%m%d%H%M')
  puts("Setting up pipeline in #{@path}".yellow)
  example_files = all_example_files

  backup_non_example_files(example_files, time_stamp)

  example_files.each do |example_file|
    target_file = example_file.gsub(@extension, '')
    call_system("#{command} #{example_file} #{target_file}")

    if File.exist?(target_file)
      add_to_log(target_file, time_stamp)
      puts("\t add #{target_file} to log #{get_log_name(time_stamp)}") if @debug
    end
  end
end

#handle_backup_user_input(backups) ⇒ Array[String]

keeps asking the use to choose a backup until they choose a valid one

Parameters:

  • backups (Array[String])

Returns:

  • (Array[String]) 


143
144
145
146
147
148
149
150
151
152
153
 
# File 'lib/nli_pipeline/file_manager.rb', line 143

def handle_backup_user_input(backups)
  puts "which backup would you like to load?\n#{backups}".yellow
  backup_date = STDIN.gets.chomp

  until backups.include?(backup_date)
    puts "#{backup_date} is not in backups".red
    puts "please choose from: #{backups}".yellow
    backup_date = STDIN.gets.chomp
  end
  ["#{@path}/**/*.backup#{backup_date}", backup_date]
end

#last_created_files ⇒ String

read latest log in .pipeline_logs

Returns:

  • (String) —

    files created by last setup_pipeline



58
59
60
61
62
63
64
65
66
67
 
# File 'lib/nli_pipeline/file_manager.rb', line 58

def last_created_files
  logs = Dir.glob("#{@path}/#{@log_dir}/*")
  if logs.empty?
    puts("No logs found in #{@path}/#{@log_dir}".red)
    return false
  end
  log_numbers = logs.map { |x| x.split(@created_get_log_name.to_s).last }
  latest_log_number = log_numbers.uniq.min
  File.readlines(get_log_name(latest_log_number))
end

#load_from_backup(command = 'cp') ⇒ Object

Returns Boolean.

Parameters:

  • command (String) (defaults to: 'cp') —

    system command. use mv to remove backup, use cp to keep backup.

Returns:

  • Boolean



121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
 
# File 'lib/nli_pipeline/file_manager.rb', line 121

def load_from_backup(command = 'cp')
  puts("loading backup in #{@path}".yellow)
  backups = all_backups
  if backups.empty?
    puts("No backups found in #{@backup_path}".red)
    return false
  end

  backup_path, backup_date = handle_backup_user_input(backups)
  files_to_load = Dir.glob(backup_path)
  puts "loading #{files_to_load.count} files from #{backup_path}".green

  files_to_load.each do |backup_file|
    target_file = backup_file.gsub("\.backup#{backup_date}", '')
    call_system("#{command} #{backup_file} #{target_file}")
  end
end