Module: S3::TransformDeletedFilesService

Defined in:
app/services/s3/transform_deleted_files_service.rb

Constant Summary collapse

BUCKET = 
'eitje-deleted-jurr-2'

Class Method Summary collapse

Class Method Details

.compose_file ⇒ Object 



122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
 
# File 'app/services/s3/transform_deleted_files_service.rb', line 122

def compose_file
  set_records
  set_file_name
  set_existing_records

  @records = (@records | @existing_records) if @existing_records
  set_json
  puts "completed successfully! new total: #{@records&.count}, old total: #{@existing_records&.count}"
  upload_file

  rescue => e
    message = "Error for env #{@env.naam} (##{@env.id}) with table '#{@table}' => #{e.class}: #{e.message}.\n\nBacktrace:#{e.backtrace}\n"
    puts message
    binding.pry
    @logger.error message
end

.format_start_date ⇒ Object 



97
98
99
 
# File 'app/services/s3/transform_deleted_files_service.rb', line 97

def format_start_date
  @start_date = @start_date.class == String ? @start_date : @start_date.strftime("%Y-%m-%d")
end

.migrate_files(start_date: Date.today) ⇒ Object 



16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'app/services/s3/transform_deleted_files_service.rb', line 16

def migrate_files(start_date: Date.today)
  set_setters(start_date)

  s3 = Aws::S3::Client.new
  envs_to_migrate = []

  set_tables.each do |table|  

    folder = table
    folder = "verlofverzoeks" if table == "verlof_verzoeken"

    puts "requesting #{table} on #{start_date.strftime("%Y-%m-%d")}..."

    object  = s3.get_object(bucket: 'eitje-backups', key: "#{folder}/#{start_date.strftime("%Y-%m-%d")}.json")
    json    = JSON.parse(object.body.read.as_json).map(&:symbolize_keys)
    
    if table == 'topics'
      env_ids = json.map {|row| row[:environment_ids]}.flatten.compact.uniq
    else
      puts "new table logic...?"
      env_ids = json.map { _1[:environment_id] }.compact.uniq

      unless env_ids.present?
        env_ids = json.map {|row| row[:env]}.uniq.map { |name| Environment.find_by(naam: name)&.id }
      end
    end

    envs_to_migrate << env_ids
  rescue Aws::S3::Errors::NoSuchKey => e
    # in case the file does not exist on S3, cause there are no deleted 
    # records, skip to next table
    puts "no records found for #{table} on #{start_date.strftime("%Y-%m-%d")}!"
    []
  end

  envs_to_migrate = envs_to_migrate.flatten.uniq.compact
  
  envs_to_migrate.each { |env_id| migrate_files_single_env(env_id, start_date: start_date, skip_setters: true) }
end

.migrate_files_multi_env(environment_ids, start_date: Date.yesterday) ⇒ Object 



70
71
72
73
 
# File 'app/services/s3/transform_deleted_files_service.rb', line 70

def migrate_files_multi_env(environment_ids, start_date: Date.yesterday)
  set_setters(start_date)
  environment_ids.each { |id| migrate_files_single_env(id, start_date: start_date, skip_setters: true) }
end

.migrate_files_single_env(environment_id, start_date: Date.yesterday, skip_setters: false) ⇒ Object 



56
57
58
59
60
61
62
63
64
65
66
67
68
 
# File 'app/services/s3/transform_deleted_files_service.rb', line 56

def migrate_files_single_env(environment_id, start_date: Date.yesterday, skip_setters: false)
  set_setters(start_date) unless skip_setters
  @env = Environment.find(environment_id)
  @tables.each do |table| 
    @table = table          
    compose_file
  end

rescue ActiveRecord::RecordNotFound => e

rescue => e
  binding.pry
end

.migrate_files_single_org(organisation_id, start_date: Date.yesterday) ⇒ Object 



75
76
77
78
 
# File 'app/services/s3/transform_deleted_files_service.rb', line 75

def migrate_files_single_org(organisation_id, start_date: Date.yesterday)
  env_ids = Organisation.find(organisation_id).environment_ids
  migrate_files_multi_env(env_ids, start_date: start_date)
end

.set_bucket ⇒ Object 



84
85
86
 
# File 'app/services/s3/transform_deleted_files_service.rb', line 84

def set_bucket
  @s3 = Aws::S3::Client.new
end

.set_dates(start_date) ⇒ Object 



92
93
94
95
 
# File 'app/services/s3/transform_deleted_files_service.rb', line 92

def set_dates(start_date)
  format_start_date
  @dates = {start_date: @start_date, end_date: Date.today.strftime("%Y-%m-%d")}
end

.set_existing_records ⇒ Object 



113
114
115
116
117
118
119
120
 
# File 'app/services/s3/transform_deleted_files_service.rb', line 113

def set_existing_records
  object = @s3.get_object(bucket: BUCKET, key: @file_name)
  @existing_records = JSON.parse(object.body.read.as_json).map(&:symbolize_keys)
  
  rescue Aws::S3::Errors::NoSuchKey => e
    binding.pry
    @existing_records = nil
end

.set_file_name ⇒ Object 



109
110
111
 
# File 'app/services/s3/transform_deleted_files_service.rb', line 109

def set_file_name
  @file_name = "env_#{@env.id}_deleted_#{@table}.json"
end

.set_json ⇒ Object 



105
106
107
 
# File 'app/services/s3/transform_deleted_files_service.rb', line 105

def set_json
  @json = JSON.pretty_generate(@records)
end

.set_logger ⇒ Object 



80
81
82
 
# File 'app/services/s3/transform_deleted_files_service.rb', line 80

def set_logger
  @logger = Logger.new "log/migrate_deleted_records_#{DateTime.now.strftime('%Y_%m_%d_%H:%M:%S')}.log"
end

.set_records ⇒ Object 



101
102
103
 
# File 'app/services/s3/transform_deleted_files_service.rb', line 101

def set_records
  @records = S3::OldDeletedRecordsService.get_records(env_id: @env.id, env_name: @env.naam, db_table: @table, **@dates)
end

.set_setters(start_date = Date.yesterday) ⇒ Object 



8
9
10
11
12
13
14
 
# File 'app/services/s3/transform_deleted_files_service.rb', line 8

def set_setters(start_date = Date.yesterday)
  @start_date = start_date
  set_logger
  set_bucket
  set_tables
  set_dates(start_date)
end

.set_tables ⇒ Object 



88
89
90
 
# File 'app/services/s3/transform_deleted_files_service.rb', line 88

def set_tables
  @tables = S3::OldDeletedRecordsService::singleton_class::DB_TABLES
end

.upload_file ⇒ Object 



139
140
141
 
# File 'app/services/s3/transform_deleted_files_service.rb', line 139

def upload_file
  @s3.put_object(bucket: BUCKET, key: @file_name, body: @json)
end