Class: Fech::Table

Inherits:
Object
  • Object
show all
Defined in:
lib/fech-ftp/table.rb,
lib/fech-ftp/table_methods.rb

Constant Summary collapse

AWS_URL = 
"https://cg-519a459a-0ea3-42c2-b7bc-fa1143481f74.s3-us-gov-west-1.amazonaws.com/bulk-downloads"

Instance Method Summary collapse

Constructor Details

#initialize(cycle, opts = {}) ⇒ Table

Returns a new instance of Table.



5
6
7
8
9
10
11
12
13
14
 
# File 'lib/fech-ftp/table.rb', line 5

def initialize(cycle, opts={})
  @cycle    = cycle
  @headers  = opts[:headers]
  @file     = opts[:file]
  @format   = opts[:format]
  @location = opts[:location]
  @passive  = opts[:passive]
  @receiver = opts[:connection] || receiver
  @parser   = parser
end

Instance Method Details

#create_table(row) ⇒ Object 



47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
 
# File 'lib/fech-ftp/table.rb', line 47

def create_table(row)
  db, table = @receiver
  table = table.to_s.pluralize.to_sym
  db.create_table(table) { primary_key :id }

  row.each do |k,v|
    v = v.nil? ? String : v.class
    db.alter_table table do
      add_column k, v
    end
  end

  @receiver = db[table]
  @receiver << row
end

#enter_row(row) ⇒ Object 



29
30
31
32
33
34
35
36
37
38
 
# File 'lib/fech-ftp/table.rb', line 29

def enter_row(row)
  case @format
  when :db
    table_exist? ? @receiver << row : create_table(row)
  when :csv
    @receiver << row.values
  else
    @receiver << row
  end
end

#fetch_file(&blk) ⇒ Object 



63
64
65
66
67
68
69
70
71
 
# File 'lib/fech-ftp/table.rb', line 63

def fetch_file(&blk)
  filename = "#{@file}#{@cycle.to_s[2..3]}.zip"
  uri = URI("#{AWS_URL}/#{@cycle}/#{filename}")
  response = Net::HTTP.get_response(uri)

  if response.code == '200'
    unzip(response.body, &blk)
  end
end

#format_row(line) ⇒ Object 



87
88
89
90
91
92
93
94
 
# File 'lib/fech-ftp/table.rb', line 87

def format_row(line)
  hash = {}
  line = line.encode('UTF-8', invalid: :replace, replace: ' ').chomp.split("|")

  @parser.each { |k,blk| hash[k] = blk.call(line) }

  return hash
end

#parse_date(date) ⇒ Object 



96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
 
# File 'lib/fech-ftp/table.rb', line 96

def parse_date(date)
  if date == '' && table_exist?
    if table_exist?
      return Date.new(@cycle, 1,1)
    else
      return ''
    end
  end

  if date.length == 8
    Date.strptime(date, "%m%d%Y")
  else
    Date.parse(date)
  end
end

#parser ⇒ Object 



73
74
75
76
77
78
79
80
81
82
83
84
85
 
# File 'lib/fech-ftp/table.rb', line 73

def parser
  @headers.map.with_index do |h,i|
    if h.to_s =~ /cash|amount|contributions|total|loan|transfer|debts|refund|expenditure/
      [h, ->(line) { line[i].to_f }]
    elsif h == :filing_id
      [h, ->(line) { line[i].to_i }]
    elsif h.to_s =~ /_date/
      [h, ->(line) { parse_date(line[i]) }]
    else
      [h, ->(line) { line[i] }]
    end
  end
end

#receiver ⇒ Object 



16
17
18
19
20
21
22
 
# File 'lib/fech-ftp/table.rb', line 16

def receiver
  if @format == :csv
    CSV.open("#{@file}#{@cycle.to_s[2..3]}.csv", 'a+', headers: @headers, write_headers: true)
  else
    []
  end
end

#retrieve_data ⇒ Object 



24
25
26
27
 
# File 'lib/fech-ftp/table.rb', line 24

def retrieve_data
  fetch_file { |row| enter_row(row) }
  return @receiver
end

#table_exist? ⇒ Boolean

the @receiver obj is the database itself. This assumes the table needs to be created.

Returns:

  • (Boolean) 


43
44
45
 
# File 'lib/fech-ftp/table.rb', line 43

def table_exist?
  @receiver.respond_to? :columns
end

#unzip(zip_file, &blk) ⇒ Object 



112
113
114
115
116
117
118
119
120
121
122
123
 
# File 'lib/fech-ftp/table.rb', line 112

def unzip(zip_file, &blk)
  Zip::File.open_buffer(zip_file) do |zip|
    zip.each do |entry|
      path = @location.nil? ? entry.name : @location + entry.name
      entry.extract(path) if !File.file?(path)

      File.foreach(path) do |row|
        blk.call(format_row(row))
      end
    end
  end
end