Module: CanvasDataClient::Helpers::CsvHelper

Included in:
Client
Defined in:
lib/canvas_data_client/helpers/csv_helper.rb

Defined Under Namespace

Classes: TableNotPresentError

Instance Method Summary collapse

Instance Method Details

#download_latest_to_csv_file(table:, path:) ⇒ Object 



35
36
37
38
 
# File 'lib/canvas_data_client/helpers/csv_helper.rb', line 35

def download_latest_to_csv_file(table:, path:)
  latest_dump = latest
  download_to_csv_file dump_id: latest_dump['dumpId'], table: table, path: path
end

#download_to_csv_file(dump_id:, table:, path:) ⇒ Object 



8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/canvas_data_client/helpers/csv_helper.rb', line 8

def download_to_csv_file(dump_id:, table:, path:)
  dump_definition = dump(dump_id)
  schema_definition = schema(dump_definition['schemaVersion'])
  raise TableNotPresentError.new("Table #{table} not present in dump #{dump_id}") unless dump_definition['artifactsByTable'][table]

  csv = CSV.open(path, 'w')
  columns = table_headers(schema_definition, table)
  csv << columns

  Dir.mktmpdir do |dir|
    dump_definition['artifactsByTable'][table]['files'].each do |file_mapping|
      renew_urls(dump_id, table, dump_definition['artifactsByTable'][table]['files']) if url_expired?(file_mapping['url'])
      logger.info("Downloading table file: #{file_mapping['filename']}")
      file_path = download_raw_file(file_mapping, dir)
      logger.info("Processing table file: #{file_mapping['filename']}")
      File.foreach(file_path) do |row|
        split_row = row.gsub(/\n/, '').split(/\t/)
        split_row.fill(nil, split_row.length...columns.length) if split_row.length < columns.length
        csv << split_row.map { |col| col == '\\N' ? nil : col }
      end
      FileUtils.rm_f file_path
    end
  end
ensure
  csv.close if csv
end