Class: Bricolage::PSQLDataSource

Inherits:
DataSource show all
Includes:
CommandUtils, VacuumLock
Defined in:
lib/bricolage/psqldatasource.rb

Constant Summary

Constants included from VacuumLock

VacuumLock::DEFAULT_VACUUM_LOCK_FILE, VacuumLock::DEFAULT_VACUUM_LOCK_TIMEOUT

Constants inherited from DataSource

DataSource::CLASSES

Instance Attribute Summary collapse

Attributes inherited from DataSource

#context, #logger, #name

Instance Method Summary collapse

Methods included from VacuumLock

cleanup_vacuum_lock, create_lockfile_cmd, create_vacuum_lock_file, enable_vacuum_lock?, locking?, psql_serialize_vacuum_begin, psql_serialize_vacuum_end, serialize_vacuum, using, #using_vacuum_lock, vacuum_lock_parameters

Methods included from CommandUtils

#command, #make_tmpfile, #new_tmpfile_path

Methods inherited from DataSource

get_class, new_for_type, #redshift_loader_source?

Constructor Details

#initialize(host: 'localhost', port: 5439, database: 'dev', username: ENV['LOGNAME'], password: nil, pgpass: nil, encoding: nil, psql: 'psql', sql_log_level: Logger::INFO, tmpdir: Dir.tmpdir) ⇒ PSQLDataSource

Returns a new instance of PSQLDataSource.

Raises:



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
 
# File 'lib/bricolage/psqldatasource.rb', line 18

def initialize(
    host: 'localhost',
    port: 5439,
    database: 'dev',
    username: ENV['LOGNAME'],
    password: nil,
    pgpass: nil,
    encoding: nil,
    psql: 'psql',
    sql_log_level: Logger::INFO,
    tmpdir: Dir.tmpdir)
  @host = host
  @port = port
  @database = database
  @user = username
  @password = password
  @pgpass = pgpass
  @encoding = encoding
  @psql = psql
  @sql_log_level = Logger.intern_severity(sql_log_level)
  @tmpdir = tmpdir
  raise ParameterError, "missing psql host" unless @host
  raise ParameterError, "missing psql port" unless @port
  raise ParameterError, "missing psql database" unless @database
  raise ParameterError, "missing psql username" unless @user
  unless @pgpass or @password
    raise ParameterError, "missing psql password"
  end
end

Instance Attribute Details

#database ⇒ Object (readonly)

Returns the value of attribute database.



50
51
52
 
# File 'lib/bricolage/psqldatasource.rb', line 50

def database
  @database
end

#host ⇒ Object (readonly)

Returns the value of attribute host.



48
49
50
 
# File 'lib/bricolage/psqldatasource.rb', line 48

def host
  @host
end

#port ⇒ Object (readonly)

Returns the value of attribute port.



49
50
51
 
# File 'lib/bricolage/psqldatasource.rb', line 49

def port
  @port
end

#sql_log_level ⇒ Object (readonly)

Returns the value of attribute sql_log_level.



53
54
55
 
# File 'lib/bricolage/psqldatasource.rb', line 53

def sql_log_level
  @sql_log_level
end

#user ⇒ Object (readonly)

Returns the value of attribute user.



51
52
53
 
# File 'lib/bricolage/psqldatasource.rb', line 51

def user
  @user
end

Instance Method Details

#analyze(table) ⇒ Object 



145
146
147
 
# File 'lib/bricolage/psqldatasource.rb', line 145

def analyze(table)
  open {|conn| conn.analyze(table) }
end

#drop_table(name) ⇒ Object 



119
120
121
 
# File 'lib/bricolage/psqldatasource.rb', line 119

def drop_table(name)
  open {|conn| conn.drop_table(name) }
end

#drop_table_force(name) ⇒ Object 



123
124
125
 
# File 'lib/bricolage/psqldatasource.rb', line 123

def drop_table_force(name)
  open {|conn| conn.drop_table_force(name) }
end

#execute(source, options = []) ⇒ Object 



64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
 
# File 'lib/bricolage/psqldatasource.rb', line 64

def execute(source, options = [])
  make_tmpfile(source, tmpdir: @tmpdir) {|path|
    st = command @psql, "--no-psqlrc", "--host=#{@host}", "--port=#{@port}",
        "--username=#{@user}", @database,
        '--echo-all',
        '-v', 'ON_ERROR_STOP=true',
        '-f', path,
        '--no-password',
        *options,
        env: get_psql_env
    unless st.success?
      begin
        msg = LogLocator.slice_last_stderr(/^psql:.*?:\d+: ERROR: (.*)/, 1)
      rescue IOError => ex
        # slice_last_stderr may fail if stderr is not a file
        logger.error ex.message
        msg = nil
      end
    end
    JobResult.for_process_status(st, msg)
  }
end

#get_psql_env ⇒ Object 



87
88
89
90
91
92
93
94
95
 
# File 'lib/bricolage/psqldatasource.rb', line 87

def get_psql_env
  env = {}
  if @pgpass
    env["PGPASSFILE"] = @pgpass
  elsif @password
    env["PGPASSWORD"] = @password
  end
  env
end

#new_task ⇒ Object 



55
56
57
 
# File 'lib/bricolage/psqldatasource.rb', line 55

def new_task
  PSQLTask.new(self)
end

#open(&block) ⇒ Object 



111
112
113
 
# File 'lib/bricolage/psqldatasource.rb', line 111

def open(&block)
  PostgresConnection.open_data_source(self, &block)
end

#open_for_batch ⇒ Object 



59
60
61
62
 
# File 'lib/bricolage/psqldatasource.rb', line 59

def open_for_batch
  # do not call #open
  yield
end

#password ⇒ Object

Ruby Library Interface



101
102
103
104
 
# File 'lib/bricolage/psqldatasource.rb', line 101

def password
  # FIXME: same user must not exist
  @password ||= read_password_from_pgpass(@pgpass, @user)
end

#query_batch(query, batch_size = 5000, &block) ⇒ Object 



115
116
117
 
# File 'lib/bricolage/psqldatasource.rb', line 115

def query_batch(query, batch_size = 5000, &block)
  open {|conn| conn.query_batch(query, batch_size, &block) }
end

#read_password_from_pgpass(path, user) ⇒ Object 



106
107
108
109
 
# File 'lib/bricolage/psqldatasource.rb', line 106

def read_password_from_pgpass(path, user)
  File.read(path).slice(/:#{user}:([^:\r\n]+)$/, 1) or
      raise ParameterError, "could not read password: #{path}, #{user}"
end

#select(table, &block) ⇒ Object 



127
128
129
 
# File 'lib/bricolage/psqldatasource.rb', line 127

def select(table, &block)
  open {|conn| conn.select(table, &block) }
end

#vacuum(table) ⇒ Object 



133
134
135
136
137
 
# File 'lib/bricolage/psqldatasource.rb', line 133

def vacuum(table)
  serialize_vacuum {
    open {|conn| conn.vacuum(table) }
  }
end

#vacuum_sort_only(table) ⇒ Object 



139
140
141
142
143
 
# File 'lib/bricolage/psqldatasource.rb', line 139

def vacuum_sort_only(table)
  serialize_vacuum {
    open {|conn| conn.vacuum_sort_only(table) }
  }
end