Class: MysqlEnv::GlueEnv

Inherits:
Object
  • Object
show all
Defined in:
lib/glue_envs/mysql_glue_env.rb

Constant Summary collapse

VersionKey =

table format primary key - autogenerated integer, should not be visible outside of the db model model_key - the actual key that will be used to store the data version key - not sure if it will be used namespace key - name to identify this is the mysql interface? used to identify metadata for models (should be consistent across models) ModelKey = :my_id

:_rev
NamespaceKey =

derived from timestamp

:mysql_namespace
PersistLayerKey =

Mysql Primary Key ID so we can use auto-incrementing int primary keys with worrying about forcing user data to conform

'__mysql_pk'
@@log = 
TinkitLog.set(self.name, :warn)

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(persist_env, data_model_bindings) ⇒ GlueEnv

Returns a new instance of GlueEnv.



60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 60

def initialize(persist_env, data_model_bindings)
#TODO: Determine if class_name is needed to segment cluster data within user data
#host = "https://sdb.amazonaws.com/"  (not provided by user)
@dbh = MysqlEnv.dbh
#@_file_mgr_table = 'blahblah' #set in file_mgr
mysql_env = persist_env[:env]
#TODO: validations on format

@user_id = mysql_env[:user_id]
@cluster_name = persist_env[:name]
#use namespace generator?
@domain_table_name = "#{mysql_env[:path]}__#{@user_id}"
#data_model_bindings from NodeElementOperations
key_fields = data_model_bindings[:key_fields] 
initial_views_data = data_model_bindings[:views]

@required_instance_keys = key_fields[:required_keys] #DataStructureModels::Tinkit::RequiredInstanceKeys
@required_save_keys = key_fields[:required_keys] #DataStructureModels::Tinkit::RequiredSaveKeys
@node_key = key_fields[:primary_key] #DataStructureModels::Tinkit::NodeKey

@version_key = VersionKey  
@model_key = @node_key #ModelKey
@persist_layer_key = PersistLayerKey
@namespace_key = NamespaceKey
@metadata_keys = [@persist_layer_key, @version_key, @namespace_key] 

initial_table_fields = @required_instance_keys + @required_save_keys + @metadata_keys
initial_table_fields.compact!
initial_table_fields.uniq!
#may want to verify a flat array
node_identifying_keys = [@model_key] #, @version_key]

@user_datastore_location = use_table!(initial_table_fields, node_identifying_keys, @domain_table_name)

@model_save_params = {:dbh => dbh, :table => user_datastore_location, :node_key => @node_key}
@_files_mgr_class = MysqlInterface::FilesMgr
#@_file_mgr_table = 'blah'  #should be overwritten later
@file_mgr_table = create_file_mgr_table
#@views = "temp"
  
end

Instance Attribute Details

#_files_mgr_class ⇒ Object

TODO, Don’t set directly to constant, use accessor



38
39
40
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 38

def _files_mgr_class
  @_files_mgr_class
end

#dbh ⇒ Object

TODO, Don’t set directly to constant, use accessor



38
39
40
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 38

def dbh
  @dbh
end

#file_mgr_table ⇒ Object

TODO, Don’t set directly to constant, use accessor



38
39
40
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 38

def file_mgr_table
  @file_mgr_table
end

#metadata_keys ⇒ Object

TODO, Don’t set directly to constant, use accessor



38
39
40
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 38

def metadata_keys
  @metadata_keys
end

#moab_data ⇒ Object

TODO, Don’t set directly to constant, use accessor



38
39
40
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 38

def moab_data
  @moab_data
end

#model_key ⇒ Object

TODO, Don’t set directly to constant, use accessor



38
39
40
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 38

def model_key
  @model_key
end

#model_save_params ⇒ Object

TODO, Don’t set directly to constant, use accessor



38
39
40
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 38

def model_save_params
  @model_save_params
end

#namespace_key ⇒ Object

TODO, Don’t set directly to constant, use accessor



38
39
40
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 38

def namespace_key
  @namespace_key
end

#node_key ⇒ Object

TODO, Don’t set directly to constant, use accessor



38
39
40
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 38

def node_key
  @node_key
end

#persist_layer_key ⇒ Object

TODO, Don’t set directly to constant, use accessor



38
39
40
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 38

def persist_layer_key
  @persist_layer_key
end

#required_instance_keys ⇒ Object

TODO, Don’t set directly to constant, use accessor



38
39
40
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 38

def required_instance_keys
  @required_instance_keys
end

#required_save_keys ⇒ Object

TODO, Don’t set directly to constant, use accessor



38
39
40
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 38

def required_save_keys
  @required_save_keys
end

#user_datastore_location ⇒ Object

TODO, Don’t set directly to constant, use accessor



38
39
40
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 38

def user_datastore_location
  @user_datastore_location
end

#user_id ⇒ Object

TODO, Don’t set directly to constant, use accessor



38
39
40
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 38

def user_id
  @user_id
end

#version_key ⇒ Object

TODO, Don’t set directly to constant, use accessor



38
39
40
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 38

def version_key
  @version_key
end

#views ⇒ Object

TODO, Don’t set directly to constant, use accessor



38
39
40
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 38

def views
  @views
end

Instance Method Details

#add_columns(table_name, add_cols) ⇒ Object 



360
361
362
363
364
365
366
367
368
369
370
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 360

def add_columns(table_name, add_cols)
  add_list = []
  add_cols.each do |col_name|
    add_list << "ADD `#{col_name}` VARCHAR (255)"
  end
  add_sql= add_list.join(", ")
  sql = "ALTER TABLE `#{table_name}` #{add_sql}"
  sth = @dbh.prepare(sql)
  sth.execute
  sth.finish
end

#create_file_mgr_table ⇒ Object 



400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 400

def create_file_mgr_table
  file_mgr_key = '__pkid-file'
  file_table_name_postfix = "files"
  #Create the table to store files when class is loaded
  #Add modified_at to the UNIQUE KEY to keep versions
  
  file_table_name = "#{@domain_table_name}_#{file_table_name_postfix}"
  sql = "CREATE TABLE IF NOT EXISTS `#{file_table_name}` (
        `#{file_mgr_key}` INT NOT NULL AUTO_INCREMENT,
        node_name VARCHAR(255),
        basename VARCHAR(255) NOT NULL,
        content_type VARCHAR(255),
        modified_at VARCHAR(255),
        raw_content LONGBLOB,
        PRIMARY KEY (`#{file_mgr_key}`),
        UNIQUE KEY (node_name, basename) )"

  @dbh.do(sql) 
  return file_table_name    
end

#create_table(fields, keys, table_name) ⇒ Object 



305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 305

def create_table(fields, keys, table_name)
  rtn_val = nil
  field_str_list = []
  fields.delete(PersistLayerKey)
  #keys.each do |k|
  #  fields.delete(k)
  #end
  fields.each do |field|
    sql_str = "`#{field}` VARCHAR(255) NOT NULL,"
    field_str_list << sql_str
  end
  field_str  = field_str_list.join("\n")
  #change this for tinkit
  mk_str = "UNIQUE KEY `_uniq_idx`(`#{keys.join("`, `")}`)"
  sql = "CREATE TABLE `#{table_name}` (
         `#{PersistLayerKey}` INT NOT NULL AUTO_INCREMENT,
         #{field_str}
         PRIMARY KEY ( `#{PersistLayerKey}` ),
         #{mk_str} )"
  @dbh.do(sql)
  rtn_val = table_name if @dbh.tables.include? table_name
end

#destroy_bulk(records) ⇒ Object 



272
273
274
275
276
277
278
279
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 272

def destroy_bulk(records)
  record_key_data = records.map{|r| r[@model_key].to_json}
  #record_rev = ?
  record_key_data_sql = record_key_data.join("', '")
  sql = "DELETE FROM `#{@user_datastore_location}` 
      WHERE `#{@model_key}` IN ('#{record_key_data_sql}')"
  @dbh.do(sql)
end

#destroy_node(model_metadata) ⇒ Object 



142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 142

def destroy_node(model_metadata)
  @@log.debug "destroy node metadata: #{model_metadata.inspect}"
  key, key_value = if model_metadata[@model_key]
    [@model_key, model_metadata[@model_key].to_json]
  elsif model_metadata[@persist_layer_key]
    [@model_key, model_metadata[@persist_layer_key].to_json ]
  else
    raise "No key in model metadata for deletion"
  end
  node_id = key_value #model_metadata[key]   #persist_layer_key has wrong value here
  node_rev = model_metadata[@version_key]
  #used to be node_id.to_json
  sql = "DELETE FROM `#{@user_datastore_location}` 
       WHERE `#{key}` = '#{node_id}'"
  @dbh.do(sql)
end

#find_contains(key, this_value) ⇒ Object 



238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 238

def find_contains(key, this_value)
  #TODO: Make into map/reduce to be more efficient
  #SQL has native support for this type of lookup, using it.
  sql = "SELECT * FROM `#{@user_datastore_location}`"
  sth = @dbh.prepare(sql)
  rtn_raw_list  = []
  final_rtn = []
  sth.execute
  while row=sth.fetch do
    rowh = row.to_h
    rowh.delete(PersistLayerKey)
    rtn_raw_list << rowh if find_contains_type_helper(rowh[key.to_s], this_value)
  end
  sth.finish
  rtn_raw_list.each do |rtn_raw|
    rtnj = {}
    rtn_raw.each {|k,v| rtnj[k] = jparse(v) }
    final_rtn <<= HashKeys.str_to_sym(rtnj)
  end
  #return full data for select results
  return final_rtn
end

#find_contains_type_helper(stored_dataj, this_value) ⇒ Object 



261
262
263
264
265
266
267
268
269
270
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 261

def find_contains_type_helper(stored_dataj, this_value)
  resp = nil
  stored_data = jparse(stored_dataj)
  if stored_data.respond_to?(:"include?")
    resp = (stored_data.include?(this_value))
  else
    resp = (stored_data == this_value)
  end
  return resp
end

#find_equals(key, this_value) ⇒ Object 



229
230
231
232
233
234
235
236
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 229

def find_equals(key, this_value)    
  results =[]
  query_all.each do |record|
    test_val = record[key]
    results << record  if test_val == this_value
  end
  results 
end

#find_nodes_where(key, relation, this_value) ⇒ Object

current relations supported:

  • :equals (data in the key field matches this_value)

  • :contains (this_value is contained in the key field data (same as equals for non-enumerable types )



219
220
221
222
223
224
225
226
227
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 219

def find_nodes_where(key, relation, this_value)
  res = case relation
    when :equals
      find_equals(key, this_value)
    when :contains
      find_contains(key, this_value)
  end #case
  return res    
end

#find_table!(fields, keys, table_name) ⇒ Object 



296
297
298
299
300
301
302
303
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 296

def find_table!(fields, keys, table_name)
  rtn_val = table_name
  tables = @dbh.tables
  unless tables.include? table_name
    create_table(fields, keys, table_name)
  end    
  rtn_val
end

#generate_model_key(namespace, node_key_value) ⇒ Object 



159
160
161
162
163
164
165
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 159

def generate_model_key(namespace, node_key_value)
  #We can get away with this because the node key value is a first class lookup in mysql
  #and uniqueness is handled within a table (rather than global like some other persistence layers)
  #However, are there any side effects?
  "#{node_key_value}"
  #generate_pk_data(node_key_value)
end

#get(id) ⇒ Object 



118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 118

def get(id)
  @@log.info {"Getting #{id} from #{@user_datastore_location} using key #{@model_key}"} if @@log.info?
  #get all records with the given id and return the one with the highest (internal) primary key
  #because tinkit _rev has no ordering properties ... this may be a bug eventually, ok so far
  sql = "SELECT * FROM `#{@user_datastore_location}` WHERE `#{@model_key}` = '#{id.to_json}'"
  sth = @dbh.prepare(sql)
  rtn = []
  sth.execute
  while row=sth.fetch do
    rtn << row.to_h
  end
  #rtn
  sth.finish
  rtn_raw = rtn.first || {} #remember in production to sort on internal primary id
  rtnj = {}
  rtn_raw.delete(PersistLayerKey)
  rtn_raw.each do |k,v|
        rtnj[k] = jparse(v)
  end
  rtn_h = HashKeys.str_to_sym(rtnj)
  rtn_h = nil if rtn_h.empty?
  return rtn_h
end

#get_existing_columns(table_name) ⇒ Object 



328
329
330
331
332
333
334
335
336
337
338
339
340
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 328

def get_existing_columns(table_name)
  existing_columns = []
  sql = "DESCRIBE #{table_name}"
  sth = @dbh.prepare(sql)
  sth.execute
  sth.each do |row|
    fld = row.to_h['Field']
    existing_columns << fld if fld
  end
  #for select queries this would work sth.column_names (but less efficient)
  sth.finish
  return existing_columns
end

#jparse(str) ⇒ Object

def



421
422
423
424
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 421

def jparse(str)
  return JSON.parse(str) if str =~ /\A\s*[{\[]/
  JSON.parse("[#{str}]")[0]
end

#query_all ⇒ Object

sql = “SELECT ‘#@persist_layer_key` FROM `#@user_datastore_location`

      WHERE `#{@model_key}` = '#{record_id}'"
sth = @dbh.prepare(sql)
rtn = []
sth.execute
while row=sth.fetch do
  rtn << row.to_h
end
#rtn
sth.finish
raise "Multiple records for the unique key: #{@model_key} => #{record_id}" if rtn.size > 1
return nil unless rtn.size > 0
pk_data = rtn.first[@persist_layer_key]

end



189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 189

def query_all
  sql = "SELECT * FROM `#{@user_datastore_location}`"
  sth = @dbh.prepare(sql)
  rtn = []
  rtn_raw_rows = []
  sth.execute
  while row=sth.fetch do
    rtn_raw_rows << row.to_h
  end
  #rtn
  sth.finish
  rtnj = {}
  rtn_raw_rows.each do |rtn_raw|
    rtn_raw.delete(PersistLayerKey)
    rtn_raw.each do |k,v|
        rtnj[k] = jparse(v)
    end
    rtn_h = HashKeys.str_to_sym(rtnj)
    rtn << rtn_h
  end
  return rtn
end

#raw_all ⇒ Object 



212
213
214
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 212

def raw_all
  query_all
end

#reconcile_table(table_name, orig_cols, new_cols, opts = {}) ⇒ Object 



342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 342

def reconcile_table(table_name, orig_cols, new_cols, opts = {} )
  opts[:allow_remove] = opts[:allow_remove] || nil  #useless code but shows options
  #remove_cols = []
  current_cols = get_existing_columns(table_name).map{|col| col.to_sym}
  orig_cols = orig_cols.map{|col| col.to_sym}
  new_cols = new_cols.map{|col| col.to_sym}
  add_cols = new_cols
  add_cols = new_cols - current_cols - [@persist_layer_key]
  add_cols.delete_if{|col| col =~ /^XXXX_/ }
  remove_cols = current_cols - new_cols - [@persist_layer_key]
  remove_cols.delete_if{|col| col !=~ /^XXXX_/ }
  if opts[:allow_remove] 
    remove_columns(table_name, remove_cols) unless remove_cols.empty?
  end
  add_columns(table_name, add_cols) unless add_cols.empty?
  return table_name
end

#remove_columns(table_name, remove_cols) ⇒ Object 



372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 372

def remove_columns(table_name, remove_cols)
  remove_list = []
  existing_cols = get_existing_columns(table_name)
  existing_cols.each do |curr_col_name|
    #existing_cols.each do |ex_col|
    expired_col_regexp = /^XXXX_XXXX_/
      if curr_col_name.match expired_col_regexp
        #iterating in ruby makes cleaner ruby code, but not as efficient
        sql = "ALTER TABLE #{table_name} DROP #{curr_col_name}"
        sth = @dbh.prepare(sql)
        sth.execute
        sth.finish
      end#if
  end
    #end#each existing_cols
  removed_cols = existing_cols - get_existing_columns(table_name)
  remove_cols = remove_cols - removed_cols 
  remove_cols.each do |rmv_col_name|
    remove_list << "CHANGE `#{rmv_col_name}` `XXXX_#{rmv_col_name}` VARCHAR(255)"
  end#each remove_cols
  existing_cols = get_existing_columns(table_name)
  remove_sql= remove_list.join(", ")
  sql = "ALTER TABLE `#{table_name}` #{remove_sql}"
  sth = @dbh.prepare(sql)
  sth.execute
  sth.finish
end

#save(new_data) ⇒ Object

def



102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 102

def save(new_data)
  #raise "Required key missing" unless @model_save_params[:required_save_key] = @required_save_key
  rev =  Time.now.hash.to_s
  new_data[@version_key] = rev
  orig_cols = get_existing_columns(@user_datastore_location)
  new_cols = new_data.keys
  table_name = reconcile_table(@user_datastore_location, orig_cols, new_cols)
  esc_col_names = new_data.keys.map{|k| "`#{k}`"}.join(",")
  json_values = new_data.values.map{|v| "'#{v.to_json}'"}.join(",")
  
  #Need to update a bit when moved to tinkit (formerly bufs) to account for revs
  sql = "REPLACE INTO `#{table_name}` (#{esc_col_names}) VALUEs (#{json_values})"
  @dbh.do(sql)
  new_data['rev'] = new_data[@version_key]
end

#use_table!(fields, keys, table_name) ⇒ Object 



281
282
283
284
285
286
287
288
289
290
291
292
293
294
 
# File 'lib/glue_envs/mysql_glue_env.rb', line 281

def use_table!(fields, keys, table_name)
  #return [fields, keys, table_name]
  table_name = find_table!(fields, keys, table_name)
  raise "No table could be found or created" unless table_name
  column_names = get_existing_columns(table_name)
  column_names = get_existing_columns(table_name)
  fields_str = fields.map{|f| f.to_s}
  unless fields_str.sort == column_names.sort
    #puts "Warning Fields Dont Match, Adding unmatched fields to table"
    #table has changed, reconcile them
    table_name = reconcile_table(table_name, column_names, fields_str)
  end
  return table_name
end