Class: MysqlEnv::GlueEnv
- Inherits:
-
Object
- Object
- MysqlEnv::GlueEnv
- Defined in:
- lib/glue_envs/mysql_glue_env.rb
Constant Summary collapse
- VersionKey =
table format primary key - autogenerated integer, should not be visible outside of the db model model_key - the actual key that will be used to store the data version key - not sure if it will be used namespace key - name to identify this is the mysql interface? used to identify metadata for models (should be consistent across models) ModelKey = :my_id
:_rev
- NamespaceKey =
derived from timestamp
:mysql_namespace
- PersistLayerKey =
Mysql Primary Key ID so we can use auto-incrementing int primary keys with worrying about forcing user data to conform
'__mysql_pk'
- @@log =
TinkitLog.set(self.name, :warn)
Instance Attribute Summary collapse
-
#_files_mgr_class ⇒ Object
TODO, Don’t set directly to constant, use accessor.
-
#dbh ⇒ Object
TODO, Don’t set directly to constant, use accessor.
-
#file_mgr_table ⇒ Object
TODO, Don’t set directly to constant, use accessor.
-
#metadata_keys ⇒ Object
TODO, Don’t set directly to constant, use accessor.
-
#moab_data ⇒ Object
TODO, Don’t set directly to constant, use accessor.
-
#model_key ⇒ Object
TODO, Don’t set directly to constant, use accessor.
-
#model_save_params ⇒ Object
TODO, Don’t set directly to constant, use accessor.
-
#namespace_key ⇒ Object
TODO, Don’t set directly to constant, use accessor.
-
#node_key ⇒ Object
TODO, Don’t set directly to constant, use accessor.
-
#persist_layer_key ⇒ Object
TODO, Don’t set directly to constant, use accessor.
-
#required_instance_keys ⇒ Object
TODO, Don’t set directly to constant, use accessor.
-
#required_save_keys ⇒ Object
TODO, Don’t set directly to constant, use accessor.
-
#user_datastore_location ⇒ Object
TODO, Don’t set directly to constant, use accessor.
-
#user_id ⇒ Object
TODO, Don’t set directly to constant, use accessor.
-
#version_key ⇒ Object
TODO, Don’t set directly to constant, use accessor.
-
#views ⇒ Object
TODO, Don’t set directly to constant, use accessor.
Instance Method Summary collapse
- #add_columns(table_name, add_cols) ⇒ Object
- #create_file_mgr_table ⇒ Object
- #create_table(fields, keys, table_name) ⇒ Object
- #destroy_bulk(records) ⇒ Object
- #destroy_node(model_metadata) ⇒ Object
- #find_contains(key, this_value) ⇒ Object
- #find_contains_type_helper(stored_dataj, this_value) ⇒ Object
- #find_equals(key, this_value) ⇒ Object
-
#find_nodes_where(key, relation, this_value) ⇒ Object
current relations supported: - :equals (data in the key field matches this_value) - :contains (this_value is contained in the key field data (same as equals for non-enumerable types ).
- #find_table!(fields, keys, table_name) ⇒ Object
- #generate_model_key(namespace, node_key_value) ⇒ Object
- #get(id) ⇒ Object
- #get_existing_columns(table_name) ⇒ Object
-
#initialize(persist_env, data_model_bindings) ⇒ GlueEnv
constructor
A new instance of GlueEnv.
-
#jparse(str) ⇒ Object
def.
-
#query_all ⇒ Object
sql = “SELECT ‘#@persist_layer_key` FROM `#@user_datastore_location` WHERE `#@model_key` = ’#record_id‘” sth = @dbh.prepare(sql) rtn = [] sth.execute while row=sth.fetch do rtn << row.to_h end #rtn sth.finish raise “Multiple records for the unique key: #@model_key => #record_id” if rtn.size > 1 return nil unless rtn.size > 0 pk_data = rtn.first end.
- #raw_all ⇒ Object
- #reconcile_table(table_name, orig_cols, new_cols, opts = {}) ⇒ Object
- #remove_columns(table_name, remove_cols) ⇒ Object
-
#save(new_data) ⇒ Object
def.
- #use_table!(fields, keys, table_name) ⇒ Object
Constructor Details
#initialize(persist_env, data_model_bindings) ⇒ GlueEnv
Returns a new instance of GlueEnv.
60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 60 def initialize(persist_env, data_model_bindings) #TODO: Determine if class_name is needed to segment cluster data within user data #host = "https://sdb.amazonaws.com/" (not provided by user) @dbh = MysqlEnv.dbh #@_file_mgr_table = 'blahblah' #set in file_mgr mysql_env = persist_env[:env] #TODO: validations on format @user_id = mysql_env[:user_id] @cluster_name = persist_env[:name] #use namespace generator? @domain_table_name = "#{mysql_env[:path]}__#{@user_id}" #data_model_bindings from NodeElementOperations key_fields = data_model_bindings[:key_fields] initial_views_data = data_model_bindings[:views] @required_instance_keys = key_fields[:required_keys] #DataStructureModels::Tinkit::RequiredInstanceKeys @required_save_keys = key_fields[:required_keys] #DataStructureModels::Tinkit::RequiredSaveKeys @node_key = key_fields[:primary_key] #DataStructureModels::Tinkit::NodeKey @version_key = VersionKey @model_key = @node_key #ModelKey @persist_layer_key = PersistLayerKey @namespace_key = NamespaceKey @metadata_keys = [@persist_layer_key, @version_key, @namespace_key] initial_table_fields = @required_instance_keys + @required_save_keys + @metadata_keys initial_table_fields.compact! initial_table_fields.uniq! #may want to verify a flat array = [@model_key] #, @version_key] @user_datastore_location = use_table!(initial_table_fields, , @domain_table_name) @model_save_params = {:dbh => dbh, :table => user_datastore_location, :node_key => @node_key} @_files_mgr_class = MysqlInterface::FilesMgr #@_file_mgr_table = 'blah' #should be overwritten later @file_mgr_table = create_file_mgr_table #@views = "temp" end |
Instance Attribute Details
#_files_mgr_class ⇒ Object
TODO, Don’t set directly to constant, use accessor
38 39 40 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 38 def _files_mgr_class @_files_mgr_class end |
#dbh ⇒ Object
TODO, Don’t set directly to constant, use accessor
38 39 40 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 38 def dbh @dbh end |
#file_mgr_table ⇒ Object
TODO, Don’t set directly to constant, use accessor
38 39 40 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 38 def file_mgr_table @file_mgr_table end |
#metadata_keys ⇒ Object
TODO, Don’t set directly to constant, use accessor
38 39 40 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 38 def @metadata_keys end |
#moab_data ⇒ Object
TODO, Don’t set directly to constant, use accessor
38 39 40 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 38 def moab_data @moab_data end |
#model_key ⇒ Object
TODO, Don’t set directly to constant, use accessor
38 39 40 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 38 def model_key @model_key end |
#model_save_params ⇒ Object
TODO, Don’t set directly to constant, use accessor
38 39 40 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 38 def model_save_params @model_save_params end |
#namespace_key ⇒ Object
TODO, Don’t set directly to constant, use accessor
38 39 40 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 38 def namespace_key @namespace_key end |
#node_key ⇒ Object
TODO, Don’t set directly to constant, use accessor
38 39 40 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 38 def node_key @node_key end |
#persist_layer_key ⇒ Object
TODO, Don’t set directly to constant, use accessor
38 39 40 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 38 def persist_layer_key @persist_layer_key end |
#required_instance_keys ⇒ Object
TODO, Don’t set directly to constant, use accessor
38 39 40 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 38 def required_instance_keys @required_instance_keys end |
#required_save_keys ⇒ Object
TODO, Don’t set directly to constant, use accessor
38 39 40 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 38 def required_save_keys @required_save_keys end |
#user_datastore_location ⇒ Object
TODO, Don’t set directly to constant, use accessor
38 39 40 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 38 def user_datastore_location @user_datastore_location end |
#user_id ⇒ Object
TODO, Don’t set directly to constant, use accessor
38 39 40 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 38 def user_id @user_id end |
#version_key ⇒ Object
TODO, Don’t set directly to constant, use accessor
38 39 40 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 38 def version_key @version_key end |
#views ⇒ Object
TODO, Don’t set directly to constant, use accessor
38 39 40 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 38 def views @views end |
Instance Method Details
#add_columns(table_name, add_cols) ⇒ Object
360 361 362 363 364 365 366 367 368 369 370 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 360 def add_columns(table_name, add_cols) add_list = [] add_cols.each do |col_name| add_list << "ADD `#{col_name}` VARCHAR (255)" end add_sql= add_list.join(", ") sql = "ALTER TABLE `#{table_name}` #{add_sql}" sth = @dbh.prepare(sql) sth.execute sth.finish end |
#create_file_mgr_table ⇒ Object
400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 400 def create_file_mgr_table file_mgr_key = '__pkid-file' file_table_name_postfix = "files" #Create the table to store files when class is loaded #Add modified_at to the UNIQUE KEY to keep versions file_table_name = "#{@domain_table_name}_#{file_table_name_postfix}" sql = "CREATE TABLE IF NOT EXISTS `#{file_table_name}` ( `#{file_mgr_key}` INT NOT NULL AUTO_INCREMENT, node_name VARCHAR(255), basename VARCHAR(255) NOT NULL, content_type VARCHAR(255), modified_at VARCHAR(255), raw_content LONGBLOB, PRIMARY KEY (`#{file_mgr_key}`), UNIQUE KEY (node_name, basename) )" @dbh.do(sql) return file_table_name end |
#create_table(fields, keys, table_name) ⇒ Object
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 305 def create_table(fields, keys, table_name) rtn_val = nil field_str_list = [] fields.delete(PersistLayerKey) #keys.each do |k| # fields.delete(k) #end fields.each do |field| sql_str = "`#{field}` VARCHAR(255) NOT NULL," field_str_list << sql_str end field_str = field_str_list.join("\n") #change this for tinkit mk_str = "UNIQUE KEY `_uniq_idx`(`#{keys.join("`, `")}`)" sql = "CREATE TABLE `#{table_name}` ( `#{PersistLayerKey}` INT NOT NULL AUTO_INCREMENT, #{field_str} PRIMARY KEY ( `#{PersistLayerKey}` ), #{mk_str} )" @dbh.do(sql) rtn_val = table_name if @dbh.tables.include? table_name end |
#destroy_bulk(records) ⇒ Object
272 273 274 275 276 277 278 279 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 272 def destroy_bulk(records) record_key_data = records.map{|r| r[@model_key].to_json} #record_rev = ? record_key_data_sql = record_key_data.join("', '") sql = "DELETE FROM `#{@user_datastore_location}` WHERE `#{@model_key}` IN ('#{record_key_data_sql}')" @dbh.do(sql) end |
#destroy_node(model_metadata) ⇒ Object
142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 142 def destroy_node() @@log.debug "destroy node metadata: #{.inspect}" key, key_value = if [@model_key] [@model_key, [@model_key].to_json] elsif [@persist_layer_key] [@model_key, [@persist_layer_key].to_json ] else raise "No key in model metadata for deletion" end node_id = key_value #model_metadata[key] #persist_layer_key has wrong value here node_rev = [@version_key] #used to be node_id.to_json sql = "DELETE FROM `#{@user_datastore_location}` WHERE `#{key}` = '#{node_id}'" @dbh.do(sql) end |
#find_contains(key, this_value) ⇒ Object
238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 238 def find_contains(key, this_value) #TODO: Make into map/reduce to be more efficient #SQL has native support for this type of lookup, using it. sql = "SELECT * FROM `#{@user_datastore_location}`" sth = @dbh.prepare(sql) rtn_raw_list = [] final_rtn = [] sth.execute while row=sth.fetch do rowh = row.to_h rowh.delete(PersistLayerKey) rtn_raw_list << rowh if find_contains_type_helper(rowh[key.to_s], this_value) end sth.finish rtn_raw_list.each do |rtn_raw| rtnj = {} rtn_raw.each {|k,v| rtnj[k] = jparse(v) } final_rtn <<= HashKeys.str_to_sym(rtnj) end #return full data for select results return final_rtn end |
#find_contains_type_helper(stored_dataj, this_value) ⇒ Object
261 262 263 264 265 266 267 268 269 270 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 261 def find_contains_type_helper(stored_dataj, this_value) resp = nil stored_data = jparse(stored_dataj) if stored_data.respond_to?(:"include?") resp = (stored_data.include?(this_value)) else resp = (stored_data == this_value) end return resp end |
#find_equals(key, this_value) ⇒ Object
229 230 231 232 233 234 235 236 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 229 def find_equals(key, this_value) results =[] query_all.each do |record| test_val = record[key] results << record if test_val == this_value end results end |
#find_nodes_where(key, relation, this_value) ⇒ Object
current relations supported:
-
:equals (data in the key field matches this_value)
-
:contains (this_value is contained in the key field data (same as equals for non-enumerable types )
219 220 221 222 223 224 225 226 227 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 219 def find_nodes_where(key, relation, this_value) res = case relation when :equals find_equals(key, this_value) when :contains find_contains(key, this_value) end #case return res end |
#find_table!(fields, keys, table_name) ⇒ Object
296 297 298 299 300 301 302 303 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 296 def find_table!(fields, keys, table_name) rtn_val = table_name tables = @dbh.tables unless tables.include? table_name create_table(fields, keys, table_name) end rtn_val end |
#generate_model_key(namespace, node_key_value) ⇒ Object
159 160 161 162 163 164 165 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 159 def generate_model_key(namespace, node_key_value) #We can get away with this because the node key value is a first class lookup in mysql #and uniqueness is handled within a table (rather than global like some other persistence layers) #However, are there any side effects? "#{node_key_value}" #generate_pk_data(node_key_value) end |
#get(id) ⇒ Object
118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 118 def get(id) @@log.info {"Getting #{id} from #{@user_datastore_location} using key #{@model_key}"} if @@log.info? #get all records with the given id and return the one with the highest (internal) primary key #because tinkit _rev has no ordering properties ... this may be a bug eventually, ok so far sql = "SELECT * FROM `#{@user_datastore_location}` WHERE `#{@model_key}` = '#{id.to_json}'" sth = @dbh.prepare(sql) rtn = [] sth.execute while row=sth.fetch do rtn << row.to_h end #rtn sth.finish rtn_raw = rtn.first || {} #remember in production to sort on internal primary id rtnj = {} rtn_raw.delete(PersistLayerKey) rtn_raw.each do |k,v| rtnj[k] = jparse(v) end rtn_h = HashKeys.str_to_sym(rtnj) rtn_h = nil if rtn_h.empty? return rtn_h end |
#get_existing_columns(table_name) ⇒ Object
328 329 330 331 332 333 334 335 336 337 338 339 340 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 328 def get_existing_columns(table_name) existing_columns = [] sql = "DESCRIBE #{table_name}" sth = @dbh.prepare(sql) sth.execute sth.each do |row| fld = row.to_h['Field'] existing_columns << fld if fld end #for select queries this would work sth.column_names (but less efficient) sth.finish return existing_columns end |
#jparse(str) ⇒ Object
def
421 422 423 424 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 421 def jparse(str) return JSON.parse(str) if str =~ /\A\s*[{\[]/ JSON.parse("[#{str}]")[0] end |
#query_all ⇒ Object
sql = “SELECT ‘#@persist_layer_key` FROM `#@user_datastore_location`
WHERE `#{@model_key}` = '#{record_id}'"
sth = @dbh.prepare(sql)
rtn = []
sth.execute
while row=sth.fetch do
rtn << row.to_h
end
#rtn
sth.finish
raise "Multiple records for the unique key: #{@model_key} => #{record_id}" if rtn.size > 1
return nil unless rtn.size > 0
pk_data = rtn.first[@persist_layer_key]
end
189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 189 def query_all sql = "SELECT * FROM `#{@user_datastore_location}`" sth = @dbh.prepare(sql) rtn = [] rtn_raw_rows = [] sth.execute while row=sth.fetch do rtn_raw_rows << row.to_h end #rtn sth.finish rtnj = {} rtn_raw_rows.each do |rtn_raw| rtn_raw.delete(PersistLayerKey) rtn_raw.each do |k,v| rtnj[k] = jparse(v) end rtn_h = HashKeys.str_to_sym(rtnj) rtn << rtn_h end return rtn end |
#raw_all ⇒ Object
212 213 214 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 212 def raw_all query_all end |
#reconcile_table(table_name, orig_cols, new_cols, opts = {}) ⇒ Object
342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 342 def reconcile_table(table_name, orig_cols, new_cols, opts = {} ) opts[:allow_remove] = opts[:allow_remove] || nil #useless code but shows options #remove_cols = [] current_cols = get_existing_columns(table_name).map{|col| col.to_sym} orig_cols = orig_cols.map{|col| col.to_sym} new_cols = new_cols.map{|col| col.to_sym} add_cols = new_cols add_cols = new_cols - current_cols - [@persist_layer_key] add_cols.delete_if{|col| col =~ /^XXXX_/ } remove_cols = current_cols - new_cols - [@persist_layer_key] remove_cols.delete_if{|col| col !=~ /^XXXX_/ } if opts[:allow_remove] remove_columns(table_name, remove_cols) unless remove_cols.empty? end add_columns(table_name, add_cols) unless add_cols.empty? return table_name end |
#remove_columns(table_name, remove_cols) ⇒ Object
372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 372 def remove_columns(table_name, remove_cols) remove_list = [] existing_cols = get_existing_columns(table_name) existing_cols.each do |curr_col_name| #existing_cols.each do |ex_col| expired_col_regexp = /^XXXX_XXXX_/ if curr_col_name.match expired_col_regexp #iterating in ruby makes cleaner ruby code, but not as efficient sql = "ALTER TABLE #{table_name} DROP #{curr_col_name}" sth = @dbh.prepare(sql) sth.execute sth.finish end#if end #end#each existing_cols removed_cols = existing_cols - get_existing_columns(table_name) remove_cols = remove_cols - removed_cols remove_cols.each do |rmv_col_name| remove_list << "CHANGE `#{rmv_col_name}` `XXXX_#{rmv_col_name}` VARCHAR(255)" end#each remove_cols existing_cols = get_existing_columns(table_name) remove_sql= remove_list.join(", ") sql = "ALTER TABLE `#{table_name}` #{remove_sql}" sth = @dbh.prepare(sql) sth.execute sth.finish end |
#save(new_data) ⇒ Object
def
102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 102 def save(new_data) #raise "Required key missing" unless @model_save_params[:required_save_key] = @required_save_key rev = Time.now.hash.to_s new_data[@version_key] = rev orig_cols = get_existing_columns(@user_datastore_location) new_cols = new_data.keys table_name = reconcile_table(@user_datastore_location, orig_cols, new_cols) esc_col_names = new_data.keys.map{|k| "`#{k}`"}.join(",") json_values = new_data.values.map{|v| "'#{v.to_json}'"}.join(",") #Need to update a bit when moved to tinkit (formerly bufs) to account for revs sql = "REPLACE INTO `#{table_name}` (#{esc_col_names}) VALUEs (#{json_values})" @dbh.do(sql) new_data['rev'] = new_data[@version_key] end |
#use_table!(fields, keys, table_name) ⇒ Object
281 282 283 284 285 286 287 288 289 290 291 292 293 294 |
# File 'lib/glue_envs/mysql_glue_env.rb', line 281 def use_table!(fields, keys, table_name) #return [fields, keys, table_name] table_name = find_table!(fields, keys, table_name) raise "No table could be found or created" unless table_name column_names = get_existing_columns(table_name) column_names = get_existing_columns(table_name) fields_str = fields.map{|f| f.to_s} unless fields_str.sort == column_names.sort #puts "Warning Fields Dont Match, Adding unmatched fields to table" #table has changed, reconcile them table_name = reconcile_table(table_name, column_names, fields_str) end return table_name end |