Class: VMC::KNIFE::DataService
- Inherits:
-
Object
- Object
- VMC::KNIFE::DataService
- Includes:
- Interactive
- Defined in:
- lib/vmc_knife/vmc_knife.rb,
lib/vmc_knife/data_services.rb
Overview
Read/Write the JSON for a dataservice. Does not map the actual JSON into a new ruby object.
Instance Attribute Summary collapse
-
#role_name ⇒ Object
Returns the value of attribute role_name.
-
#root ⇒ Object
Returns the value of attribute root.
-
#wrapped ⇒ Object
Returns the value of attribute wrapped.
Instance Method Summary collapse
-
#apply_privileges(app_name = nil) ⇒ Object
Make sure that all users who can connect to the DB can also access the tables.
-
#credentials(app_name = nil) ⇒ Object
The credentials hash for this data-service param app_name The favored app usually bound to this data_service.
- #drop(collection_or_table_names = nil) ⇒ Object
- #export(app_name = nil, file = nil) ⇒ Object
- #import(app_name = nil, file = nil) ⇒ Object
-
#initialize(root, data, role_name) ⇒ DataService
constructor
A new instance of DataService.
- #is_mongodb ⇒ Object
- #is_postgresql ⇒ Object
- #log(target_folder) ⇒ Object
-
#name ⇒ Object
returns the name of the service for cloudfoundry.
-
#shell(commands_file = nil, as_admin = false, return_value = false) ⇒ Object
Connect to the mongo js shell or the psql shell.
-
#shrink(collection_or_table_names = nil) ⇒ Object
shrink the size of the databses on the file system.
-
#to_vcap_manifest ⇒ Object
Returns a vcap manifest that can be used to create a new data-service to vcap’s cloud_controller.
- #vendor ⇒ Object
Constructor Details
#initialize(root, data, role_name) ⇒ DataService
Returns a new instance of DataService.
97 98 99 100 101 |
# File 'lib/vmc_knife/vmc_knife.rb', line 97 def initialize(root, data, role_name) @root = root @wrapped = data @role_name = role_name end |
Instance Attribute Details
#role_name ⇒ Object
Returns the value of attribute role_name.
96 97 98 |
# File 'lib/vmc_knife/vmc_knife.rb', line 96 def role_name @role_name end |
#root ⇒ Object
Returns the value of attribute root.
96 97 98 |
# File 'lib/vmc_knife/vmc_knife.rb', line 96 def root @root end |
#wrapped ⇒ Object
Returns the value of attribute wrapped.
96 97 98 |
# File 'lib/vmc_knife/vmc_knife.rb', line 96 def wrapped @wrapped end |
Instance Method Details
#apply_privileges(app_name = nil) ⇒ Object
Make sure that all users who can connect to the DB can also access the tables. This workarounds the privilege issue reported .… and was added to “my” branch of vcap’s services
Another workaround though really not perfect so it will stay in vmc_knife: the ownership of the SQL functions.
453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 |
# File 'lib/vmc_knife/data_services.rb', line 453 def apply_privileges(app_name=nil) return if credentials(app_name).nil? if is_postgresql() cmd_acl="GRANT CREATE ON SCHEMA PUBLIC TO PUBLIC;\ GRANT ALL ON ALL TABLES IN SCHEMA PUBLIC TO PUBLIC;\ GRANT ALL ON ALL FUNCTIONS IN SCHEMA PUBLIC TO PUBLIC;\ GRANT ALL ON ALL SEQUENCES IN SCHEMA PUBLIC TO PUBLIC;\ ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO PUBLIC;\ ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO PUBLIC;\ ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON FUNCTIONS TO PUBLIC;" # shell(cmd_acl,true) # reset the owner of the functions to the current user # when there is a 'privileged' app. app_name ||= @wrapped['director']['bound_app'] if @wrapped['director'] if app_name cmd_select_fcts="SELECT pg_proc.proname FROM pg_catalog.pg_proc WHERE \ pg_proc.pronamespace=(SELECT pg_namespace.oid FROM pg_catalog.pg_namespace WHERE pg_namespace.nspname = 'public') \ AND pg_proc.proowner!=(SELECT oid FROM pg_roles WHERE rolname = 'postgres')" current_owner=credentials()['username'] unless current_owner STDERR.puts "The application #{app_name} is not bound to the data-service #{name}; not applying the database privileges." return end fcts_name=shell(cmd_select_fcts,true,true) if fcts_name.empty? puts "No need to re-assign the ownership of the functions in #{credentials()['name']}; the DB does not define its own functions." if VMC::Cli::Config.trace else fcts = fcts_name.split("\n").collect do |line| line.strip! "'#{line}'" end.join(',') cmd_change_fcts_owner="UPDATE pg_catalog.pg_proc \ SET proowner = (SELECT oid FROM pg_roles WHERE rolname = '#{current_owner}')\ WHERE pg_proc.proname IN (#{fcts})" puts `sudo -u postgres psql --dbname #{credentials()['name']} -c \"#{cmd_change_fcts_owner}\" #{PSQL_RAW_RES_ARGS}` end end end end |
#credentials(app_name = nil) ⇒ Object
The credentials hash for this data-service param app_name The favored app usually bound to this data_service
239 240 241 242 243 244 245 246 247 |
# File 'lib/vmc_knife/data_services.rb', line 239 def credentials(app_name=nil) #bound_app is the name of an app bound to the dat-service and that credentials #should be used to access the data-service. #for example if an app creates large objects you will need to use #the credentials of that app to find the objects. app_name ||= @wrapped['director']['bound_app'] if @wrapped['director'] @credentials ||= VMC::KNIFE.get_credentials(name(), app_name) @credentials end |
#drop(collection_or_table_names = nil) ⇒ Object
520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 |
# File 'lib/vmc_knife/data_services.rb', line 520 def drop(collection_or_table_names=nil) if is_postgresql sel_tables = "SELECT table_name FROM information_schema.tables WHERE table_schema='public'" if collection_or_table_names sel_tables = "#{sel_tables} AND table_name LIKE '#{collection_or_table_names}'" end tables = shell(sel_tables, false, true) tables_arr = Array.new tables.split("\n").each do | line | line.strip! tables_arr << line if line end if tables_arr.size > 0 && ask("Delete the tables \"#{tables_arr.join(',')}\"?", :default => true) #let's create a file in case there are a lot of tables: file = Tempfile.new('droptables') begin File.open(file.path, 'w') do |f2| tables_arr.each do |table| f2.puts "DROP TABLE public.#{table} CASCADE;" end end puts shell(file.path) ensure file.unlink # deletes the temp file end end puts "Vacuum orphaned large objects..." cmd = VMC::KNIFE.pg_connect_cmd(credentials(), 'vacuumlo') puts cmd puts `#{cmd}` elsif is_mongodb # generate the command file from the erb template. filter = ".*" filterIsNegated = "false"; skipSystem = "true"; if collection_or_table_names if collection_or_table_names.start_with?('!') filterIsNegated = "true"; filter = collection_or_table_names[1..-1] else filter = collection_or_table_names end end # this command is applied to each collection. # the name of the variable is 'collection' as can bee seen in the erb file. cmd="collection.drop();" file = Tempfile.new('dropcollections') begin File.open(file.path, 'w') do |f2| template = ERB.new File.new("#{VMCKNIFE::ROOT_REL}/vmc_knife/mongo/mongo_cmd.js.erb").read, nil, "%" f2.puts template.result(binding) end puts shell(file.path) ensure file.unlink # deletes the temp file end #TODO: iterate over the collections and drop them according to the filter. #raise "TODO: Unsupported operation 'drop' for the data-service #{name()}" else puts "Unsupported operation 'drop' for the data-service #{name()}" end end |
#export(app_name = nil, file = nil) ⇒ Object
366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 |
# File 'lib/vmc_knife/data_services.rb', line 366 def export(app_name=nil,file=nil) if is_postgresql if file.nil? extension = @wrapped['director']['file_extension'] if @wrapped['director'] file = "#{name()}.#{extension}" else unless File.exists?(File.dirname(file)) raise "The output folder #{File.dirname(file)} does not exist." end end extension ||= "dump" archive_unzipped = "#{name()}.#{extension}" #archive_unzipped="#{name()}.sql" unless /\.sql$/ =~ extension `touch #{archive_unzipped}` unless File.exists? archive_unzipped raise "Unable to create the file #{archive_unzipped}" end `chmod o+w #{archive_unzipped}` puts "Exports the database #{credentials(app_name)['name']} in #{file}" #sudo -u postgres env PGPASSWORD=intalio DBNAME=intalio DUMPFILE=intalio_dump.sql pg_dump --format=p --file=$DUMPFILE --no-owner --clean --blobs --no-acl --oid --no-tablespaces $DBNAME #sudo -u postgres env PGPASSWORD=$PGPASSWORD DUMPFILE=$DUMPFILE pg_dump --format=p --file=$DUMPFILE --no-owner --clean --blobs --no-acl --oid --no-tablespaces $DBNAME cmd = VMC::KNIFE.pg_connect_cmd(credentials(app_name), 'pg_dump', false, "--format=c --file=#{archive_unzipped} --no-owner --clean --oids --blobs --no-acl --no-privileges --no-tablespaces") puts cmd puts `#{cmd}` unless File.exists? archive_unzipped raise "Unable to read the file #{archive_unzipped}" end `chmod o-w #{archive_unzipped}` elsif is_mongodb if file.nil? extension = @wrapped['director']['file_extension'] if @wrapped['director'] extension ||= "bson.tar.gz" file = "#{name()}.#{extension}" end creds=credentials(app_name) puts "Exports the database #{creds['name']} in #{file}" #mongodump --host localhost:27017 mongodump_exec=VMC::KNIFE.get_mongo_exec('mongodump') # see if we go through the filesystem or through the network: base_dir=VMC::KNIFE.get_mongodb_base_dir() instance_name=creds['name'] dbpath=File.join(base_dir, instance_name, 'data') mongod_lock=File.join(dbpath,'mongod.lock') puts "looking at #{mongod_lock} exists? #{File.exists?(mongod_lock)} size #{File.size(mongod_lock)}" if File.exists?(mongod_lock) && File.size(mongod_lock)>0 cmd = "#{mongodump_exec} -u #{creds['username']} -p #{creds['password']} --host #{creds['hostname']}:#{creds['port']} --db db" else cmd = "#{mongodump_exec} --dbpath #{dbpath}" end puts cmd puts `#{cmd}` archive_unzipped="dump" end # this produces a dump folder in the working directory. # let's zip it: if /\.zip$/ =~ file # just zip `zip -r #{file} #{archive_unzipped}` elsif /\.tar$/ =~ file # just tar `tar -cvf #{file} #{archive_unzipped}` else # tar-gzip by default `tar -czvf #{file} #{archive_unzipped}` end `rm -rf #{archive_unzipped}` if archive_unzipped != file end |
#import(app_name = nil, file = nil) ⇒ Object
254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 |
# File 'lib/vmc_knife/data_services.rb', line 254 def import(app_name=nil,file=nil) file ||= @wrapped['director']['import_url'] if @wrapped['director'] if file.nil? files = Dir.glob("#{name()}.*") raise "Unable to locate the database file to import." if files.empty? file = files.first end tmp_download_filename="_download_.zip" data_download_dir="#{ENV['HOME']}/vmc_knife_downloads/data_#{@wrapped['name']}" current_wd=Dir.pwd FileUtils.mkdir_p data_download_dir Dir.chdir(data_download_dir) do if file =~ /^https?:\/\// || file =~ /^ftp:\/\// basename = Pathname.new(URI.parse(file).path).basename.to_s else file=File.(file,current_wd) basename = File.basename(file).to_s end unless File.exists?(basename) Dir.foreach(Dir.pwd) { |f| File.delete(f) unless f == '.' || f == '..' } if file =~ /^https?:\/\// || file =~ /^ftp:\/\// wget_args = @wrapped['director']['wget_args'] if wget_args.nil? wget_args_str = "" elsif wget_args.kind_of? Array wget_args_str = wget_args.join(' ') elsif wget_args.kind_of? String wget_args_str = wget_args end `wget #{wget_args_str} --output-document=#{basename} #{file}` if $? != 0 File.delete("#{data_download_dir}/#{basename}") raise "Unable to successfully download #{file}" end else FileUtils.cp(file, basename) end end #unzip if necessary (in progress) is_unzipped=true p "unzip #{basename}" if /\.tgz$/ =~ basename || /\.tar\.gz$/ =~ basename `tar zxvf #{basename}` elsif /\.tar$/ =~ basename `tar xvf #{basename}` elsif /\.zip$/ =~ basename `unzip #{basename}` else is_unzipped=false end if is_unzipped #`rm #{basename}` files = Dir.glob(["*.sql", "*.dump"]) if is_postgresql files = Dir.glob("**/*.bson") if is_mongodb files ||= Dir.glob("*") raise "Can't find the db-dump file." if files.empty? file = files.first else file = basename end creds=credentials(app_name) if is_postgresql p "chmod o+w #{file}" `chmod o+w #{file}` if /\.sql$/ =~ file other_params="--file #{file} --quiet" cmd = VMC::KNIFE.pg_connect_cmd(creds, 'psql',as_admin=false, other_params) #`psql --dbname #{dbname} --file #{file} --clean --quiet --username #{rolename}` else other_params="--clean --no-acl --no-privileges --no-owner #{file}" cmd = VMC::KNIFE.pg_connect_cmd(creds, 'pg_restore',false, other_params) #`pg_restore --dbname=#{dbname} --username=#{username} --no-acl --no-privileges --no-owner #{file}` end puts cmd puts `#{cmd}`.strip `chmod o-w #{file}` elsif is_mongodb # see if we go through the filesystem to shrink or # if we are only interested in the data itself. base_dir=VMC::KNIFE.get_mongodb_base_dir() instance_name=creds['name'] dbpath=File.join(base_dir, instance_name, 'data') mongod_lock=File.join(dbpath,'mongod.lock') if File.exists?(mongod_lock) && File.size(mongod_lock)>0 # the mongodb instance is currently working. connect to it and do the work. # in that case import the 'db' alone. don't do the 'admin' VMC::KNIFE.data_service_console(creds, File.dirname(file), false, 'mongorestore') else # the mongodb instance is not currently working # go directly on the filesystem `rm -rf #{dbpath}` `mkdir -p #{dbpath}` #sudo mongorestore --dbpath /var/lib/mongodb mongorestore_exec=VMC::KNIFE.get_mongo_exec('mongorestore') `#{mongorestore_exec} --dbpath #{dbpath} #{File.dirname(File.dirname(file))}` end else raise "Unsupported type of data-service. Postgresql and mongodb are the only supported services at the moment." end end end |
#is_mongodb ⇒ Object
442 443 444 |
# File 'lib/vmc_knife/data_services.rb', line 442 def is_mongodb() credentials()['db'] != nil end |
#is_postgresql ⇒ Object
438 439 440 |
# File 'lib/vmc_knife/data_services.rb', line 438 def is_postgresql() credentials()['db'] == nil end |
#log(target_folder) ⇒ Object
117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 |
# File 'lib/vmc_knife/vmc_knife.rb', line 117 def log(target_folder) if is_postgresql() postgres_log_folder = "/var/log/postgresql" pg_target_dir = "#{target_folder}/postgresql" FileUtils.mkdir(pg_target_dir) Dir.glob(File.join(postgres_log_folder, "*")).each do |f| fname="#{File.basename(f)}" FileUtils.cp(File.join(postgres_log_folder, fname), pg_target_dir) end end if is_mongodb() mongo_log_folder = "/var/vcap/services/mongodb/logs" mg_target_dir = "#{target_folder}/mongodb" FileUtils.mkdir(mg_target_dir) Dir.glob(File.join(mongo_log_folder, "*")).each do |f| fname="#{File.basename(f)}" FileUtils.cp_r(File.join(mongo_log_folder, fname), mg_target_dir) end end end |
#name ⇒ Object
returns the name of the service for cloudfoundry
103 104 105 |
# File 'lib/vmc_knife/vmc_knife.rb', line 103 def name() @wrapped['name'] end |
#shell(commands_file = nil, as_admin = false, return_value = false) ⇒ Object
Connect to the mongo js shell or the psql shell.
250 251 252 |
# File 'lib/vmc_knife/data_services.rb', line 250 def shell(commands_file=nil,as_admin=false,return_value=false) VMC::KNIFE.data_service_console(credentials(),commands_file,as_admin,nil,return_value) end |
#shrink(collection_or_table_names = nil) ⇒ Object
shrink the size of the databses on the file system. Specifically act on the mongodb instances when they are stopped.
496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 |
# File 'lib/vmc_knife/data_services.rb', line 496 def shrink(collection_or_table_names=nil) return unless is_mongodb creds=credentials() base_dir=VMC::KNIFE.get_mongodb_base_dir() instance_name=creds['name'] dbpath=File.join(base_dir, instance_name, 'data') mongod_lock=File.join(dbpath,'mongod.lock') raise "Can't shrink #{name}; the mongodb is currently running" if File.exists?(mongod_lock) && File.size(mongod_lock)>0 mongodump_exec=VMC::KNIFE.get_mongo_exec('mongodump') raise "Can't find mongodump" unless File.exist? mongodump_exec mongorestore_exec=VMC::KNIFE.get_mongo_exec('mongorestore') raise "Can't find mongorestore" unless File.exist? mongorestore_exec cmd = "#{mongodump_exec} --dbpath #{dbpath}" puts "#{cmd}" puts `#{cmd}` `rm -rf #{dbpath}` `mkdir #{dbpath}` cmd = "#{mongorestore_exec} --dbpath #{dbpath} dump/" puts "#{cmd}" puts `#{cmd}` `rm -rf dump` end |
#to_vcap_manifest ⇒ Object
Returns a vcap manifest that can be used to create a new data-service to vcap’s cloud_controller.
112 113 114 115 |
# File 'lib/vmc_knife/vmc_knife.rb', line 112 def to_vcap_manifest() #TODO @wrapped end |
#vendor ⇒ Object
106 107 108 |
# File 'lib/vmc_knife/vmc_knife.rb', line 106 def vendor() @wrapped['vendor'] end |