Module: CommonFunctions

Defined in:
lib/common_functions.rb

Class Method Summary collapse

Class Method Details

.clear_app(app_path, force = false) ⇒ Object 



191
192
193
194
195
196
 
# File 'lib/common_functions.rb', line 191

def self.clear_app(app_path, force=false)
  return if !File.exists?(app_path)
  return if app_path !~ /\A\/tmp/ and !force
  path_to_remove = File.dirname(app_path)
  FileUtils.rm_f(path_to_remove)
end

.convert_fqdn_to_ip(host) ⇒ Object 



380
381
382
383
384
385
386
387
388
 
# File 'lib/common_functions.rb', line 380

def self.convert_fqdn_to_ip(host)
  nslookup = CommonFunctions.shell("nslookup #{host}")
  ip = nslookup.scan(/#{host}\nAddress:\s+(#{IP_REGEX})/).flatten.to_s
  if ip.nil? or ip.empty?
    raise AppScaleException.new("Couldn't convert #{host} to an IP " +
      "address. Result of nslookup was \n#{nslookup}")
  end
  return ip
end

.create_user(user, test, head_node_ip, secret_key, uac, pass = nil) ⇒ Object 



248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
 
# File 'lib/common_functions.rb', line 248

def self.create_user(user, test, head_node_ip, secret_key, uac, pass=nil)
  if pass
    pass = pass # TODO - can we just remove this?
  elsif test
    pass = "aaaaaa"
  else
    pass = CommonFunctions.get_password
  end

  encrypted_pass = CommonFunctions.encrypt_password(user, pass)
  uac.commit_new_user(user, encrypted_pass)

   = CommonFunctions.(head_node_ip, secret_key)
 
  # create xmpp account
  # for user [email protected], this translates to a@login_ip

  pre = user.scan(/\A(.*)@/).flatten.to_s
  xmpp_user = "#{pre}@#{}"
  xmpp_pass = CommonFunctions.encrypt_password(xmpp_user, pass)
  uac.commit_new_user(xmpp_user, xmpp_pass)
  Kernel.puts "Your XMPP username is #{xmpp_user}"
end

.encrypt_password(user, pass) ⇒ Object 



391
392
393
 
# File 'lib/common_functions.rb', line 391

def self.encrypt_password(user, pass)
  Digest::SHA1.hexdigest(user + pass)
end

.find_real_ssh_key(ssh_keys, host) ⇒ Object

Raises:



561
562
563
564
565
566
567
568
569
570
571
 
# File 'lib/common_functions.rb', line 561

def self.find_real_ssh_key(ssh_keys, host)
  ssh_keys.each { |key|
    key = File.expand_path(key)
    return_value = CommonFunctions.shell("ssh -i #{key} #{SSH_OPTIONS} 2>&1 root@#{host} 'touch /tmp/foo'; echo $? ").chomp
    if return_value == "0"
      return key
    end
  }
  
  raise AppScaleException.new(NO_SSH_KEY_FOUND)
end

.get_all_public_ips(keyname, required = true) ⇒ Object 



755
756
757
758
759
760
761
762
 
# File 'lib/common_functions.rb', line 755

def self.get_all_public_ips(keyname, required=true)
  ips = []
  nodes = self.read_nodes_json(keyname)
  nodes.each { |node|
    ips << node['public_ip']
  }
  return ips
end

.get_app_info(fullpath, app_file) ⇒ Object 



911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
 
# File 'lib/common_functions.rb', line 911

def self.get_app_info(fullpath, app_file)
  if !File.exists?(fullpath)
    raise AppEngineConfigException.new("AppEngine file not found")
  end
  filename = fullpath.scan(/\/?([\w\.]+\Z)/).flatten.to_s

  temp_dir = CommonFunctions.get_random_alphanumeric
  FileUtils.rm_rf("/tmp/#{temp_dir}", :secure => true)
  FileUtils.mkdir_p("/tmp/#{temp_dir}")

  CommonFunctions.move_app(temp_dir, filename, app_file, fullpath)
  app_yaml_loc = app_file
  if !File.exists?("/tmp/#{temp_dir}/#{app_file}")
    FileUtils.rm_rf("/tmp/#{temp_dir}", :secure => true)
    return nil, nil, nil
  end

  if app_file == PYTHON_CONFIG
    app_name = CommonFunctions.get_app_name_via_yaml(temp_dir, app_yaml_loc)
    language = "python"
    if File.directory?(fullpath)
      temp_dir2 = CommonFunctions.get_random_alphanumeric
      FileUtils.rm_rf("/tmp/#{temp_dir2}", :secure => true)
      FileUtils.mkdir_p("/tmp/#{temp_dir2}")
      CommonFunctions.shell("cd /tmp/#{temp_dir}; tar -czf ../#{temp_dir2}/#{app_name}.tar.gz .")
      file = "/tmp/#{temp_dir2}/#{app_name}.tar.gz"
    else
      file = fullpath
    end
  elsif app_file == JAVA_CONFIG
    app_name = CommonFunctions.get_app_name_via_xml(temp_dir, app_yaml_loc)
    language = "java"
    # don't remove user's jar files, they may have their own jars in it
    #FileUtils.rm_rf("/tmp/#{temp_dir}/war/WEB-INF/lib/", :secure => true)
    FileUtils.mkdir_p("/tmp/#{temp_dir}/war/WEB-INF/lib")
    temp_dir2 = CommonFunctions.get_random_alphanumeric
    FileUtils.rm_rf("/tmp/#{temp_dir2}", :secure => true)
    FileUtils.mkdir_p("/tmp/#{temp_dir2}")
    FileUtils.rm_f("/tmp/#{temp_dir}/#{filename}")
    CommonFunctions.shell("cd /tmp/#{temp_dir}; tar -czf ../#{temp_dir2}/#{app_name}.tar.gz .")
    file = "/tmp/#{temp_dir2}/#{app_name}.tar.gz"
  else
    FileUtils.rm_rf("/tmp/#{temp_dir}", :secure => true)
    raise AppEngineConfigException.new("app_name was #{app_file}, " +
      "which was not a recognized value.")
  end

  if app_name.nil?
    FileUtils.rm_rf("/tmp/#{temp_dir}", :secure => true)
    raise AppEngineConfigException.new("AppEngine tar file is invalid - " +
      "Doesn't have an app name in #{app_file}")
  end

  FileUtils.rm_rf("/tmp/#{temp_dir}", :secure => true)
  CommonFunctions.warn_on_large_app_size(file)
  return app_name, file, language 
end

.get_app_info_from_options(options) ⇒ Object 



856
857
858
859
860
861
862
863
864
865
866
867
868
 
# File 'lib/common_functions.rb', line 856

def self.get_app_info_from_options(options)
  file_location = options['file_location']
  table = options['table']

  if file_location.nil?
    apps_to_start = ["none"]
    return apps_to_start, {}
  else
    app_info = CommonFunctions.get_app_name_from_tar(file_location)
    apps_to_start = [CommonFunctions.validate_app_name(app_info[:app_name], table)]
    return apps_to_start, app_info
  end
end

.get_app_name_from_tar(fullpath) ⇒ Object 



871
872
873
874
875
876
877
878
879
880
881
882
883
 
# File 'lib/common_functions.rb', line 871

def self.get_app_name_from_tar(fullpath)
  app_name, file, language = CommonFunctions.get_app_info(fullpath, PYTHON_CONFIG)

  if app_name.nil? or file.nil? or language.nil?
    app_name, file, language = CommonFunctions.get_app_info(fullpath, JAVA_CONFIG)
  end

  if app_name.nil? or file.nil? or language.nil?
    raise AppScaleException.new(NO_CONFIG_FILE)
  end

  return {:app_name => app_name, :file => file, :language => language}
end

.get_app_name_via_xml(temp_dir, xml_loc) ⇒ Object 



984
985
986
987
988
989
990
 
# File 'lib/common_functions.rb', line 984

def self.get_app_name_via_xml(temp_dir, xml_loc)
  xml_loc = "/tmp/" + temp_dir + "/" + xml_loc
  web_xml_contents = (File.open(xml_loc) { |f| f.read }).chomp
  app_name = web_xml_contents.scan(/<application>([\w\d-]+)<\/application>/).flatten.to_s
  app_name = nil if app_name == ""
  return app_name
end

.get_app_name_via_yaml(temp_dir, app_yaml_loc) ⇒ Object 



970
971
972
973
974
975
976
977
978
979
980
981
 
# File 'lib/common_functions.rb', line 970

def self.get_app_name_via_yaml(temp_dir, app_yaml_loc)
  app_yaml_loc = "/tmp/" + temp_dir + "/" + app_yaml_loc
  
  begin
    tree = YAML.load_file(app_yaml_loc.chomp)
  rescue ArgumentError
    raise AppScaleException.new(MALFORMED_YAML)
  end
  
  app_name = String(tree["application"])
  return app_name
end

.get_credentials(testing) ⇒ Object 



239
240
241
242
243
244
245
 
# File 'lib/common_functions.rb', line 239

def self.get_credentials(testing)
  if testing
    return DEFAULT_USERNAME, DEFAULT_PASSWORD
  else
    return CommonFunctions.get_email, CommonFunctions.get_password
  end
end

.get_db_master_ip(keyname, required = true) ⇒ Object 



765
766
767
 
# File 'lib/common_functions.rb', line 765

def self.get_db_master_ip(keyname, required=true)
  return self.get_role_from_nodes(keyname, 'db_master')
end

.get_email ⇒ Object 



638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
 
# File 'lib/common_functions.rb', line 638

def self.get_email
  email = nil
  Kernel.puts "\nThis AppScale instance is linked to an e-mail address giving it administrator privileges."
  
  loop {
    Kernel.print "Enter your desired administrator e-mail address: "
    STDOUT.flush
    email = STDIN.gets.chomp
    
    if email =~ EMAIL_REGEX
      break
    else
      Kernel.puts "The response you typed in was not an e-mail address. Please try again.\n\n"
    end
  }
  
  return email
end

.get_from_yaml(keyname, tag, required = true) ⇒ Object 



696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
 
# File 'lib/common_functions.rb', line 696

def self.get_from_yaml(keyname, tag, required=true)
  location_file = File.expand_path("~/.appscale/locations-#{keyname}.yaml")

  if !File.exists?(location_file)
    raise AppScaleException.new("An AppScale instance is not currently " +
      "running with the provided keyname, \"#{keyname}\".")
  end
  
  begin
    tree = YAML.load_file(location_file)
  rescue ArgumentError
    if required
      raise AppScaleException.new(MALFORMED_YAML)
    else
      return nil
    end
  end
  
  if !tree
    raise AppScaleException.new("The location file is in the wrong format.")
  end

  value = tree[tag]

  if value.nil? and required
    raise AppScaleException.new("The location file did not contain a #{tag} tag.")
  end

  return value
end

.get_head_node_ip(keyname, required = true) ⇒ Object 



770
771
772
 
# File 'lib/common_functions.rb', line 770

def self.get_head_node_ip(keyname, required=true)
  return self.get_role_from_nodes(keyname, 'shadow')
end

.get_infrastructure(keyname, required = true) ⇒ Object 



780
781
782
 
# File 'lib/common_functions.rb', line 780

def self.get_infrastructure(keyname, required=true)
  CommonFunctions.get_from_yaml(keyname, :infrastructure)
end

.get_ips_from_yaml(ips) ⇒ Object 



220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
 
# File 'lib/common_functions.rb', line 220

def self.get_ips_from_yaml(ips)
  return "using_tools" if ips.nil?

  ips_to_use = []
  if !ips[:servers].nil?
    ips[:servers].each { |ip|
      if ip =~ IP_REGEX
        ips_to_use << ip
      else
        ips_to_use << CommonFunctions.convert_fqdn_to_ip(ip)
      end
    }
    ips_to_use = ips_to_use.join(":")
  end
  
  return ips_to_use
end

.get_line_from_stdin_no_echo ⇒ Object 



686
687
688
689
690
691
692
693
 
# File 'lib/common_functions.rb', line 686

def self.get_line_from_stdin_no_echo
  state = CommonFunctions.shell("stty -g")
  system "stty -echo" # Turn off character echoing
  STDOUT.flush
  line = STDIN.gets.chomp
  system "stty #{state}"
  return line
end

.get_load_balancer_id(keyname, required = true) ⇒ Object 



745
746
747
 
# File 'lib/common_functions.rb', line 745

def self.get_load_balancer_id(keyname, required=true)
  return CommonFunctions.get_from_yaml(keyname, :instance_id)  
end

.get_load_balancer_ip(keyname, required = true) ⇒ Object 



740
741
742
 
# File 'lib/common_functions.rb', line 740

def self.get_load_balancer_ip(keyname, required=true)
  return self.get_role_from_nodes(keyname, 'load_balancer')
end

.get_login_ip(head_node_ip, secret_key) ⇒ Object

Raises:



177
178
179
180
181
182
183
184
185
186
187
188
 
# File 'lib/common_functions.rb', line 177

def self.(head_node_ip, secret_key)
  acc = AppControllerClient.new(head_node_ip, secret_key)
  all_nodes = acc.get_all_public_ips()
  
  all_nodes.each { |node|
    acc_new = AppControllerClient.new(node, secret_key)
    roles = acc_new.status()
    return node if roles.match(/Is currently:(.*)login/)
  }

  raise AppScaleException.new("Unable to find login ip address!")
end

.get_password ⇒ Object 



658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
 
# File 'lib/common_functions.rb', line 658

def self.get_password()
  pass = nil
  Kernel.puts "\nThe new administrator password must be at least six characters long and can include non-alphanumeric characters."
  
  loop {
    Kernel.print "Enter your new password: "
    new_pass = self.get_line_from_stdin_no_echo()
    Kernel.print "\nEnter again to verify: "
    verify_pass = self.get_line_from_stdin_no_echo()
    Kernel.print "\n"
    
    if new_pass == verify_pass
      pass = new_pass
      
      if pass =~ PASSWORD_REGEX
        break
      else
        Kernel.puts "\n\nThe password you typed in was not at least six characters long. Please try again.\n\n"
      end
    else
      Kernel.puts "\n\nPasswords entered do not match. Please try again.\n\n"
    end
  }
  
  return pass
end

.get_random_alphanumeric(length = 10) ⇒ Object 



843
844
845
846
847
848
849
850
851
852
853
 
# File 'lib/common_functions.rb', line 843

def self.get_random_alphanumeric(length=10)
  random = ""
  possible = "0123456789abcdefghijklmnopqrstuvxwyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
  possibleLength = possible.length

  length.times { |index|
    random << possible[rand(possibleLength)]
  }

  return random
end

.get_remote_appscale_home(ip, key) ⇒ Object 



462
463
464
465
466
467
468
469
470
471
 
# File 'lib/common_functions.rb', line 462

def self.get_remote_appscale_home(ip, key)
  cat = "cat /etc/appscale/home"
  remote_cmd = "ssh -i #{key} #{SSH_OPTIONS} 2>&1 root@#{ip} '#{cat}'"
  possible_home = CommonFunctions.shell("#{remote_cmd}").chomp
  if possible_home.nil? or possible_home.empty?
    return "/root/appscale/"
  else
    return possible_home
  end
end

.get_role_from_nodes(keyname, role) ⇒ Object 



728
729
730
731
732
733
734
735
736
737
 
# File 'lib/common_functions.rb', line 728

def self.get_role_from_nodes(keyname, role)
  nodes = self.read_nodes_json(keyname)
  nodes.each { |node|
    if node['jobs'].include?(role)
      return node['public_ip']
    end
  }

  return ""
end

.get_secret_key(keyname, required = true) ⇒ Object 



775
776
777
 
# File 'lib/common_functions.rb', line 775

def self.get_secret_key(keyname, required=true)
  CommonFunctions.get_from_yaml(keyname, :secret)
end

.get_table(keyname, required = true) ⇒ Object 



750
751
752
 
# File 'lib/common_functions.rb', line 750

def self.get_table(keyname, required=true)
  return CommonFunctions.get_from_yaml(keyname, :table, required)
end

.get_username_from_options(options) ⇒ Object 



627
628
629
630
631
632
633
634
635
 
# File 'lib/common_functions.rb', line 627

def self.get_username_from_options(options)
  if options['test']
    return DEFAULT_USERNAME
  elsif options['email']
    return options['email']
  else
    return CommonFunctions.get_email
  end
end

.is_port_open?(ip, port, use_ssl = false) ⇒ Boolean

Returns:

  • (Boolean) 


412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
 
# File 'lib/common_functions.rb', line 412

def self.is_port_open?(ip, port, use_ssl=false)
  begin
    Timeout::timeout(1) do
      begin
        sock = TCPSocket.new(ip, port)
        if use_ssl
          ssl_context = OpenSSL::SSL::SSLContext.new() 
          unless ssl_context.verify_mode 
            ssl_context.verify_mode = OpenSSL::SSL::VERIFY_NONE 
          end 
          sslsocket = OpenSSL::SSL::SSLSocket.new(sock, ssl_context) 
          sslsocket.sync_close = true 
          sslsocket.connect          
        end
        sock.close
        return true
      rescue Exception
        return false
      end
    end
  rescue Timeout::Error
  end

  return false
end

.make_appscale_directory ⇒ Object 



785
786
787
788
789
790
791
792
793
 
# File 'lib/common_functions.rb', line 785

def self.make_appscale_directory()
  # AppScale generates private keys for each cloud that machines are 
  # running in. It stores this data (and a YAML file detailing IP address 
  # mappings) in ~/.appscale - create this location if it doesn't exist.
  appscale_path = File.expand_path("~/.appscale")
  if !File.exists?(appscale_path)
    FileUtils.mkdir(appscale_path)
  end
end

.move_app(temp_dir, filename, app_file, fullpath) ⇒ Object 



886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
 
# File 'lib/common_functions.rb', line 886

def self.move_app(temp_dir, filename, app_file, fullpath)
  if File.directory?(fullpath)
    CommonFunctions.shell("cp -r #{fullpath}/* /tmp/#{temp_dir}/")
    return
  else
    FileUtils.cp(fullpath, "/tmp/#{temp_dir}/#{filename}")
    FileUtils.rm_f("/tmp/#{temp_dir}/#{app_file}")
    tar_file = CommonFunctions.shell("cd /tmp/#{temp_dir}; tar zxvfm #{filename} 2>&1; echo $?").chomp
    tar_ret_val = tar_file.scan(/\d+\Z/).to_s
    if tar_ret_val != "0"
      raise AppScaleException.new("Untar'ing the given tar file in /tmp failed")
    end
  end
  return
end

.read_file(location, chomp = true) ⇒ Object

A convenience function that returns a file’s contents as a string.



91
92
93
94
95
96
97
98
 
# File 'lib/common_functions.rb', line 91

def self.read_file(location, chomp=true)
  file = File.open(location) { |f| f.read }
  if chomp
    return file.chomp
  else
    return file
  end
end

.read_nodes_json(keyname) ⇒ Object

Reads the JSON file that stores information about which roles run on which nodes.



798
799
800
801
 
# File 'lib/common_functions.rb', line 798

def self.read_nodes_json(keyname)
  filename = File.expand_path("~/.appscale/locations-#{keyname}.json")
  return JSON.load(self.read_file(filename))
end

.require_commands(commands) ⇒ Object 



369
370
371
372
373
374
375
376
377
 
# File 'lib/common_functions.rb', line 369

def self.require_commands(commands)
  commands.each { |cmd|
    if !CommonFunctions.user_has_cmd?(cmd)
      raise BadConfigurationException.new("You do not have the '#{cmd}' " +
        "command in your PATH. Please ensure that it is in your path and " +
        "try again.")
    end
  }
end

.rsync_files(dest_ip, ssh_key, local) ⇒ Object

Uses rsync to copy over a copy of the AppScale main codebase (e.g., not the AppScale Tools) from this machine to a remote machine. TODO(cgb): This function doesn’t copy files in the main directory, like the firewall rules. It should be changed accordingly.



113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
 
# File 'lib/common_functions.rb', line 113

def self.rsync_files(dest_ip, ssh_key, local)
  local = File.expand_path(local)
  controller = "#{local}/AppController"
  server = "#{local}/AppServer"
  loadbalancer = "#{local}/AppLoadBalancer"
  monitoring = "#{local}/AppMonitoring"
  appdb = "#{local}/AppDB"
  neptune = "#{local}/Neptune"
  loki = "#{local}/Loki"
  iaas_manager = "#{local}/InfrastructureManager"

  if !File.exists?(controller)
    raise BadConfigurationException.new("The location you " +
      "specified to rsync from, #{local}, doesn't exist or contain " +
      "AppScale data.")
  end

  self.shell("rsync -e 'ssh -i #{ssh_key}' -arv #{controller}/* root@#{dest_ip}:/root/appscale/AppController")
  self.shell("rsync -e 'ssh -i #{ssh_key}' -arv #{server}/* root@#{dest_ip}:/root/appscale/AppServer")
  self.shell("rsync -e 'ssh -i #{ssh_key}' -arv #{loadbalancer}/* root@#{dest_ip}:/root/appscale/AppLoadBalancer")
  self.shell("rsync -e 'ssh -i #{ssh_key}' -arv #{monitoring}/* root@#{dest_ip}:/root/appscale/AppMonitoring")
  self.shell("rsync -e 'ssh -i #{ssh_key}' -arv --exclude='logs/*' --exclude='hadoop-*' --exclude='hbase/hbase-*' --exclude='voldemort/voldemort/*' --exclude='cassandra/cassandra/*' #{appdb}/* root@#{dest_ip}:/root/appscale/AppDB")
  self.shell("rsync -e 'ssh -i #{ssh_key}' -arv #{neptune}/* root@#{dest_ip}:/root/appscale/Neptune")
  #self.shell("rsync -e 'ssh -i #{ssh_key}' -arv #{loki}/* root@#{dest_ip}:/root/appscale/Loki")
  self.shell("rsync -e 'ssh -i #{ssh_key}' -arv #{iaas_manager}/* root@#{dest_ip}:/root/appscale/InfrastructureManager")
end

.run_remote_command(ip, command, public_key_loc, want_output) ⇒ Object 



439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
 
# File 'lib/common_functions.rb', line 439

def self.run_remote_command(ip, command, public_key_loc, want_output)
  if public_key_loc.class == Array
    public_key_loc.each { |key|
      key = File.expand_path(key)
    }

    remote_cmd = "ssh -i #{public_key_loc.join(' -i ')} #{SSH_OPTIONS} 2>&1 root@#{ip} '#{command}"
  else
    public_key_loc = File.expand_path(public_key_loc)
    remote_cmd = "ssh -i #{public_key_loc} #{SSH_OPTIONS} root@#{ip} '#{command} "
  end
  
  if want_output
    remote_cmd << "> /tmp/#{ip}.log 2>&1 &' &"
  else
    remote_cmd << "> /dev/null 2>&1 &' &"
  end

  Kernel.system remote_cmd
  return remote_cmd
end

.scp_app_to_ip(app_name, user, language, keyname, head_node_ip, file_location, uac) ⇒ Object 



273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
 
# File 'lib/common_functions.rb', line 273

def self.scp_app_to_ip(app_name, user, language, keyname, head_node_ip,
  file_location, uac)

  Kernel.puts "Uploading #{app_name}..."
  uac.commit_new_app_name(user, app_name, language)

  local_file_path = File.expand_path(file_location)

  app_dir = "/var/apps/#{app_name}/app"
  remote_file_path = "#{app_dir}/#{app_name}.tar.gz"
  make_app_dir = "mkdir -p #{app_dir}"
  true_key = File.expand_path("~/.appscale/#{keyname}.key")
  
  Kernel.puts "Creating remote directory to copy app into"
  CommonFunctions.run_remote_command(head_node_ip,
    make_app_dir, true_key, false)
  Kernel.sleep(1)
  Kernel.puts "Copying over app"
  CommonFunctions.scp_file(local_file_path, remote_file_path,
    head_node_ip, true_key)

  return remote_file_path
end

.scp_file(local_file_loc, remote_file_loc, target_ip, public_key_loc) ⇒ Object 



574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
 
# File 'lib/common_functions.rb', line 574

def self.scp_file(local_file_loc, remote_file_loc, target_ip, public_key_loc)
  cmd = ""
  local_file_loc = File.expand_path(local_file_loc)
  retval_file = File.expand_path("~/.appscale/retval-#{rand()}")
 
  if public_key_loc.class == Array
    public_key_loc.each { |key|
      key = File.expand_path(key)
    }
    
    cmd = "scp -i #{public_key_loc.join(' -i ')} #{SSH_OPTIONS} 2>&1 #{local_file_loc} root@#{target_ip}:#{remote_file_loc}"
  else
    public_key_loc = File.expand_path(public_key_loc)
    cmd = "scp -i #{public_key_loc} #{SSH_OPTIONS} 2>&1 #{local_file_loc} root@#{target_ip}:#{remote_file_loc}"
  end

  cmd << "; echo $? > #{retval_file}"

  FileUtils.rm_f(retval_file)

  begin
    Timeout::timeout(INFINITY) { CommonFunctions.shell("#{cmd}") }
  rescue Timeout::Error
    raise AppScaleException.new("Remotely copying over files failed. Is " +
      "the destination machine on and reachable from this computer? We " +
      "tried the following command:\n\n#{cmd}")
  end

  loop {
    break if File.exists?(retval_file)
    Kernel.sleep(5)
  }

  retval = (File.open(retval_file) { |f| f.read }).chomp

  fails = 0
  loop {
    break if retval == "0"
    Kernel.puts "\n\n[#{cmd}] returned #{retval} instead of 0 as expected. Will try to copy again momentarily..."
    fails += 1
    if fails >= 5
      raise AppScaleException.new("SCP failed")
    end
    Kernel.sleep(1)
    CommonFunctions.shell("#{cmd}")
    retval = (File.open(retval_file) { |f| f.read }).chomp
  }

  FileUtils.rm_f(retval_file)
  return cmd
end

.shell(command) ⇒ Object

cgb: added in shell function for backticks so that we can unit test it Flexmock doesn’t like backticks since its name is non-alphanumeric e.g., its name is Kernel:‘



104
105
106
 
# File 'lib/common_functions.rb', line 104

def self.shell(command)
  `#{command}`
end

.sleep_until_port_is_closed(ip, port, use_ssl = false) ⇒ Object 



404
405
406
407
408
409
 
# File 'lib/common_functions.rb', line 404

def self.sleep_until_port_is_closed(ip, port, use_ssl=false)
  loop {
    return unless CommonFunctions.is_port_open?(ip, port, use_ssl)
    Kernel.sleep(1)
  }
end

.sleep_until_port_is_open(ip, port, use_ssl = false) ⇒ Object 



396
397
398
399
400
401
 
# File 'lib/common_functions.rb', line 396

def self.sleep_until_port_is_open(ip, port, use_ssl=false)
  loop {
    return if CommonFunctions.is_port_open?(ip, port, use_ssl)
    Kernel.sleep(1)
  }
end

.start_head_node(options, node_layout, apps_to_start) ⇒ Object 



474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
 
# File 'lib/common_functions.rb', line 474

def self.start_head_node(options, node_layout, apps_to_start)
  # since we changed the key's name sometimes it works with storing the key
  # in ssh.key and sometimes it needs to be in keyname.key{.private}
  # always do both just in case

  # TODO: check here to make sure that if hybrid, the keyname is free
  # in all clouds specified

  keyname = options['keyname']
  named_key_loc = "~/.appscale/#{keyname}.key"
  named_backup_key_loc = "~/.appscale/#{keyname}.private"
  ssh_key_location = named_key_loc
  ssh_keys = [ssh_key_location, named_key_loc, named_backup_key_loc]
  secret_key, secret_key_location = EncryptionHelper.generate_secret_key(keyname)
  self.verbose("New secret key is #{CommonFunctions.obscure_string(secret_key)}", options['verbose'])

  # TODO: serialize via json instead of this hacky way
  ips_hash = node_layout.to_hash
  ips_to_use = ips_hash.map { |node,roles| "#{node}--#{roles}" }.join("..")
   
  head_node = node_layout.head_node
  infrastructure = options['infrastructure']
  if infrastructure == "hybrid"   
    head_node_infra = VMTools.lookup_cloud_env(head_node.cloud)
    VMTools.set_hybrid_creds(node_layout)
    machine = VMTools.get_hybrid_machine(head_node_infra, "1")
  else
    head_node_infra = infrastructure
    machine = options['machine']
  end
   
  locations = VMTools.spawn_head_node(head_node, head_node_infra, keyname,
    ssh_key_location, ssh_keys, options['force'], machine,
    options['instance_type'], options['group'], options['verbose'])
 
  head_node_ip = locations.split(":")[0]
  instance_id = locations.scan(/i-\w+/).flatten.to_s
  locations = [locations]
 
  true_key = CommonFunctions.find_real_ssh_key(ssh_keys, head_node_ip)
 
  self.verbose("Log in to your head node: ssh -i #{true_key} " +
    "root@#{head_node_ip}", options['verbose'])
 
  CommonFunctions.ensure_image_is_appscale(head_node_ip, true_key)
  CommonFunctions.ensure_db_is_supported(head_node_ip, options['table'],
    true_key)
 
  scp = options['scp']
  if scp
    Kernel.puts "Copying over local copy of AppScale from #{scp}"
    CommonFunctions.rsync_files(head_node_ip, true_key, scp)
  end
 
  keypath = true_key.scan(/([\d|\w|\.]+)\Z/).flatten.to_s
  remote_key_location = "/root/.appscale/#{keyname}.key"
  CommonFunctions.scp_file(true_key, remote_key_location, head_node_ip, true_key)
 
  creds = CommonFunctions.generate_appscale_credentials(options, node_layout,
    head_node_ip, ips_to_use, true_key)
  self.verbose(CommonFunctions.obscure_creds(creds).inspect, options['verbose'])
 
  Kernel.puts "Head node successfully created at #{head_node_ip}. It is now " +
    "starting up #{options['table']} via the command line arguments given."
 
  RemoteLogging.remote_post(options['max_images'], options['table'],
    infrastructure, "started headnode", "success")
 
  Kernel.sleep(10) # sometimes this helps out with ec2 / euca deployments
    # gives them an extra moment to come up and accept scp requests
 
  CommonFunctions.copy_keys(secret_key_location, head_node_ip, true_key,
    options)
 
  CommonFunctions.start_appcontroller(head_node_ip, true_key,
    options['verbose'])
 
  acc = AppControllerClient.new(head_node_ip, secret_key)
  creds = creds.to_a.flatten
  acc.set_parameters(locations, creds, apps_to_start)

  return {:acc => acc, :head_node_ip => head_node_ip,
    :instance_id => instance_id, :true_key => true_key,
    :secret_key => secret_key}
end

.update_appcontroller(head_node_ip, secret, app_name, remote_file_path) ⇒ Object 



298
299
300
301
302
303
304
305
 
# File 'lib/common_functions.rb', line 298

def self.update_appcontroller(head_node_ip, secret, app_name,
  remote_file_path)

  acc = AppControllerClient.new(head_node_ip, secret)
  acc.done_uploading(app_name, remote_file_path)
  Kernel.puts "Updating AppController"
  acc.update([app_name])
end

.update_locations_file(keyname, ips = nil) ⇒ Object

This function tries to contact a node in the AppScale deployment to get the most up-to-date information on the current state of the deployment (as the number of nodes could change, or the roles that nodes have taken on).



145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
 
# File 'lib/common_functions.rb', line 145

def self.update_locations_file(keyname, ips=nil)
  secret = self.get_secret_key(keyname)

  # Don't worry about prioritizing the Shadow over any other nodes,
  # as all nodes should be checking ZooKeeper and keeping themselves
  # up-to-date already.
  new_role_info = nil

  if ips
    all_ips = ips
  else
    all_ips = self.get_all_public_ips(keyname)
  end

  all_ips.each { |ip|
    begin
      acc = AppControllerClient.new(ip, secret)
      new_role_info = acc.get_role_info()
      break
    rescue Exception
      Kernel.puts("Couldn't contact AppController at #{ip}, skipping...")
    end
  }

  if new_role_info.nil?
    abort("Couldn't contact any AppControllers - is AppScale running?")
  end

  CommonFunctions.write_nodes_json(new_role_info, keyname)
end

.user_has_cmd?(command) ⇒ Boolean

Returns:

  • (Boolean) 


359
360
361
362
363
364
365
366
 
# File 'lib/common_functions.rb', line 359

def self.user_has_cmd?(command)
  output = CommonFunctions.shell("which #{command}")
  if output.empty?
    return false
  else
    return true
  end
end

.validate_app_name(app_name, database) ⇒ Object 



199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
 
# File 'lib/common_functions.rb', line 199

def self.validate_app_name(app_name, database)
  disallowed = ["none", "auth", "login", "new_user", "load_balancer"]
  if disallowed.include?(app_name)
    raise AppEngineConfigException.new("App cannot be called " +
      "'#{not_allowed}' - this is a reserved name.")
  end

  if app_name =~ /[^[a-z]-]/
    raise AppEngineConfigException.new("App name can only contain " +
      "numeric and lowercase alphabetical characters.")
  end

  if app_name.include?("-") and database == "hypertable"
    raise AppEngineConfigException.new("App name cannot contain " +
      "dashes when Hypertable is the underlying database.") 
  end

  return app_name
end

.wait_for_app_to_start(head_node_ip, secret, app_name) ⇒ Object 



325
326
327
328
329
330
331
332
333
334
335
336
337
 
# File 'lib/common_functions.rb', line 325

def self.wait_for_app_to_start(head_node_ip, secret, app_name)
  acc = AppControllerClient.new(head_node_ip, secret)

  Kernel.puts "Please wait for your app to start up."
  loop {
    break if acc.app_is_running?(app_name)
    Kernel.sleep(5)
  }

  url_suffix = "/apps/#{app_name}"
  url = "http://#{head_node_ip}#{url_suffix}"
  Kernel.puts "\nYour app can be reached at the following URL: #{url}"
end

.wait_for_nodes_to_load(head_node_ip, secret) ⇒ Object 



307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
 
# File 'lib/common_functions.rb', line 307

def self.wait_for_nodes_to_load(head_node_ip, secret)
  head_acc = AppControllerClient.new(head_node_ip, secret)
  
  Kernel.puts "Please wait for AppScale to finish starting up."
  all_ips = head_acc.get_all_public_ips()
  loop {
    done_starting = true
    all_ips.each { |ip|
      acc = AppControllerClient.new(ip, secret)
      if !acc.is_done_initializing?
        done_starting = false
      end
    }
    break if done_starting
    Kernel.sleep(5)
  }
end

.wait_until_redirect(host, url_suffix) ⇒ Object 



340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
 
# File 'lib/common_functions.rb', line 340

def self.wait_until_redirect(host, url_suffix)
  uri = "http://#{host}#{url_suffix}"
  loop {
    response = ""
    begin
      response = Net::HTTP.get_response(URI.parse(uri))
    rescue Errno::ECONNREFUSED, EOFError
      Kernel.sleep(1)
      next
    rescue Exception => e
      raise e
    end
    
    return if response['location'] != "http://#{host}/status"
    Kernel.sleep(1)
  }
end

.warn_on_large_app_size(fullpath) ⇒ Object 



903
904
905
906
907
908
 
# File 'lib/common_functions.rb', line 903

def self.warn_on_large_app_size(fullpath)
  size = File.size(fullpath)
  if size > MAX_FILE_SIZE
    Kernel.puts "Warning: Your application is large enough that it may take a while to upload."
  end
end

.write_and_copy_node_file(options, node_layout, head_node_result) ⇒ Object 



812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
 
# File 'lib/common_functions.rb', line 812

def self.write_and_copy_node_file(options, node_layout, head_node_result)
  keyname = options['keyname']
  head_node_ip = head_node_result[:head_node_ip]

  all_ips = head_node_result[:acc].get_all_public_ips()
  db_master_ip = node_layout.db_master.id

  locations_yaml = File.expand_path("~/.appscale/locations-#{keyname}.yaml")
  CommonFunctions.write_node_file(head_node_ip,
    head_node_result[:instance_id], options['table'],
    head_node_result[:secret_key], db_master_ip, all_ips,
    options['infrastructure'], locations_yaml)
  remote_locations_file = "/root/.appscale/locations-#{keyname}.yaml"
  CommonFunctions.scp_file(locations_yaml, remote_locations_file,
    head_node_ip, head_node_result[:true_key])
end

.write_file(location, contents) ⇒ Object

A convenience function that can be used to write a string to a file.



85
86
87
 
# File 'lib/common_functions.rb', line 85

def self.write_file(location, contents)
  File.open(location, "w+") { |file| file.write(contents) }
end

.write_node_file(head_node_ip, instance_id, table, secret, db_master, ips, infrastructure, locations_yaml) ⇒ Object 



830
831
832
833
834
835
836
837
838
839
840
 
# File 'lib/common_functions.rb', line 830

def self.write_node_file(head_node_ip, instance_id, table, secret, db_master,
  ips, infrastructure, locations_yaml)

  infrastructure ||= "xen"
  tree = { :load_balancer => head_node_ip, :instance_id => instance_id , 
           :table => table, :shadow => head_node_ip, 
           :secret => secret , :db_master => db_master,
           :ips => ips , :infrastructure => infrastructure }
  loc_path = File.expand_path(locations_yaml)
  File.open(loc_path, "w") {|file| YAML.dump(tree, file)}
end

.write_nodes_json(new_role_info, keyname) ⇒ Object

Writes the given JSON to the ~/.appscale directory so that we can read it later and determine what nodes run what services.



806
807
808
809
 
# File 'lib/common_functions.rb', line 806

def self.write_nodes_json(new_role_info, keyname)
  filename = File.expand_path("~/.appscale/locations-#{keyname}.json")
  self.write_file(filename, JSON.dump(new_role_info))
end