Class: CloudProviders::Ec2

Inherits:
CloudProvider show all
Defined in:
lib/cloud_providers/ec2/ec2.rb

Instance Attribute Summary

Attributes inherited from CloudProvider

#init_opts, #name

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from CloudProvider

#after_initialized, #default_keypair_path, default_keypair_path, #initialize, #method_missing

Constructor Details

This class inherits a constructor from CloudProviders::CloudProvider

Dynamic Method Handling

This class handles dynamic methods through the method_missing method in the class CloudProviders::CloudProvider

Class Method Details

.default_access_key ⇒ Object

Set the aws keys from the environment, or load from /etc/poolparty/env.yml if the environment variable is not set



16
17
18
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 16

def self.default_access_key
  ENV['EC2_ACCESS_KEY'] || load_keys_from_file[:access_key] || load_keys_from_credential_file[:access_key]
end

.default_cert ⇒ Object 



28
29
30
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 28

def self.default_cert
  ENV['EC2_CERT'] || load_keys_from_file[:cert]
end

.default_cloud_cert ⇒ Object 



44
45
46
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 44

def self.default_cloud_cert
  ENV['CLOUD_CERT'] || ENV['EUCALYPTUS_CERT'] || load_keys_from_file[:cloud_cert]
end

.default_credential_file ⇒ Object 



48
49
50
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 48

def self.default_credential_file
  ENV['AWS_CREDENTIAL_FILE'] || load_keys_from_file[:credential_file]
end

.default_ec2_url ⇒ Object 



36
37
38
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 36

def self.default_ec2_url
  ENV['EC2_URL'] || load_keys_from_file[:ec2_url]
end

.default_private_key ⇒ Object 



24
25
26
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 24

def self.default_private_key
  ENV['EC2_PRIVATE_KEY'] || load_keys_from_file[:private_key]
end

.default_s3_url ⇒ Object 



40
41
42
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 40

def self.default_s3_url
  ENV['S3_URL'] || load_keys_from_file[:s3_url]
end

.default_secret_access_key ⇒ Object 



20
21
22
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 20

def self.default_secret_access_key
  ENV['EC2_SECRET_KEY'] || load_keys_from_file[:secret_access_key] || load_keys_from_credential_file[:secret_access_key]
end

.default_user_id ⇒ Object 



32
33
34
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 32

def self.default_user_id
  ENV['EC2_USER_ID'] || load_keys_from_file[:user_id]
end

.load_keys_from_credential_file(filename = default_credential_file, caching = true) ⇒ Object

Load credentials from file



61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 61

def self.load_keys_from_credential_file(filename=default_credential_file, caching=true)
  return {:access_key => @access_key, :secret_access_key => @secret_access_key} if @access_key and @secret_access_key and caching
  return {} if filename.nil? or not File.exists?(filename)
  puts("Reading keys from file: #{filename}")
  File.open(filename).each_line { |line|
    if line =~ /AWSAccessKeyId=([a-zA-Z0-9]+)$/
      @access_key=$1.chomp
    elsif line =~ /AWSSecretKey=([^   ]+)$/
      @secret_access_key=$1.chomp
    end
  }
  return {:access_key => @access_key, :secret_access_key => @secret_access_key}
end

.load_keys_from_file(filename = "#{ENV["HOME"]}/.poolparty/aws", caching = true) ⇒ Object

Load the yaml file containing keys. If the file does not exist, return an empty hash



53
54
55
56
57
58
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 53

def self.load_keys_from_file(filename="#{ENV["HOME"]}/.poolparty/aws", caching=true)
  return @aws_yml if @aws_yml && caching==true
  return {} unless File.exists?(filename)
  puts("Reading keys from file: #{filename}")
  @aws_yml = YAML::load( open(filename).read ) || {}
end

Instance Method Details

#all_nodes ⇒ Object 



303
304
305
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 303

def all_nodes
  @nodes ||= describe_instances.select {|i| security_group_names.include?(i.security_groups) }.sort {|a,b| DateTime.parse(a.launchTime) <=> DateTime.parse(b.launchTime)}
end

#as ⇒ Object

Proxy to the raw Grempe amazon-aws autoscaling instance



368
369
370
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 368

def as
  @as = AWS::Autoscaling::Base.new( :access_key_id => access_key, :secret_access_key => secret_access_key )
end

#assign_ebs_volumes ⇒ Object 



412
413
414
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 412

def assign_ebs_volumes
  ebs_volume_groups.each{|ebs_volume_group| ebs_volume_group.attach(nodes)}
end

#assign_elastic_ips ⇒ Object 



265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 265

def assign_elastic_ips
  unless elastic_ips.empty?
    unused_elastic_ip_addresses = ElasticIp.unused_elastic_ips(self).map {|i| i.public_ip }
    used_elastic_ip_addresses = ElasticIp.elastic_ips(self).map {|i| i.public_ip }

    elastic_ip_objects = ElasticIp.unused_elastic_ips(self).select {|ip_obj| elastic_ips.include?(ip_obj.public_ip) }

    assignee_nodes = nodes.select {|n| !ElasticIp.elastic_ips(self).include?(n.public_ip) }

    elastic_ip_objects.each_with_index do |eip, idx|
      # Only get the nodes that do not have elastic ips associated with them
      begin
        if assignee_nodes[idx]
          puts "Assigning elastic ip: #{eip.public_ip} to node: #{assignee_nodes[idx].instance_id}"
          ec2.associate_address(:instance_id => assignee_nodes[idx].instance_id, :public_ip => eip.public_ip)
        end
      rescue Exception => e
        p [:error, e.inspect]
      end
      reset!
    end
  end
end

#autoscale(given_name = cloud.proper_name, o = {}, &block) ⇒ Object 



337
338
339
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 337

def autoscale(given_name=cloud.proper_name, o={}, &block)
  autoscalers << ElasticAutoScaler.new(given_name, sub_opts.merge(o || {}), &block)
end

#autoscalers ⇒ Object 



390
391
392
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 390

def autoscalers
  @autoscalers ||= []
end

#available_rds_instances ⇒ Object 



351
352
353
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 351

def available_rds_instances
  rds_instances.select{|r| r.available? }
end

#awsrds ⇒ Object 



377
378
379
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 377

def awsrds
  @awsrds ||= AWS::RDS::Base.new( :access_key_id => access_key, :secret_access_key => secret_access_key )
end

#block_device_mapping(o = [], given_name = cloud.proper_name) ⇒ Object

Extras!



330
331
332
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 330

def block_device_mapping(o=[], given_name=cloud.proper_name )
  @mappings ||= o
end

#bootstrap_nodes!(tmp_path = nil) ⇒ Object 



239
240
241
242
243
244
245
246
247
248
249
250
251
252
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 239

def bootstrap_nodes!(tmp_path=nil)
  unless security_groups.map {|a| a.authorizes.map {|t| t.from_port.to_i }.flatten }.flatten.include?(22)      
    warn "Cloud security_groups are not authorized for ssh. Cannot bootstrap."
    return
  end
  tmp_path ||= cloud.tmp_path
  nodes.each do |node|
    next unless node.in_service?
    node.cloud_provider = self
    node.rsync_dir(tmp_path)
    node.bootstrap_chef!
    node.run_chef!
  end
end

#cleanup_ssh_known_hosts!(nodes_to_cleanup = nodes, even_unavailable = false) ⇒ Object 



289
290
291
292
293
294
295
296
297
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 289

def cleanup_ssh_known_hosts!(nodes_to_cleanup=nodes,
                             even_unavailable=false)
  puts "cleaning up .ssh/known_hosts"
  nodes_to_cleanup.find_all do |node|
    even_unavailable || node.ssh_available?
  end.each do |node|
    node.ssh_cleanup_known_hosts!
  end
end

#configure_nodes!(tmp_path = nil) ⇒ Object 



254
255
256
257
258
259
260
261
262
263
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 254

def configure_nodes!(tmp_path=nil)
  # removed duplicated code (now configure_nodes! invokes
  # node.bootstrap_chef!, while old version did not, but I believe
  # this is harmless)
  bootstrap_nodes!(tmp_path) 

  ebs_volume_groups.each do |vol_grp|
    vol_grp.verify_attachments nodes
  end
end

#contract_by(num = 1) ⇒ Object

Raises:

  • (RuntimeError) 


228
229
230
231
232
233
234
235
236
237
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 228

def contract_by(num=1)
  raise RuntimeError, "Contracting instances by #{num} will lower the number of instances below specified minimum" unless nodes.size - num > minimum_instances
  num.times do |i|
    node = nodes[-num]
    id = node.instance_id
    node.ssh_cleanup_known_hosts!
    Ec2Instance.terminate!(:instance_id => id, :cloud => cloud)
  end
  reset!
end

#create! ⇒ Object

Called when the create command is called on the cloud



100
101
102
103
104
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 100

def create!
  [:security_groups, :load_balancers, :rds_instances].each do |type|
    self.send(type).each {|ele| ele.create! }
  end
end

#credential_file(file = nil) ⇒ Object

Read credentials from credential_file if one exists



448
449
450
451
452
453
454
455
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 448

def credential_file(file=nil)
  unless file.nil?
    dsl_options[:credential_file]=file 
    dsl_options.merge!(Ec2.load_keys_from_credential_file(file))
  else
    fetch(:credential_file)
  end
end

#decoded_user_data ⇒ Object 



212
213
214
215
216
217
218
219
220
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 212

def decoded_user_data
  if user_data
    if File.file?(user_data)
      open(user_data).read
    else
      user_data
    end
  end
end

#describe_instances(id = nil) ⇒ Object

Describe instances Describe the instances that are available on this cloud with the id given will be returned if not given, details for all instances will be returned



312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 312

def describe_instances(id=nil)
  begin
    @describe_instances = ec2.describe_instances.reservationSet.item.map do |r|
      r.instancesSet.item.map do |i|
        inst_options = i.merge(r.merge(:cloud => cloud)).merge(cloud.cloud_provider.dsl_options)
        Ec2Instance.new(inst_options)
      end
    end.flatten
  rescue AWS::InvalidClientTokenId => e # AWS credentials invalid
	puts "Error contacting AWS: #{e}"
	raise e
  rescue Exception => e
    []
  end
end

#ebs_volume_groups ⇒ Object 



397
398
399
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 397

def ebs_volume_groups
  @ebs_volume_groups ||= []
end

#ebs_volumes(name = nil, &block) ⇒ Object

dsl method for EBS volumes. E.G.: 

ebs_volumes do
  volumes "vol-001248ff", "vol-01ff4b85" # use existing volumes, not mandatory
  device "/dev/sdf"
  snapshot_id "snap-602030dd"
  size 200
end



408
409
410
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 408

def ebs_volumes(name=nil, &block)
  ebs_volume_groups << ElasticBlockStoreGroup.new(sub_opts,&block) if block
end

#ec2 ⇒ Object

Proxy to the raw Grempe amazon-aws @ec2 instance



356
357
358
359
360
361
362
363
364
365
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 356

def ec2
  @ec2 ||= begin
   AWS::EC2::Base.new( :access_key_id => access_key, :secret_access_key => secret_access_key )
  rescue AWS::ArgumentError => e # AWS credentials missing?
	puts "Error contacting AWS: #{e}"
	raise e
  rescue Exception => e
	puts "Generic error #{e.class}: #{e}"
  end
end

#elastic_ip(*ips) ⇒ Object 



343
344
345
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 343

def elastic_ip(*ips)
  ips.each {|ip| elastic_ips << ip}
end

#elastic_ips ⇒ Object 



393
394
395
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 393

def elastic_ips
  @elastic_ips ||= []
end

#elb ⇒ Object

Proxy to the raw Grempe amazon-aws elastic_load_balancing instance



373
374
375
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 373

def elb
  @elb ||= AWS::ELB::Base.new( :access_key_id => access_key, :secret_access_key => secret_access_key )
end

#expand_by(num = 1) ⇒ Object 



189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 189

def expand_by(num=1)
  e = Ec2Instance.run!({
    :image_id => image_id,
    :min_count => num,
    :max_count => num,
    :key_name => keypair.basename,
    :security_groups => security_groups,
    :user_data => decoded_user_data,
    :instance_type => instance_type,
    :availability_zone => availability_zones.first,
    :base64_encoded => true,
    :cloud => cloud,
    :block_device_mapping => block_device_mapping,
    :disable_api_termination => disable_api_termination,
    :instance_initiated_shutdown_behavior => instance_initiated_shutdown_behavior,
    :subnet_id => subnet_id,
  })
  progress_bar_until("Waiting for node to launch...") do
    wait_for_node(e)
  end
  all_nodes.detect {|n| n.instance_id == e.instance_id }
end

#list_ec2_volumes(filters = nil) ⇒ Object

Get existing volumes on EC2. filters is a hash of filters, either single valued or multivalued (value is an array of possible values). The function will return volumes matching all filters. A volume is a filter match if any one of the filter values equals the volume parameter value.



427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 427

def list_ec2_volumes(filters=nil)
  @volumes_on_ec2=ec2.describe_volumes.volumeSet.item unless @volumes_on_ec2
  (if filters.nil? # no filter to check, so return at once
    @volumes_on_ec2
  else
    @volumes_on_ec2.select{|vol| # select volumes for which no filter failed
      not filters.map {|filter_key, filter_val|
        filter_key=filter_key.to_s if filter_key.is_a?(Symbol) # filter_key may be given as a symbol
        raise ArgumentError, "Filter key #{filter_key} is invalid" unless vol.has_key?(filter_key)
        if filter_val.is_a?(Array) # Deal with multiple filter values
          filter_val.map{|val| val.is_a?(String) ? val : val.to_s}.member?(vol[filter_key]) # make sure fiter_val array values are Strings before checking for match
        else
          (filter_val.is_a?(String) ? filter_val : filter_val.to_s)==vol[filter_key] # make sure fiter_val is a String before comparing
        end
        }.member?(false) # Check if a filter failed, the 'not' statement at the beginning of the map block negates this so 'select' will choose only when no filter failed
      }.compact # remove nil results from volume set.
    end
    ).map{|vol| ElasticBlockStore.new(vol,:cloud => cloud)}
end

#load_balancer(given_name = cloud.proper_name, o = {}, &block) ⇒ Object 



334
335
336
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 334

def load_balancer(given_name=cloud.proper_name, o={}, &block)
  load_balancers << ElasticLoadBalancer.new(given_name, sub_opts.merge(o || {}), &block)
end

#load_balancers ⇒ Object 



387
388
389
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 387

def load_balancers
  @load_balancers ||= []
end

#nodes ⇒ Object 



299
300
301
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 299

def nodes
  all_nodes.select {|i| i.in_service? }#describe_instances.select {|i| i.in_service? && security_groups.include?(i.security_groups) }
end

#rds(given_name = cloud.proper_name, o = {}, &block) ⇒ Object 



347
348
349
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 347

def rds(given_name=cloud.proper_name, o={}, &block)
  rds_instances << RdsInstance.new(given_name, sub_opts.merge(o || {}), &block)
end

#rds_instances ⇒ Object 



416
417
418
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 416

def rds_instances
  @rds_instances ||= []
end

#reset! ⇒ Object

Clear the cache



421
422
423
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 421

def reset!
  @nodes = @describe_instances = nil
end

#run ⇒ Object 



106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 106

def run
  puts "  for cloud: #{cloud.name}"
  puts "  minimum_instances: #{minimum_instances}"
  puts "  maximum_instances: #{maximum_instances}"
  puts "  security_groups: #{security_group_names.join(", ")}"
  puts "  using keypair: #{keypair}"
  puts "  user: #{user}\n"

  security_groups.each do |sg|
    sg.run
  end

  unless load_balancers.empty?
    load_balancers.each do |lb|
      puts "    load balancer: #{lb.name}"
      lb.run
    end
  end

  unless rds_instances.empty?
    rds_instances.each do |rdsi|
      puts "    rds instance: #{rdsi.name}"
      rdsi.run
    end
  end

  if autoscalers.empty? # not using autoscaling
    puts "---- live, running instances (#{nodes.size}) ----"
    if nodes.size < minimum_instances
      expansion_count = minimum_instances - nodes.size
      puts "-----> expanding the cloud because the #{expansion_count} minimum_instances is not satisified: "
      expand_by(expansion_count)
    elsif nodes.size > maximum_instances
      contraction_count = nodes.size - maximum_instances
      puts "-----> contracting the cloud because the instances count exceeds the #{maximum_instances} maximum_instances by #{contraction_count}"
      contract_by(contraction_count)
    end
    progress_bar_until("Waiting for the instances to be launched") do
      reset!
      running_nodes = nodes.select {|n| n.running? }
      running_nodes.size >= minimum_instances
    end
    reset!
    # ELASTIC IPS
  else
    autoscalers.each do |a|
      puts "    autoscaler: #{a.name}"
      puts "-----> The autoscaling groups will launch the instances"
      a.run
      
      progress_bar_until("Waiting for autoscaler to launch instances") do
        reset!
        running_nodes = nodes.select {|n| n.running? }
        running_nodes.size >= minimum_instances
      end
      reset!
    end
  end
  
  from_ports = security_groups.map {|a| a.authorizes.map {|t| t.from_port.to_i }.flatten }.flatten      
  if from_ports.include?(22)
    progress_bar_until("Waiting for the instances to be accessible by ssh") do
      running_nodes = nodes.select {|n| n.running? }
      accessible_count = running_nodes.map do |node|
        node.accessible?
      end.size
      accessible_count == running_nodes.size
    end
  end
  
  assign_elastic_ips
  cleanup_ssh_known_hosts!
  puts "Attaching EBS volumes"
  assign_ebs_volumes # Assign EBS volumes
end

#security_group(given_name = cloud.proper_name, o = {}, &block) ⇒ Object 



340
341
342
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 340

def security_group(given_name=cloud.proper_name, o={}, &block)
  security_groups << SecurityGroup.new(given_name, sub_opts.merge(o || {}), &block)
end

#security_group_names ⇒ Object 



381
382
383
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 381

def security_group_names
  security_groups.map {|a| a.to_s }
end

#security_groups ⇒ Object 



384
385
386
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 384

def security_groups
  @security_groups ||= []
end

#teardown ⇒ Object 



182
183
184
185
186
187
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 182

def teardown
  puts "------ Tearing down and cleaning up #{cloud.name} cloud"
  unless autoscalers.empty?
    puts "Tearing down autoscalers"
  end
end

#wait_for_node(instance) ⇒ Object 



222
223
224
225
226
 
# File 'lib/cloud_providers/ec2/ec2.rb', line 222

def wait_for_node(instance)
  reset!
  inst = all_nodes.detect {|n| n.instance_id == instance.instance_id }
  inst.running? if inst
end