Class: NodeLayout

Inherits:
Object
  • Object
show all
Defined in:
lib/node_layout.rb

Constant Summary collapse

SIMPLE_FORMAT_KEYS = 
[:controller, :servers]
ADVANCED_FORMAT_KEYS = 
[:master, :database, :appengine, :open, :login, :zookeeper, :memcache, :rabbitmq]

Instance Method Summary collapse

Constructor Details

#initialize(input_yaml, options, skip_replication = false) ⇒ NodeLayout

Required options are: database_type



30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/node_layout.rb', line 30

def initialize(input_yaml, options, skip_replication=false)
  @input_yaml = (input_yaml.kind_of?(String) ? YAML.load(input_yaml) : input_yaml)

  @infrastructure = options[:infrastructure]
  @database_type = options[:database]
  @database_type = @database_type.to_sym if !@database_type.nil?
  @min_images = options[:min_images]
  @max_images = options[:max_images]
  @replication = options[:replication]
  @read_factor = options[:read_factor]
  @write_factor = options[:write_factor]
  
  @nodes = []
  @skip_replication = skip_replication
end

Instance Method Details

#db_master ⇒ Object 



517
518
519
520
521
522
523
 
# File 'lib/node_layout.rb', line 517

def db_master
  return nil unless valid?

  db_master = @nodes.select { |n| n.is_db_master? }.compact
  
  db_master.empty? ? nil : db_master[0]
end

#errors ⇒ Object 



56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/node_layout.rb', line 56

def errors
  return [] if valid?

  if is_simple_format? 
    valid_simple_format?[:message]
  elsif is_advanced_format?
    valid_advanced_format?[:message]
  elsif @input_yaml.nil?
    [INPUT_YAML_REQUIRED]
  else
    keys = @input_yaml.keys

    keys.each { |key|
      if !(SIMPLE_FORMAT_KEYS.include?(key) || ADVANCED_FORMAT_KEYS.include?(key))
        return ["The flag #{key} is not a supported flag"]
      end
    }

    return [USED_SIMPLE_AND_ADVANCED_KEYS]
  end
end

#generate_cloud_layout ⇒ Object

Generates an yaml file for non-hybrid cloud layouts which don’t have them



447
448
449
450
451
452
453
454
455
456
457
 
# File 'lib/node_layout.rb', line 447

def generate_cloud_layout
  layout = {:controller => "node-0"}
  servers = []
  num_slaves = @min_images - 1
  num_slaves.times do |i|
    servers << "node-#{i+1}"
  end

  layout[:servers] = servers
  YAML.load(layout.to_yaml)
end

#head_node ⇒ Object

head node -> shadow



500
501
502
503
504
505
506
507
 
# File 'lib/node_layout.rb', line 500

def head_node
  return nil unless valid?

  head_node = @nodes.select { |n| n.is_shadow? }.compact
  
  # TODO: is the last guard necessary?
  head_node.empty? ? nil : head_node[0]
end

#is_advanced_format? ⇒ Boolean

Returns:

  • (Boolean) 


97
98
99
100
101
102
103
104
105
 
# File 'lib/node_layout.rb', line 97

def is_advanced_format?
  return false if @input_yaml.nil?

  @input_yaml.keys.each do |key|
    return false if !ADVANCED_FORMAT_KEYS.include?(key)
  end

  true
end

#is_simple_format? ⇒ Boolean

Returns:

  • (Boolean) 


78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
 
# File 'lib/node_layout.rb', line 78

def is_simple_format?
  if @input_yaml.nil?
    if VALID_CLOUD_TYPES.include?(@infrastructure) and @infrastructure != "hybrid"
      # When used with the cloud, the simple format doesn't require a yaml
      # Note this is not so in the hybrid model - a yaml is required in
      # that scenario.
      return true
    else
      return false
    end
  end

  @input_yaml.keys.each do |key|
    return false if !SIMPLE_FORMAT_KEYS.include?(key)
  end

 true
end

#login_node ⇒ Object 



525
526
527
528
529
530
531
 
# File 'lib/node_layout.rb', line 525

def 
  return nil unless valid?

   = @nodes.select { |n| n.is_login? }.compact
  
  .empty? ? nil : [0]
end

#max_images ⇒ Object 



486
487
488
489
490
 
# File 'lib/node_layout.rb', line 486

def max_images
  return nil unless valid?
  
  @max_images
end

#min_images ⇒ Object 



480
481
482
483
484
 
# File 'lib/node_layout.rb', line 480

def min_images
  return nil unless valid? 

  @min_images
end

#nodes ⇒ Object 



492
493
494
495
496
497
 
# File 'lib/node_layout.rb', line 492

def nodes
  return [] unless valid?
  
  # Since the valid? check has succeded @nodes has been initialized
  @nodes
end

#other_nodes ⇒ Object 



509
510
511
512
513
514
515
 
# File 'lib/node_layout.rb', line 509

def other_nodes
  return [] unless valid?

  other_nodes = @nodes.select { |n| !n.is_shadow? }.compact
  
  other_nodes.empty? ? [] : other_nodes
end

#parse_ip(ip) ⇒ Object 



107
108
109
110
111
112
113
114
115
116
117
118
119
120
 
# File 'lib/node_layout.rb', line 107

def parse_ip(ip)
  id, cloud = nil, nil

  match = NODE_ID_REGEX.match(ip)
  if match.nil?
    id = ip
    cloud = "not-cloud"
  else
    id = match[0]
    cloud = match[1]
  end

  return id, cloud
end

#read_factor ⇒ Object

TODO: can we just replace the if w/ unless and change ! to = ? or does that not exactly work due to the || ?



468
469
470
471
472
 
# File 'lib/node_layout.rb', line 468

def read_factor
  return nil if !valid? || @database_type != :voldemort

  @read_factor
end

#replication_factor ⇒ Object 



459
460
461
462
463
 
# File 'lib/node_layout.rb', line 459

def replication_factor
  return nil unless valid?

  @replication
end

#to_hash ⇒ Object 



534
535
536
537
538
539
540
541
 
# File 'lib/node_layout.rb', line 534

def to_hash
  result = {}
  # Put all nodes except the head node in the hash
  other_nodes.each do |node|
    result[node.id] = node.roles.join(":")
  end
  result
end

#valid? ⇒ Boolean

Returns:

  • (Boolean) 


46
47
48
49
50
51
52
53
54
 
# File 'lib/node_layout.rb', line 46

def valid?
  if is_simple_format? 
    valid_simple_format?[:result]
  elsif is_advanced_format?
    valid_advanced_format?[:result]
  else
    false
  end
end

#valid_advanced_format? ⇒ Boolean

Returns:

  • (Boolean) 


227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
 
# File 'lib/node_layout.rb', line 227

def valid_advanced_format?
  # We already computed the nodes, its valid
  return valid if !@nodes.empty?

  node_hash = {}
  @input_yaml.each_pair do |role, ips|
    
    ips.each_with_index do |ip, index|
      node = nil
      if node_hash[ip].nil?
        id, cloud = parse_ip(ip)
        node = AdvancedNode.new(id, cloud)
      else
        node = node_hash[ip]
      end

      if role.to_sym == :database
        # The first database node is the master
        is_master = index.zero?
        node.add_db_role @database_type, is_master
      elsif role.to_sym == :db_master
        node.add_role :zookeeper
        node.add_role role
      elsif role.to_sym == :rabbitmq
        # Like the database, the first rabbitmq node is the master
        is_master = index.zero?
        node.add_role :rabbitmq
        node.add_rabbitmq_role is_master
      else
        node.add_role role
      end
      
      node_hash[ip] = node
    end
  end

  # Dont need the hash any more, make a nodes list
  nodes = node_hash.values

  nodes.each do |node|
    return invalid(node.errors.join(",")) unless node.valid?

    if VALID_CLOUD_TYPES.include?(@infrastructure)
      error_message = "Invalid cloud node ID: #{node.id} \n" + 
        "Cloud node ID must be in the format 'node-{IDNUMBER}'" +
        "\nor of the form cloud{CLOUDNUMBER}-{IDNUMBER} for hybrid deployments"
      return invalid(error_message) if NODE_ID_REGEX.match(node.id.to_s).nil?
    else
      # Xen/KVM should be using the ip address as the node id
      error_message = "Invalid virtualized node ID: #{node.id} \n" + 
        "Virtualized node IDs must be a valid IP address"
      return invalid(error_message) if IP_REGEX.match(node.id.to_s).nil?
    end
  end

  master_nodes = nodes.select { |node| node.is_shadow? }.compact

  # need exactly one master
  if master_nodes.length == 0
    return invalid("No master was specified")
  elsif master_nodes.length > 1
    return invalid("Only one master is allowed")
  end

  master_node = master_nodes.first

   = nodes.select { |node| node.is_login? }.compact
  # If a login node was not specified, make the master into the login node
  if .empty?
    master_node.add_role :login
  end

  appengine_count = 0
  nodes.each do |node|
    if node.is_appengine?
      appengine_count += 1
    end
  end

  if appengine_count < 1
    return invalid("Not enough appengine nodes were provided.")
  end

  memcache_count = 0
  nodes.each do |node|
    if node.is_memcache?
      memcache_count += 1
    end
  end

  # if no memcache nodes were specified, make all appengine nodes
  # into memcache nodes
  if memcache_count < 1
    nodes.each { |node|
      node.add_role :memcache if node.is_appengine?
    }
  end

  if VALID_CLOUD_TYPES.include?(@infrastructure)
    # If min and max aren't specified, they default to the number of nodes in the system
    @min_images ||= nodes.length
    @max_images ||= nodes.length

    # TODO: look into if that first guard is really necessary with the preceding lines

    if @min_images && nodes.length < @min_images
      return invalid("Too few nodes were provided, #{nodes.length} were specified but #{@min_images} was the minimum")
    end
 
    if @max_images && nodes.length > @max_images
      return invalid("Too many nodes were provided, #{nodes.length} were specified but #{@max_images} was the maximum")
    end
  end

  zookeeper_count = 0
  nodes.each do |node|
    if node.is_zookeeper?
      zookeeper_count += 1
    end
  end
  master_node.add_role :zookeeper if zookeeper_count.zero?

  # If no rabbitmq nodes are specified, make the shadow the rabbitmq_master
  rabbitmq_count = 0
  nodes.each do |node|
    if node.is_rabbitmq?
      rabbitmq_count += 1
    end
  end
  if rabbitmq_count.zero?
    master_node.add_role :rabbitmq
    master_node.add_role :rabbitmq_master
  end

  # Any node that runs appengine needs rabbitmq to dispatch task requests to
  # It's safe to add the slave role since we ensure above that somebody
  # already has the master role
  nodes.each do |node|
    if node.is_appengine? and !node.is_rabbitmq?
      node.add_role :rabbitmq_slave
    end
  end

  database_count = 0
  nodes.each do |node|
    if node.is_database?
      database_count += 1
    end
  end

  if @skip_replication
    @nodes = nodes
    return valid
  end

  rep = valid_database_replication? nodes
  return rep unless rep[:result]

  # Wait until it is validated to assign it
  @nodes = nodes

  return valid
end

#valid_database_replication?(nodes) ⇒ Boolean

Returns:

  • (Boolean) 


391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
 
# File 'lib/node_layout.rb', line 391

def valid_database_replication? nodes
  database_node_count = 0
  nodes.each do |node|
    if node.is_database? or node.is_db_master?
      database_node_count += 1
    end
  end

  if database_node_count.zero?
    return invalid("At least one database node must be provided.")
  end

  if @replication.nil?
    if database_node_count > 3
      # If there are a lot of database nodes, we default to 3x replication
      @replication = 3
    else
      # If there are only a few nodes, replicate to each one of the nodes
      @replication = database_node_count
    end
  end

  if @replication > database_node_count
    return invalid("The provided replication factor is too high. The replication factor (-n flag) cannot be greater than the number of database nodes.")
  end

  # Perform all the database specific checks here
  if @database_type == :mysql && database_node_count % @replication != 0
    return invalid("MySQL requires that the amount of replication be divisible by the number of nodes (e.g. with 6 nodes, 2 or 3 times replication). You specified #{database_node_count} database nodes which is not divisible by #{@replication} times replication.")
  end

  if @database_type == :voldemort
    @read_factor ||= @replication
    @write_factor ||= @replication

    if @read_factor > @replication
      return invalid("The provided read factor is too high. The read factor (-r flag) cannot be greater than the replication factor.")
    elsif @write_factor > @replication
      return invalid("The provided write factor is too high. The write factor (-w flag) cannot be greater than the replication factor.")
    end
  end

  if @database_type == :simpledb
    if ENV['SIMPLEDB_ACCESS_KEY'].nil?
      return invalid("SimpleDB deployments require that the environment variable SIMPLEDB_ACCESS_KEY be set to your AWS access key.")
    end

    if ENV['SIMPLEDB_SECRET_KEY'].nil?
      return invalid("SimpleDB deployments require that the environment variable SIMPLEDB_SECRET_KEY be set to your AWS secret key.")
    end
  end
  
  valid
end

#valid_simple_format? ⇒ Boolean

Returns:

  • (Boolean) 


122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
 
# File 'lib/node_layout.rb', line 122

def valid_simple_format?
  # We already computed the nodes, its valid
  # cgb: an optimization to ensure we don't keep calling this
  # when it always returns the same thing anyways
  return valid if !@nodes.empty?

  if @input_yaml.nil?
    if VALID_CLOUD_TYPES.include?(@infrastructure) and @infrastructure != "hybrid"
      if @min_images.nil?
        return invalid(NO_INPUT_YAML_REQUIRES_MIN_IMAGES)
      end

      if @max_images.nil?
        return invalid(NO_INPUT_YAML_REQUIRES_MAX_IMAGES)
      end

      # No yaml was created so we will create a generic one and then allow it to be validated
      @input_yaml = generate_cloud_layout
    else
      return invalid(INPUT_YAML_REQUIRED)
    end
  end

  nodes = []
  @input_yaml.each_pair do |role, ips|
    next if ips.nil?

    ips.each do |ip|
      id, cloud = parse_ip(ip)
      node = SimpleNode.new id, cloud, [role]

      # In simple deployments the db master and rabbitmq master is always on
      # the shadow node, and db slave / rabbitmq slave is always on the other
      # nodes
      is_master = node.is_shadow?
      node.add_db_role @database_type, is_master
      node.add_rabbitmq_role is_master

      return invalid(node.errors.join(",")) if !node.valid?

      if VALID_CLOUD_TYPES.include?(@infrastructure)
        error_message = "Invalid cloud node ID: #{node.id} \n" +
          "Cloud node IDs must be in the format 'node-{IDNUMBER}'" +
          "\nor of the form cloud{CLOUDNUMBER}-{IDNUMBER} for hybrid deployments"
        return invalid(error_message) if NODE_ID_REGEX.match(node.id.to_s).nil?
      else
        # Xen/KVM should be using the ip address as the node id
        error_message = "Invalid virtualized node ID: #{node.id} \n" + 
          "Virtualized node IDs must be a valid IP address"
        return invalid(error_message) if IP_REGEX.match(node.id.to_s).nil?
      end

      nodes << node
    end
  end

  # make sure that the user hasn't erroneously specified the same ip
  # address more than once
  all_ips = @input_yaml.values.flatten
  duplicate_ips = all_ips.length - all_ips.uniq.length

  unless duplicate_ips.zero?
    return invalid(DUPLICATE_IPS)
  end

  if nodes.length == 1
    # Singleton node should be master and app engine
    nodes.first.add_role :appengine
    nodes.first.add_role :memcache
  end

  # controller -> shadow
  controller_count = 0
  nodes.each do |node|
    if node.is_shadow?
      controller_count += 1
    end
  end

  if controller_count == 0
    return invalid(NO_CONTROLLER)
  elsif controller_count > 1
    return invalid(ONLY_ONE_CONTROLLER)
  end

  database_count = 0
  nodes.each do |node|
    if node.is_database?
      database_count += 1
    end
  end

  if @skip_replication
    @nodes = nodes
    return valid
  end

  rep = valid_database_replication? nodes
  return rep unless rep[:result]

  # Wait until it is validated to assign it
  @nodes = nodes
  valid
end

#write_factor ⇒ Object 



474
475
476
477
478
 
# File 'lib/node_layout.rb', line 474

def write_factor
  return nil if !valid? || @database_type != :voldemort

  @write_factor
end