Class: Common::Utils

Inherits:
Object
  • Object
show all
Includes:
ServiceConstants
Defined in:
lib/fluent/plugin/common.rb

Overview

Utilities for managing the resource used when writing to the Google API.

Defined Under Namespace

Modules: CredentialsInfo

Constant Summary

Constants included from ServiceConstants

ServiceConstants::APPENGINE_CONSTANTS, ServiceConstants::COMPUTE_CONSTANTS, ServiceConstants::DATAFLOW_CONSTANTS, ServiceConstants::DATAPROC_CONSTANTS, ServiceConstants::EC2_CONSTANTS, ServiceConstants::GKE_CONSTANTS, ServiceConstants::K8S_CONTAINER_CONSTANTS, ServiceConstants::K8S_NODE_CONSTANTS, ServiceConstants::K8S_POD_CONSTANTS, ServiceConstants::ML_CONSTANTS, ServiceConstants::SUBSERVICE_MAP, ServiceConstants::SUBSERVICE_METADATA_ATTRIBUTES

Instance Method Summary collapse

Constructor Details

#initialize(log) ⇒ Utils

Returns a new instance of Utils.



103
104
105
 
# File 'lib/fluent/plugin/common.rb', line 103

def initialize(log)
  @log = log
end

Instance Method Details

#check_required_metadata_variables(platform, project_id, zone, vm_id) ⇒ Object

Check required variables like @project_id, @vm_id, @vm_name and @zone.

Raises:

  • (Fluent::ConfigError) 


161
162
163
164
165
166
167
168
169
170
171
172
 
# File 'lib/fluent/plugin/common.rb', line 161

def check_required_metadata_variables(platform, project_id, zone, vm_id)
  missing = []
  missing << 'project_id' unless project_id
  if platform != Platform::OTHER
    missing << 'zone' unless zone
    missing << 'vm_id' unless vm_id
  end
  return if missing.empty?

  raise Fluent::ConfigError,
        "Unable to obtain metadata parameters: #{missing.join(' ')}"
end

#create_monitored_resource(type, labels) ⇒ Object

Create a monitored resource from type and labels.



227
228
229
230
231
 
# File 'lib/fluent/plugin/common.rb', line 227

def create_monitored_resource(type, labels)
  Google::Apis::LoggingV2::MonitoredResource.new(
    type: type, labels: labels.to_h
  )
end

#detect_platform(use_metadata_service) ⇒ Object

Determine what platform we are running on by consulting the metadata service (unless the user has explicitly disabled using that).



109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
 
# File 'lib/fluent/plugin/common.rb', line 109

def detect_platform(use_metadata_service)
  unless use_metadata_service
    @log.info 'use_metadata_service is false; not detecting platform'
    return Platform::OTHER
  end

  begin
    URI.open("http://#{METADATA_SERVICE_ADDR}", proxy: false) do |f|
      if f.meta['metadata-flavor'] == 'Google'
        @log.info 'Detected GCE platform'
        return Platform::GCE
      end
      if f.meta['server'] == 'EC2ws'
        @log.info 'Detected EC2 platform'
        return Platform::EC2
      end
    end
  rescue StandardError => e
    @log.error 'Failed to access metadata service: ', error: e
  end

  @log.info 'Unable to determine platform'
  Platform::OTHER
end

#determine_agent_level_monitored_resource_labels(platform, type, vm_id, zone) ⇒ Object

Determine agent level monitored resource labels based on the resource type. Each resource type has its own labels that need to be filled in.



288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
 
# File 'lib/fluent/plugin/common.rb', line 288

def determine_agent_level_monitored_resource_labels(
  platform, type, vm_id, zone
)
  case type
  # GAE app.
  when APPENGINE_CONSTANTS[:resource_type]
    return {
      'module_id' =>
        fetch_gce_metadata(platform,
                           'instance/attributes/gae_backend_name'),
      'version_id' =>
        fetch_gce_metadata(platform,
                           'instance/attributes/gae_backend_version')
    }

  # GCE.
  when COMPUTE_CONSTANTS[:resource_type]
    raise "Cannot construct a #{type} resource without vm_id and zone" \
      unless vm_id && zone

    return {
      'instance_id' => vm_id,
      'zone' => zone
    }

  # GKE container.
  when GKE_CONSTANTS[:resource_type]
    raise "Cannot construct a #{type} resource without vm_id and zone" \
      unless vm_id && zone

    return {
      'instance_id' => vm_id,
      'zone' => zone,
      'cluster_name' =>
        fetch_gce_metadata(platform, 'instance/attributes/cluster-name')
    }

  # Cloud Dataproc.
  when DATAPROC_CONSTANTS[:resource_type]
    return {
      'cluster_uuid' =>
        fetch_gce_metadata(platform,
                           'instance/attributes/dataproc-cluster-uuid'),
      'cluster_name' =>
        fetch_gce_metadata(platform,
                           'instance/attributes/dataproc-cluster-name'),
      'region' =>
        fetch_gce_metadata(platform,
                           'instance/attributes/dataproc-region')
    }

  # EC2.
  when EC2_CONSTANTS[:resource_type]
    raise "Cannot construct a #{type} resource without vm_id and zone" \
      unless vm_id && zone

    labels = {
      'instance_id' => vm_id,
      'region' => zone
    }
    labels['aws_account'] = ec2_metadata(platform)['accountId'] if
      ec2_metadata(platform).key?('accountId')
    return labels
  end

  {}
rescue StandardError => e
  if [Platform::GCE, Platform::EC2].include?(platform)
    @log.error "Failed to set monitored resource labels for #{type}: ",
               error: e
  end
  {}
end

#determine_agent_level_monitored_resource_type(platform, subservice_name, detect_subservice) ⇒ Object

Determine agent level monitored resource type.



253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
 
# File 'lib/fluent/plugin/common.rb', line 253

def determine_agent_level_monitored_resource_type(
  platform, subservice_name, detect_subservice
)
  case platform
  when Platform::OTHER
    # Unknown platform will be defaulted to GCE instance.
    COMPUTE_CONSTANTS[:resource_type]

  when Platform::EC2
    EC2_CONSTANTS[:resource_type]

  when Platform::GCE
    # Resource types determined by subservice_name config.
    return SUBSERVICE_MAP[subservice_name] if subservice_name

    # Resource types determined by detect_subservice config.
    if detect_subservice
      begin
        attributes = fetch_gce_metadata(platform,
                                        'instance/attributes/').split.to_set
        SUBSERVICE_METADATA_ATTRIBUTES.each do |resource_type, expected|
          return resource_type if attributes.superset?(expected)
        end
      rescue StandardError => e
        @log.error 'Failed to detect subservice: ', error: e
      end
    end

    # GCE instance.
    COMPUTE_CONSTANTS[:resource_type]
  end
end

#determine_agent_level_monitored_resource_via_legacy(platform, subservice_name, detect_subservice, vm_id, zone) ⇒ Object

Retrieve monitored resource via the legacy way.

Note: This is just a failover plan if we fail to get metadata from Metadata Agent. Thus it should be equivalent to what Metadata Agent returns.



238
239
240
241
242
243
244
245
246
247
248
249
250
 
# File 'lib/fluent/plugin/common.rb', line 238

def determine_agent_level_monitored_resource_via_legacy(
  platform, subservice_name, detect_subservice, vm_id, zone
)
  resource_type = determine_agent_level_monitored_resource_type(
    platform, subservice_name, detect_subservice
  )
  create_monitored_resource(
    resource_type,
    determine_agent_level_monitored_resource_labels(
      platform, resource_type, vm_id, zone
    )
  )
end

#ec2_metadata(platform) ⇒ Object

EC2 Metadata server returns everything in one call. Store it after the first fetch to avoid making multiple calls.



145
146
147
148
149
150
151
152
153
154
155
156
157
158
 
# File 'lib/fluent/plugin/common.rb', line 145

def ec2_metadata(platform)
  raise "Called ec2_metadata with platform=#{platform}" unless
    platform == Platform::EC2

  unless @ec2_metadata
    # See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
    URI.open("http://#{METADATA_SERVICE_ADDR}/latest/dynamic/instance-identity/document", proxy: false) do |f|
      contents = f.read
      @ec2_metadata = JSON.parse(contents)
    end
  end

  @ec2_metadata
end

#fetch_gce_metadata(platform, metadata_path) ⇒ Object 



134
135
136
137
138
139
140
141
 
# File 'lib/fluent/plugin/common.rb', line 134

def fetch_gce_metadata(platform, metadata_path)
  raise "Called fetch_gce_metadata with platform=#{platform}" unless
    platform == Platform::GCE

  # See https://cloud.google.com/compute/docs/metadata
  URI.open("http://#{METADATA_SERVICE_ADDR}/computeMetadata/v1/#{metadata_path}",
           'Metadata-Flavor' => 'Google', :proxy => false, &:read)
end

#get_location(platform, zone, use_aws_availability_zone) ⇒ Object

  1. Return the value if it is explicitly set in the config already.

  2. If not, try to retrieve it locally.



207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
 
# File 'lib/fluent/plugin/common.rb', line 207

def get_location(platform, zone, use_aws_availability_zone)
  # Response format: "projects/<number>/zones/<zone>"
  if platform == Platform::GCE
    zone ||= fetch_gce_metadata(platform,
                                'instance/zone').rpartition('/')[2]
  end
  aws_location_key = if use_aws_availability_zone
                       'availabilityZone'
                     else
                       'region'
                     end
  zone ||= "aws:#{ec2_metadata(platform)[aws_location_key]}" if
    platform == Platform::EC2 &&
    ec2_metadata(platform).key?(aws_location_key)
  zone
rescue StandardError => e
  @log.error 'Failed to obtain location: ', error: e
end

#get_project_id(platform, project_id) ⇒ Object

  1. Return the value if it is explicitly set in the config already.

  2. If not, try to retrieve it by calling metadata server directly.

  3. If still not set, try to obtain it from the credentials.



177
178
179
180
181
182
 
# File 'lib/fluent/plugin/common.rb', line 177

def get_project_id(platform, project_id)
  project_id ||= CredentialsInfo.project_id
  project_id ||= fetch_gce_metadata(platform, 'project/project-id') if
    platform == Platform::GCE
  project_id
end

#get_vm_id(platform, vm_id) ⇒ Object

  1. Return the value if it is explicitly set in the config already.

  2. If not, try to retrieve it by calling metadata servers directly.



186
187
188
189
190
191
192
193
194
 
# File 'lib/fluent/plugin/common.rb', line 186

def get_vm_id(platform, vm_id)
  vm_id ||= fetch_gce_metadata(platform, 'instance/id') if
    platform == Platform::GCE
  vm_id ||= ec2_metadata(platform)['instanceId'] if
    platform == Platform::EC2
  vm_id
rescue StandardError => e
  @log.error 'Failed to obtain vm_id: ', error: e
end

#get_vm_name(vm_name) ⇒ Object

  1. Return the value if it is explicitly set in the config already.

  2. If not, try to retrieve it locally.



198
199
200
201
202
203
 
# File 'lib/fluent/plugin/common.rb', line 198

def get_vm_name(vm_name)
  vm_name ||= Socket.gethostname
  vm_name
rescue StandardError => e
  @log.error 'Failed to obtain vm name: ', error: e
end