Class: Common::Utils

Inherits:
Object
  • Object
show all
Includes:
ServiceConstants
Defined in:
lib/fluent/plugin/common.rb

Overview

Utilities for managing the resource used when writing to the Google API.

Defined Under Namespace

Modules: CredentialsInfo

Constant Summary

Constants included from ServiceConstants

ServiceConstants::APPENGINE_CONSTANTS, ServiceConstants::COMPUTE_CONSTANTS, ServiceConstants::DATAFLOW_CONSTANTS, ServiceConstants::DATAPROC_CONSTANTS, ServiceConstants::EC2_CONSTANTS, ServiceConstants::GKE_CONSTANTS, ServiceConstants::K8S_CONTAINER_CONSTANTS, ServiceConstants::K8S_NODE_CONSTANTS, ServiceConstants::K8S_POD_CONSTANTS, ServiceConstants::ML_CONSTANTS, ServiceConstants::SUBSERVICE_MAP, ServiceConstants::SUBSERVICE_METADATA_ATTRIBUTES

Instance Method Summary collapse

Constructor Details

#initialize(log) ⇒ Utils

Returns a new instance of Utils.


101
102
103
 
# File 'lib/fluent/plugin/common.rb', line 101

def initialize(log)
  @log = log
end

Instance Method Details

#check_required_metadata_variables(platform, project_id, zone, vm_id) ⇒ Object

Check required variables like @project_id, @vm_id, @vm_name and @zone.

Raises:

  • (Fluent::ConfigError) 

159
160
161
162
163
164
165
166
167
168
169
 
# File 'lib/fluent/plugin/common.rb', line 159

def check_required_metadata_variables(platform, project_id, zone, vm_id)
  missing = []
  missing << 'project_id' unless project_id
  if platform != Platform::OTHER
    missing << 'zone' unless zone
    missing << 'vm_id' unless vm_id
  end
  return if missing.empty?
  raise Fluent::ConfigError,
        "Unable to obtain metadata parameters: #{missing.join(' ')}"
end

#create_monitored_resource(type, labels) ⇒ Object

Create a monitored resource from type and labels.


223
224
225
226
 
# File 'lib/fluent/plugin/common.rb', line 223

def create_monitored_resource(type, labels)
  Google::Apis::LoggingV2::MonitoredResource.new(
    type: type, labels: labels.to_h)
end

#detect_platform(use_metadata_service) ⇒ Object

Determine what platform we are running on by consulting the metadata service (unless the user has explicitly disabled using that).


107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
 
# File 'lib/fluent/plugin/common.rb', line 107

def detect_platform(use_metadata_service)
  unless use_metadata_service
    @log.info 'use_metadata_service is false; not detecting platform'
    return Platform::OTHER
  end

  begin
    open('http://' + METADATA_SERVICE_ADDR, proxy: false) do |f|
      if f.meta['metadata-flavor'] == 'Google'
        @log.info 'Detected GCE platform'
        return Platform::GCE
      end
      if f.meta['server'] == 'EC2ws'
        @log.info 'Detected EC2 platform'
        return Platform::EC2
      end
    end
  rescue StandardError => e
    @log.error 'Failed to access metadata service: ', error: e
  end

  @log.info 'Unable to determine platform'
  Platform::OTHER
end

#determine_agent_level_monitored_resource_labels(platform, type, vm_id, zone) ⇒ Object

Determine agent level monitored resource labels based on the resource type. Each resource type has its own labels that need to be filled in.


278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
 
# File 'lib/fluent/plugin/common.rb', line 278

def determine_agent_level_monitored_resource_labels(
      platform, type, vm_id, zone)
  case type
  # GAE app.
  when APPENGINE_CONSTANTS[:resource_type]
    return {
      'module_id' =>
        fetch_gce_metadata(platform,
                           'instance/attributes/gae_backend_name'),
      'version_id' =>
        fetch_gce_metadata(platform,
                           'instance/attributes/gae_backend_version')
    }

  # GCE.
  when COMPUTE_CONSTANTS[:resource_type]
    raise "Cannot construct a #{type} resource without vm_id and zone" \
      unless vm_id && zone
    return {
      'instance_id' => vm_id,
      'zone' => zone
    }

  # GKE container.
  when GKE_CONSTANTS[:resource_type]
    raise "Cannot construct a #{type} resource without vm_id and zone" \
      unless vm_id && zone
    return {
      'instance_id' => vm_id,
      'zone' => zone,
      'cluster_name' =>
        fetch_gce_metadata(platform, 'instance/attributes/cluster-name')
    }

  # Cloud Dataproc.
  when DATAPROC_CONSTANTS[:resource_type]
    return {
      'cluster_uuid' =>
        fetch_gce_metadata(platform,
                           'instance/attributes/dataproc-cluster-uuid'),
      'cluster_name' =>
        fetch_gce_metadata(platform,
                           'instance/attributes/dataproc-cluster-name'),
      'region' =>
        fetch_gce_metadata(platform,
                           'instance/attributes/dataproc-region')
    }

  # EC2.
  when EC2_CONSTANTS[:resource_type]
    raise "Cannot construct a #{type} resource without vm_id and zone" \
      unless vm_id && zone
    labels = {
      'instance_id' => vm_id,
      'region' => zone
    }
    labels['aws_account'] = ec2_metadata(platform)['accountId'] if
      ec2_metadata(platform).key?('accountId')
    return labels
  end

  {}
rescue StandardError => e
  if [Platform::GCE, Platform::EC2].include?(platform)
    @log.error "Failed to set monitored resource labels for #{type}: ",
               error: e
  end
  {}
end

#determine_agent_level_monitored_resource_type(platform, subservice_name, detect_subservice) ⇒ Object

Determine agent level monitored resource type.


244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
 
# File 'lib/fluent/plugin/common.rb', line 244

def determine_agent_level_monitored_resource_type(
      platform, subservice_name, detect_subservice)
  case platform
  when Platform::OTHER
    # Unknown platform will be defaulted to GCE instance.
    return COMPUTE_CONSTANTS[:resource_type]

  when Platform::EC2
    return EC2_CONSTANTS[:resource_type]

  when Platform::GCE
    # Resource types determined by subservice_name config.
    return SUBSERVICE_MAP[subservice_name] if subservice_name

    # Resource types determined by detect_subservice config.
    if detect_subservice
      begin
        attributes = fetch_gce_metadata(platform,
                                        'instance/attributes/').split.to_set
        SUBSERVICE_METADATA_ATTRIBUTES.each do |resource_type, expected|
          return resource_type if attributes.superset?(expected)
        end
      rescue StandardError => e
        @log.error 'Failed to detect subservice: ', error: e
      end
    end

    # GCE instance.
    return COMPUTE_CONSTANTS[:resource_type]
  end
end

#determine_agent_level_monitored_resource_via_legacy(platform, subservice_name, detect_subservice, vm_id, zone) ⇒ Object

Retrieve monitored resource via the legacy way.

Note: This is just a failover plan if we fail to get metadata from Metadata Agent. Thus it should be equivalent to what Metadata Agent returns.


233
234
235
236
237
238
239
240
241
 
# File 'lib/fluent/plugin/common.rb', line 233

def determine_agent_level_monitored_resource_via_legacy(
      platform, subservice_name, detect_subservice, vm_id, zone)
  resource_type = determine_agent_level_monitored_resource_type(
    platform, subservice_name, detect_subservice)
  create_monitored_resource(
    resource_type,
    determine_agent_level_monitored_resource_labels(
      platform, resource_type, vm_id, zone))
end

#ec2_metadata(platform) ⇒ Object

EC2 Metadata server returns everything in one call. Store it after the first fetch to avoid making multiple calls.


143
144
145
146
147
148
149
150
151
152
153
154
155
156
 
# File 'lib/fluent/plugin/common.rb', line 143

def ec2_metadata(platform)
  raise "Called ec2_metadata with platform=#{platform}" unless
    platform == Platform::EC2
  unless @ec2_metadata
    # See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
    open('http://' + METADATA_SERVICE_ADDR +
         '/latest/dynamic/instance-identity/document', proxy: false) do |f|
      contents = f.read
      @ec2_metadata = JSON.parse(contents)
    end
  end

  @ec2_metadata
end

#fetch_gce_metadata(platform, metadata_path) ⇒ Object 


132
133
134
135
136
137
138
139
 
# File 'lib/fluent/plugin/common.rb', line 132

def fetch_gce_metadata(platform, metadata_path)
  raise "Called fetch_gce_metadata with platform=#{platform}" unless
    platform == Platform::GCE
  # See https://cloud.google.com/compute/docs/metadata
  open('http://' + METADATA_SERVICE_ADDR + '/computeMetadata/v1/' +
       metadata_path, 'Metadata-Flavor' => 'Google', :proxy => false,
       &:read)
end

#get_location(platform, zone, use_aws_availability_zone) ⇒ Object

  1. Return the value if it is explicitly set in the config already.

  2. If not, try to retrieve it locally.


204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
 
# File 'lib/fluent/plugin/common.rb', line 204

def get_location(platform, zone, use_aws_availability_zone)
  # Response format: "projects/<number>/zones/<zone>"
  zone ||= fetch_gce_metadata(platform,
                              'instance/zone').rpartition('/')[2] if
    platform == Platform::GCE
  aws_location_key = if use_aws_availability_zone
                       'availabilityZone'
                     else
                       'region'
                     end
  zone ||= 'aws:' + ec2_metadata(platform)[aws_location_key] if
    platform == Platform::EC2 &&
    ec2_metadata(platform).key?(aws_location_key)
  zone
rescue StandardError => e
  @log.error 'Failed to obtain location: ', error: e
end

#get_project_id(platform, project_id) ⇒ Object

  1. Return the value if it is explicitly set in the config already.

  2. If not, try to retrieve it by calling metadata server directly.

  3. If still not set, try to obtain it from the credentials.


174
175
176
177
178
179
 
# File 'lib/fluent/plugin/common.rb', line 174

def get_project_id(platform, project_id)
  project_id ||= CredentialsInfo.project_id
  project_id ||= fetch_gce_metadata(platform, 'project/project-id') if
    platform == Platform::GCE
  project_id
end

#get_vm_id(platform, vm_id) ⇒ Object

  1. Return the value if it is explicitly set in the config already.

  2. If not, try to retrieve it by calling metadata servers directly.


183
184
185
186
187
188
189
190
191
 
# File 'lib/fluent/plugin/common.rb', line 183

def get_vm_id(platform, vm_id)
  vm_id ||= fetch_gce_metadata(platform, 'instance/id') if
    platform == Platform::GCE
  vm_id ||= ec2_metadata(platform)['instanceId'] if
    platform == Platform::EC2
  vm_id
rescue StandardError => e
  @log.error 'Failed to obtain vm_id: ', error: e
end

#get_vm_name(vm_name) ⇒ Object

  1. Return the value if it is explicitly set in the config already.

  2. If not, try to retrieve it locally.


195
196
197
198
199
200
 
# File 'lib/fluent/plugin/common.rb', line 195

def get_vm_name(vm_name)
  vm_name ||= Socket.gethostname
  vm_name
rescue StandardError => e
  @log.error 'Failed to obtain vm name: ', error: e
end