Module: Fluent::Plugin::PublicLoggingUtils

Included in:
OCILoggingOutput
Defined in:
lib/fluent/plugin/logging_utils.rb

Overview

Common utility methods

Constant Summary collapse

PUBLIC_CLIENT_SPEC_VERSION = 
'1.0'
PUBLIC_LOGGING_PREFIX = 
'com.oraclecloud.logging.custom.'

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.determine_config_profile_name(linux_cfg, windows_cfg) ⇒ String

Return the correct oci configuration user profile for auth

Parameters:

  • linux_cfg (String) —

    Path to linux configuration file

  • windows_cfg (String) —

    Path to windows configuration file

Returns:

  • (String) —

    username to use



193
194
195
196
197
198
199
200
201
202
203
204
 
# File 'lib/fluent/plugin/logging_utils.rb', line 193

def self.determine_config_profile_name(linux_cfg, windows_cfg)
  managed_agent='UNIFIED_MONITORING_AGENT'
  unmanaged_agent='DEFAULT'
  location = OS.windows? ? windows_cfg : linux_cfg
  if location.start_with?('/etc/unified-monitoring-agent',  'C:\\oracle_unified_agent')
    return managed_agent
  end
  oci_config = OCI::ConfigFileLoader.load_config(config_file_location: location)
  oci_config.profile
rescue StandardError
  unmanaged_agent
end

.determine_linux_config_path ⇒ String

Return the correct oci configuration file path for linux platforms.

Returns:

  • (String) —

    path to the configuration file



164
165
166
167
168
169
170
171
 
# File 'lib/fluent/plugin/logging_utils.rb', line 164

def self.determine_linux_config_path
  managed_agent = '/etc/unified-monitoring-agent/.oci/config'
  unmanaged_agent = File.join(Dir.home, '.oci/config')
  config = File.file?(managed_agent) ? managed_agent : unmanage_agent
  return config
rescue StandardError
  return unmanaged_agent
end

.determine_windows_config_path ⇒ String

Return the correct oci configuration file path for windows platforms.

Returns:

  • (String) —

    path to the configuration file



177
178
179
180
181
182
183
184
 
# File 'lib/fluent/plugin/logging_utils.rb', line 177

def self.determine_windows_config_path
  managed_agent = 'C:\\oracle_unified_agent\\.oci\\config'
  unmanaged_agent = File.join("C:\\Users\\#{Etc.getlogin}\\.oci\\config")
  config = File.file?(managed_agent) ? managed_agent : unmanage_agent
  return config
rescue StandardError
  return unmanaged_agent
end

Instance Method Details

#build_request(time, record, tagpath, log_batches_map, sourceidentifier) ⇒ Array

Build the requests to be sent to the Lumberjack endpoint.

Parameters:

  • time (Time) —

    Fluentd event time (may be different from event’s timestamp)

  • record (Hash) —

    The fluentd record.

  • tagpath (String) —

    The fluentd path if populated.

  • log_batches_map (Hash) —

    List of pre-existing log batch.

  • sourceidentifier (String) —

    The fluentd contianing the source path.

Returns:

  • (Array) —

    requests List of requests for Lumberjack’s backend.



40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/fluent/plugin/logging_utils.rb', line 40

def build_request(time, record, tagpath, log_batches_map, sourceidentifier)
  log = @log || Logger.new($stdout)
  content = flatten_hash(record)

  # create lumberjack request records
  logentry = ::OCI::Loggingingestion::Models::LogEntry.new
  logentry.time = Time.at(time).utc.strftime('%FT%T.%LZ')
  begin
    logentry.data = content.to_json
  rescue StandardError
    begin
      log.warn('ruby "to_json" expected UTF-8 encoding will retry parsing with forced encoding')
      # json requires UTF-8 encoding some logs may not follow that format so we need to fix that
      content = encode_to_utf8(content)
      logentry.data = content.to_json
    rescue StandardError
      log.warn('unexpected encoding in the log request, will send log as a string instead of json')
      # instead of loosing data because of an unknown encoding issue send the data as a string
      logentry.data = content.to_s
    end
  end
  logentry.id = SecureRandom.uuid

  requestkey = tagpath + sourceidentifier

  unless log_batches_map.key?(requestkey)
    log_entry_batch = OCI::Loggingingestion::Models::LogEntryBatch.new
    log_entry_batch.source = @hostname
    log_entry_batch.type = PUBLIC_LOGGING_PREFIX + tagpath
    log_entry_batch.subject = sourceidentifier
    log_entry_batch.defaultlogentrytime = Time.at(time).utc.strftime('%FT%T.%LZ')
    log_entry_batch.entries = []

    log_batches_map[requestkey] = log_entry_batch
  end

  log_batches_map[requestkey].entries << logentry
end

#encode_to_utf8(record) ⇒ Hash

Force all the string values in the hash to be encoded to UTF-8.

Parameters:

  • record (Hash) —

    the flattened hash needing to have the encoding changes

Returns:

  • (Hash) —

    The updated hash.



142
143
144
145
146
147
 
# File 'lib/fluent/plugin/logging_utils.rb', line 142

def encode_to_utf8(record)
  # the reason for using ISO-8859-1 is that it covers most of the out
  # of band characters other encoding's don't have this way we can
  # encode it to a known value and then encode it back into UTF-8
  record.transform_values { |v| v.to_s.force_encoding('ISO-8859-1').encode('UTF-8') }
end

#flatten_hash(record) ⇒ Hash

Flatten the keys of a hash.

Parameters:

  • record (Hash) —

    The hash object to flatten.

Returns:

  • (Hash) —

    The updated, flattened, hash.



124
125
126
127
128
129
130
131
132
133
134
 
# File 'lib/fluent/plugin/logging_utils.rb', line 124

def flatten_hash(record)
  record.each_with_object({}) do |(k, v), h|
    if v.is_a? Hash
      flatten_hash(v).map { |h_k, h_v| h["#{k}.#{h_k}"] = h_v }
    elsif k == 'log'
      h['msg'] = v
    else
      h[k] = v
    end
  end
end

#get_modified_tag(rawtag) ⇒ String

Parse out the log_type from the chunk metadata tag

Parameters:

  • rawtag (String) —

    the string of the chunk metadata tag that needs to be parsed

Returns:

  • (String) —

    take out the tag after the first ‘.’ character or return the whole tag if there is no ‘.’



155
156
157
158
 
# File 'lib/fluent/plugin/logging_utils.rb', line 155

def get_modified_tag(rawtag)
  tag = rawtag || 'empty'
  tag.split('.').length > 1 ? tag.partition('.')[2] : tag
end

#get_put_logs_details_request ⇒ OCI::LoggingClient::Models::PutLogsDetails

Build the wrapper for all the log entries to be sent to lumberjack.

Returns:

  • (OCI::LoggingClient::Models::PutLogsDetails) —

    PutLogsDetails wrapper to be filled with log entries



23
24
25
26
27
 
# File 'lib/fluent/plugin/logging_utils.rb', line 23

def get_put_logs_details_request
  request = OCI::Loggingingestion::Models::PutLogsDetails.new
  request.specversion = PUBLIC_CLIENT_SPEC_VERSION
  request
end

#send_requests(log_batches_map) ⇒ Object

Send prebuilt requests to the logging endpoint.

Parameters:

  • log_batches_map (Hash) 


83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/fluent/plugin/logging_utils.rb', line 83

def send_requests(log_batches_map)
  log = @log || Logger.new($stdout) # for testing

  request = get_put_logs_details_request

  request.log_entry_batches = log_batches_map.values
  begin
    resp = @client.put_logs(@log_object_id, request)
  rescue OCI::Errors::ServiceError => e
    log.error "Service Error received sending request: #{e}"
    if e.status_code == 400
      log.info 'Eating service error as it is caused by Bad Request[400 HTTP code]'
    else
      log.error "Throwing service error for status code:#{e.status_code} as we want fluentd to re-try"
      raise
    end
  rescue OCI::Errors::NetworkError => e
    log.error "Network Error received sending request: #{e}"
    if e.code == 400
      log.info 'Eating network error as it is caused by Bad Request[400 HTTP code]'
    else
      log.error "Throwing network error for code:#{e.code} as we want fluentd to re-try"
      raise
    end
  rescue StandardError => e
    log.error "Standard Error received sending request: #{e}"
    raise
  end
  request.log_entry_batches.each do |batch|
    log.info "log batch type #{batch.type}, log batch subject #{batch.subject}, size #{batch.entries.size}"
  end

  log.info "response #{resp.status} id: #{resp.request_id}"
end