Class: LogStash::Outputs::Application_insights

Inherits:
Base
  • Object
show all
Defined in:
lib/logstash/outputs/application_insights/validate_notification.rb,
lib/logstash/outputs/application_insights/notification_recovery.rb,
lib/logstash/outputs/application_insights/test_notification.rb,
lib/logstash/outputs/application_insights/shutdown_recovery.rb,
lib/logstash/outputs/application_insights/validate_storage.rb,
lib/logstash/outputs/application_insights/storage_recovery.rb,
lib/logstash/outputs/application_insights/storage_cleanup.rb,
lib/logstash/outputs/application_insights/multi_io_logger.rb,
lib/logstash/outputs/application_insights/test_storage.rb,
lib/logstash/outputs/application_insights/notification.rb,
lib/logstash/outputs/application_insights/flow_control.rb,
lib/logstash/outputs/application_insights/upload_pipe.rb,
lib/logstash/outputs/application_insights/sub_channel.rb,
lib/logstash/outputs/application_insights/state_table.rb,
lib/logstash/outputs/application_insights/local_file.rb,
lib/logstash/outputs/application_insights/exceptions.rb,
lib/logstash/outputs/application_insights/telemetry.rb,
lib/logstash/outputs/application_insights/constants.rb,
lib/logstash/outputs/application_insights/shutdown.rb,
lib/logstash/outputs/application_insights/channels.rb,
lib/logstash/outputs/application_insights/context.rb,
lib/logstash/outputs/application_insights/clients.rb,
lib/logstash/outputs/application_insights/channel.rb,
lib/logstash/outputs/application_insights/config.rb,
lib/logstash/outputs/application_insights/client.rb,
lib/logstash/outputs/application_insights/utils.rb,
lib/logstash/outputs/application_insights/timer.rb,
lib/logstash/outputs/application_insights/state.rb,
lib/logstash/outputs/application_insights/block.rb,
lib/logstash/outputs/application_insights/blob.rb,
lib/logstash/outputs/application_insights.rb

Overview

Logstash Output Application Insights

Copyright © Microsoft Corporation

All rights reserved.

Licensed under the Apache License, Version 2.0 (the License); you may not use this file except in compliance with the License. You may obtain a copy of the License at www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

See the Apache Version 2.0 License for specific language governing permissions and limitations under the License.

Defined Under Namespace

Classes: AssertError, Blob, Block, BlockOverflowError, BlockTooSmallError, Channel, ChannelExistError, Channels, Client, Clients, Config, ConfigurationError, Context, Flow_control, Local_file, Multi_io_logger, NoChannelError, NotRecoverableError, Notification, Notification_recovery, Shutdown, Shutdown_recovery, State, State_table, StorageAccountsOffError, Storage_cleanup, Storage_recovery, Sub_channel, Telemetry, Test_notification, Test_storage, Timer, UnexpectedBranchError, UploadRetryError, Upload_pipe, Utils, Validate_notification, Validate_storage

Constant Summary collapse

BOOLEAN_PROPERTIES = 
[  :disable_notification, :disable_blob_upload, 
:stop_on_unknown_io_errors, :enable_telemetry_to_microsoft, 
:disable_cleanup, :delete_not_notified_blobs,
:validate_notification, :validate_storage,
:save_notified_blobs_records, :case_insensitive_columns,
:disable_compression, :disable_truncation,
:table_columns, :serialized_event_field ]
GUID_NULL = 
"00000000-0000-0000-0000-000000000000"
INSTRUMENTATION_KEY_TEMPLATE = 
"KKKKKKKK-KKKK-KKKK-KKKK-KKKKKKKKKKKK"
TABLE_ID_TEMPLATE = 
"SSSSSSSS-SSSS-SSSS-SSSS-SSSSSSSSSSSS"
REQUEST_NAME =

notification payload constants

"Microsoft.ApplicationInsights.OpenSchema"
BASE_DATA_REQUIRED_VERSION = 
"2"
DATA_BASE_TYPE = 
"OpenSchemaData"
LOGGER_LEVEL_MAP =

logger constants

{:DEBUG => Logger::DEBUG, :INFO => Logger::INFO, :WARN => Logger::WARN, :ERROR => Logger::ERROR, :FATAL => Logger::FATAL, :UNKNOWN => Logger::UNKNOWN}
BLOB_BLOCK_MAX_BYTESIZE =

in bytes - 4 Mega bytes - blob limitation

4 * 1024 * 1024
BLOB_MAX_BLOCKS =

in blocks - 50,0000 blocks - blob limitation

50000
MIN_BLOB_MAX_BYTESIZE =

BLOB_BLOCK_MAX_BYTESIZE

BLOB_BLOCK_MAX_BYTESIZE
MAX_BLOB_MAX_BYTESIZE =

192 Giga bytes

BLOB_MAX_BLOCKS * BLOB_BLOCK_MAX_BYTESIZE
MIN_BLOB_MAX_EVENTS =

256 Kilo events

1
MAX_BLOB_MAX_EVENTS =

No Limit

0
MIN_BLOB_MAX_DELAY =

in seconds - one second

1
MAX_BLOB_MAX_DELAY =

in seconds - one day

24 * 3600
MIN_BLOB_RETENTION_TIME =

in seconds - one minute

60
MAX_BLOB_RETENTION_TIME =

in seconds - one year

60 * 60 * 24 * 365
MIN_BLOB_ACCESS_EXPIRY_TIME =

in seconds - one minute

60
MAX_BLOB_ACCESS_EXPIRY_TIME =

in seconds - one year

60 * 60 * 24 * 365
MIN_STORAGE_RESURRECT_DELAY =

in seconds - one second

1
MAX_STORAGE_RESURRECT_DELAY =

in seconds - one hour

3600
MIN_LOGGER_SHIFT_SIZE =

in bytes - 1 Kilo bytes

1024
MAX_LOGGER_SHIFT_SIZE =

in bytes - 1 Giga bytes

1 * 1024 * 1024 * 1024
MIN_LOGGER_SHIFT_AGE =

in retension version - no

0
MAX_LOGGER_SHIFT_AGE =

in retension version - almost limitless

100000
MIN_IO_RETRY_DELAY =

in seconds

0
MAX_IO_RETRY_DELAY =

in seconds - 5 minutes

300
MIN_IO_MAX_RETRIES = 
0
MAX_IO_MAX_RETRIES = 
3
MIN_FLOW_CONTROL_SUSPEND_BYTES =

in bytes,

0
MAX_FLOW_CONTROL_SUSPEND_BYTES =

in bytes, 0 means no upper limit

0
MIN_FLOW_CONTROL_RESUME_BYTES =

in bytes

0
MAX_FLOW_CONTROL_RESUME_BYTES =

in bytes, 0 means no upper limit

0
MIN_FLOW_CONTROL_DELAY =

in seconds, 1 seconds, can be less than 1 seconds, like 0.5, 0.1

0.1
MAX_FLOW_CONTROL_DELAY =

in seconds, 1 seconds, can be less than 1 seconds, like 0.5, 0.1

0
MAX_CHANNEL_UPLOAD_PIPES = 
40
CHANNEL_THRESHOLD_TO_ADD_UPLOAD_PIPE =

not relevant for file upload mode

3
MAX_FIELD_BYTES = 
1024 * 1024
METADATA_FIELD_INSTRUMENTATION_KEY = 
"[@metadata]instrumentation_key"
METADATA_FIELD_TABLE_ID = 
"[@metadata]table_id"
FIELD_INSTRUMENTATION_KEY = 
"instrumentation_key"
FIELD_TABLE_ID = 
"table_id"
STATE_TABLE_NAME = 
"BlobsState"
STORAGE_TEST_CONTAINER_NAME = 
"test-container"
STORAGE_TEST_TABLE_NAME = 
"TestTable"
AZURE_STORAGE_CONTAINER_LOGSTASH_PREFIX =

lower case only, dash allowed

"logstash"
AZURE_STORAGE_BLOB_LOGSTASH_PREFIX = 
"logstash"
AZURE_STORAGE_TABLE_LOGSTASH_PREFIX =

case sensitive, no dash

"Logstash"
LOCAL_FS_FILE_PREFIX = 
"application_insights"
AZURE_STORAGE_ORPHAN_BLOBS_CONTAINER_NAME = 
"orphan-blobs"
AZURE_STORAGE_NOTIFIED_BLOBS_TABLE_NAME = 
"orphan-blobs"
COLUMN_PROPERTY_NAME = 
"name"
COLUMN_PROPERTY_TYPE = 
"type"
COLUMN_PROPERTY_DEFAULT = 
"default"
VALID_FIELDS_MAP_TYPES = 
[ "string", "hash", "array", "number", "json", "boolean", "float", "integer", "dynamic", "datetime", "object" ]
VALID_LOGGER_SHIFT_AGES = 
[ "daily", "weekly", "monthly" ]
EXT_EVENT_FORMAT_JSON = 
"json"
EXT_EVENT_FORMAT_CSV = 
"csv"
VALID_EXT_EVENT_FORMAT = 
[EXT_EVENT_FORMAT_JSON, EXT_EVENT_FORMAT_CSV ]
DEFAULT_INSTRUMENTATION_KEY = 
GUID_NULL
DEFAULT_TABLE_ID = 
GUID_NULL
DEFAULT_EVENT_SEPARATOR = 
"\r\n"
DEFAULT_CSV_SEPARATOR = 
","
DEFAULT_CSV_DEFAULT_VALUE = 
""
DEFAULT_AZURE_STORAGE_BLOB_PREFIX = 
nil
DEFAULT_AZURE_STORAGE_CONTAINER_PREFIX = 
nil
DEFAULT_AZURE_STORAGE_TABLE_PREFIX = 
nil
DEFAULT_BLOB_SERIALIZATION = 
EXT_EVENT_FORMAT_JSON
DEFAULT_BLOB_MAX_BYTESIZE =

4 Giga bytes

4 * 1024 * 1024 * 1024
DEFAULT_BLOB_MAX_EVENTS =

1,000,000

1000 * 1000
DEFAULT_BLOB_MAX_DELAY =

in seconds

60
DEFAULT_BLOB_RETENTION_TIME =

in seconds - one week

60 * 60 * 24 * 7
DEFAULT_BLOB_ACCESS_EXPIRY_TIME =

in seconds - one day

60 * 60 * 24 * 1
DEFAULT_STORAGE_RESURRECT_DELAY = 
10
DEFAULT_APPLICATION_INSIGHTS_ENDPOINT = 
"https://dc.services.visualstudio.com/v2/track"
DEFAULT_AZURE_STORAGE_HOST_SUFFIX = 
"core.windows.net"
DEFAULT_NOTIFICATION_VERSION = 
1
DEFAULT_DISABLE_NOTIFICATION = 
false
DEFAULT_DISABLE_BLOB_UPLOAD = 
false
DEFAULT_STOP_ON_UNKNOWN_IO_ERRORS = 
false
DEFAULT_ENABLE_TELEMETRY_TO_MICROSOFT = 
false
DEFAULT_DISABLE_CLEANUP = 
false
DEFAULT_DELETE_NOT_NOTIFIED_BLOBS = 
false
DEFAULT_SAVE_NOTIFIED_BLOBS_RECORDS = 
false
DEFAULT_DISABLE_COMPRESSION = 
false
DEFAULT_DISABLE_TRUNCATION = 
false
DEFAULT_CASE_INSENSITIVE = 
false
DEFAULT_VALIDATE_NOTIFICATION = 
false
DEFAULT_VALIDATE_STORAGE = 
false
DEFAULT_LOGGER_FILES = 
[ "logstash-output-application-insights.log" ]
DEFAULT_LOG_LEVEL = 
"INFO"
DEFAULT_LOGGER_PROGNAME = 
"AI"
DEFAULT_LOGGER_SHIFT_AGE =

daily back retension

"daily"
DEFAULT_LOGGER_SHIFT_SIZE =

in bytes - one Mega bytes

1024 * 1024
DEFAULT_IO_RETRY_DELAY =

in seconds

10
DEFAULT_IO_MAX_RETRIES = 
2
DEFAULT_FLOW_CONTROL_SUSPEND_BYTES =

high water mark, -in bytes, based on my laptop experience, without Java to break to NO MEMORY error

50 *1024 * 1024
DEFAULT_FLOW_CONTROL_RESUME_BYTES =

low water mark,

40 *1024 * 1024
DEFAULT_FLOW_CONTROL_DELAY =

in seconds, 1 seconds, can be less than 1 seconds, like 0.5, 0.1

1

Instance Method Summary collapse

Instance Method Details

#close ⇒ Object 



384
385
386
387
388
389
390
391
 
# File 'lib/logstash/outputs/application_insights.rb', line 384

def close
  @telemetry.track_event { { :name => "close" } }
  @telemetry.flush
  @shutdown_recovery.close
  @storage_recovery.close
  @notification_recovery.close
  @shutdown.submit
end

#default_configuration ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
 
# File 'lib/logstash/outputs/application_insights/constants.rb', line 24

def default_configuration
  {
    :notification_version => @notification_version || DEFAULT_NOTIFICATION_VERSION,
    :event_separator => @event_separator || DEFAULT_EVENT_SEPARATOR,

    :azure_storage_host_suffix => @azure_storage_host_suffix || DEFAULT_AZURE_STORAGE_HOST_SUFFIX, 
    :application_insights_endpoint => @application_insights_endpoint || DEFAULT_APPLICATION_INSIGHTS_ENDPOINT,
    :azure_storage_blob_prefix => @azure_storage_blob_prefix || DEFAULT_AZURE_STORAGE_BLOB_PREFIX || Utils.to_storage_name( Socket.gethostname.strip ) || "",
    :azure_storage_container_prefix => @azure_storage_container_prefix || DEFAULT_AZURE_STORAGE_CONTAINER_PREFIX || Utils.to_storage_name( Socket.gethostname.strip ) || "",
    :azure_storage_table_prefix => @azure_storage_table_prefix || DEFAULT_AZURE_STORAGE_TABLE_PREFIX || Utils.to_storage_name( Socket.gethostname.strip ) || "",
    :storage_account_name_key => @storage_account_name_key || [  ],
    :disable_notification => @disable_notification || DEFAULT_DISABLE_NOTIFICATION,
    :disable_blob_upload => @disable_blob_upload || DEFAULT_DISABLE_BLOB_UPLOAD,
    :stop_on_unknown_io_errors => @stop_on_unknown_io_errors || DEFAULT_STOP_ON_UNKNOWN_IO_ERRORS,
    :delete_not_notified_blobs => @delete_not_notified_blobs || DEFAULT_DELETE_NOT_NOTIFIED_BLOBS,
    :save_notified_blobs_records => @save_notified_blobs_records || DEFAULT_SAVE_NOTIFIED_BLOBS_RECORDS,
    :enable_telemetry_to_microsoft => @enable_telemetry_to_microsoft || DEFAULT_ENABLE_TELEMETRY_TO_MICROSOFT,
    :disable_cleanup => @disable_cleanup || DEFAULT_DISABLE_CLEANUP,
    :blob_max_bytesize => @blob_max_bytesize || DEFAULT_BLOB_MAX_BYTESIZE,
    :blob_max_events => @blob_max_events || DEFAULT_BLOB_MAX_EVENTS,
    :blob_retention_time => @blob_retention_time || DEFAULT_BLOB_RETENTION_TIME,
    :blob_access_expiry_time => @blob_access_expiry_time || DEFAULT_BLOB_ACCESS_EXPIRY_TIME,

    :validate_notification => @validate_notification || DEFAULT_VALIDATE_NOTIFICATION,
    :validate_storage => @validate_storage || DEFAULT_VALIDATE_STORAGE,

    :resurrect_delay => @resurrect_delay || DEFAULT_STORAGE_RESURRECT_DELAY,
    :io_retry_delay => @io_retry_delay || DEFAULT_IO_RETRY_DELAY,
    :io_max_retries => @io_max_retries || DEFAULT_IO_MAX_RETRIES,

    :logger_level => @logger_level || DEFAULT_LOG_LEVEL,
    :logger_files => @logger_files || DEFAULT_LOGGER_FILES,
    :logger_progname => @logger_progname || DEFAULT_LOGGER_PROGNAME,
    :logger_shift_size => @logger_shift_size || DEFAULT_LOGGER_SHIFT_SIZE,
    :logger_shift_age => @logger_shift_age || DEFAULT_LOGGER_SHIFT_AGE,

    :flow_control_suspend_bytes => @flow_control_suspend_bytes || DEFAULT_FLOW_CONTROL_SUSPEND_BYTES,
    :flow_control_resume_bytes => @flow_control_resume_bytes || DEFAULT_FLOW_CONTROL_RESUME_BYTES,
    :flow_control_delay => @flow_control_delay || DEFAULT_FLOW_CONTROL_DELAY,

    :ca_file => @ca_file || "",

    :tables => @tables || {  },
    :table_id => @table_id || DEFAULT_TABLE_ID,
    :instrumentation_key => @instrumentation_key || DEFAULT_INSTRUMENTATION_KEY,
    :table_columns => @table_columns,
    :case_insensitive_columns => @case_insensitive_columns || DEFAULT_CASE_INSENSITIVE,
    :serialized_event_field => @serialized_event_field,
    :blob_max_delay => @blob_max_delay || DEFAULT_BLOB_MAX_DELAY,
    :blob_serialization => @blob_serialization || DEFAULT_BLOB_SERIALIZATION,
    :csv_separator => @csv_separator || DEFAULT_CSV_SEPARATOR,
    :csv_default_value => @csv_default_value || DEFAULT_CSV_DEFAULT_VALUE,
    :disable_compression => @disable_compression || DEFAULT_DISABLE_COMPRESSION,
    :disable_truncation => @disable_truncation || DEFAULT_DISABLE_TRUNCATION,

  }
end

#receive(event) ⇒ Object 



378
379
380
381
382
 
# File 'lib/logstash/outputs/application_insights.rb', line 378

def receive ( event )
  # @codec.encode( event )
  @channels.receive( event, nil )
  return "ok\n"
end

#register ⇒ Object 



313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
 
# File 'lib/logstash/outputs/application_insights.rb', line 313

def register

  # puts "++ SETTINGS: #{LogStash::SETTINGS.to_hash}"
  # @logger.info("test test", :xx => "xx", :yy => 1)
  # puts "++ LOGSTASH_HOME: #{LogStash::Environment::LOGSTASH_HOME}"

  # logstash define: @original_params = original_params
  # logstash define: @config = params

  # set configuration
  Config.validate_and_adjust_configuration( default_configuration )
  configuration = Config.current
  masked_configuration = Config.masked_current

  Multi_io_logger.config( configuration )

  # be careful don't use here @logger, as it will override Logstash @logger, and may create starnge behaviour
  @private_logger = configuration[:logger]
  @private_logger.info { "configuration: #{masked_configuration}" }

  @telemetry = Telemetry.instance
  configuration[:telemetry_channel] = @telemetry.telemetry_channel

  Timer.config( configuration )

  @notification_recovery = Notification_recovery.instance
  @storage_recovery = Storage_recovery.instance
  @shutdown_recovery = Shutdown_recovery.instance

  if @validate_notification
    result = Validate_notification.new.validate
    raise ConfigurationError, "Failed to access application insights at #{configuration[:application_insights_endpoint]}, due to error #{result[:error].inspect}" unless result[:success]
  end

  if @validate_storage
    results = Validate_storage.new.validate
    results.each do |result|
      raise ConfigurationError, "Failed to #{result[:test]} in azure storage account #{result[:storage_account_name]}, due to error #{result[:error].inspect}" unless result[:success]
    end
  end

  @notification_recovery.start
  @storage_recovery.start
  @shutdown_recovery.start

  @shutdown = Shutdown.instance
  @channels = Channels.instance

  @shutdown.start
  @channels.start

  @storage_cleanup = Storage_cleanup.start

  @private_logger.info { "plugin registered" }

  # @codec.on_event do |event, encoded_event|
  #   @channels.receive( event, encoded_event )
  # end

  @telemetry.track_event { { :name => "register", :properties => masked_configuration } }

  return "ok\n"
end