Class: Fluent::Plugin::SQSOutput

Inherits:
Output
  • Object
show all
Defined in:
lib/fluent/plugin/out_sqs.rb

Constant Summary collapse

DEFAULT_BUFFER_TYPE = 
"memory"

Instance Method Summary collapse

Instance Method Details

#client ⇒ Object 



41
42
43
44
45
46
47
48
 
# File 'lib/fluent/plugin/out_sqs.rb', line 41

def client
  @client ||= Aws::SQS::Client.new(
    access_key_id: @aws_key_id,
    secret_access_key: @aws_sec_key,
    region: @region,
    stub_responses: @stub_responses
  )
end

#configure(conf) ⇒ Object 



32
33
34
35
36
37
38
39
 
# File 'lib/fluent/plugin/out_sqs.rb', line 32

def configure(conf)
  compat_parameters_convert(conf, :buffer, :inject)
  super

  if (!@queue_name.nil? && @queue_name.end_with?('.fifo'))
    raise Fluent::ConfigError, 'message_group_id parameter is required for FIFO queue' if @message_group_id.nil?
  end
end

#format(tag, time, record) ⇒ Object 



70
71
72
73
74
75
 
# File 'lib/fluent/plugin/out_sqs.rb', line 70

def format(tag, time, record)
  record[@tag_property_name] = tag if @include_tag
  record = inject_values_to_record(tag, time, record)

  record.to_msgpack
end

#formatted_to_msgpack_binary ⇒ Object 



77
78
79
 
# File 'lib/fluent/plugin/out_sqs.rb', line 77

def formatted_to_msgpack_binary
  true
end

#multi_workers_ready? ⇒ Boolean

Returns:

  • (Boolean) 


81
82
83
 
# File 'lib/fluent/plugin/out_sqs.rb', line 81

def multi_workers_ready?
  true
end

#queue ⇒ Object 



50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
 
# File 'lib/fluent/plugin/out_sqs.rb', line 50

def queue
  return @queue if @queue

  begin
    sqs_url = client.get_queue_url(queue_name: @queue_name, queue_owner_aws_account_id: @queue_owner_aws_account_id).queue_url
  rescue Aws::SQS::Errors::NonExistentQueue => e
    if @create_queue
      sqs_url = client.create_queue(queue_name: @queue_name).queue_url
    else
      raise
    end
  end

  @queue = Aws::SQS::Queue.new(
    url: sqs_url,
    client: client
  )
  @queue
end

#write(chunk) ⇒ Object 



85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
 
# File 'lib/fluent/plugin/out_sqs.rb', line 85

def write(chunk)
  batch_records = []
  batch_size = 0
  send_batches = [batch_records]

  chunk.msgpack_each do |record|
    body = Yajl.dump(record)
    batch_size += body.bytesize

    if batch_size > SQS_BATCH_SEND_MAX_SIZE ||
       batch_records.length >= SQS_BATCH_SEND_MAX_MSGS
      batch_records = []
      batch_size = body.bytesize
      send_batches << batch_records
    end

    if batch_size > SQS_BATCH_SEND_MAX_SIZE
      log.warn 'Could not push message to SQS, payload exceeds ' \
               "#{SQS_BATCH_SEND_MAX_SIZE} bytes.  " \
               "(Truncated message: #{body[0..200]})"
    else
      id = "#{@tag_property_name}#{SecureRandom.hex(16)}"
      batch_record = { id: id, message_body: body, delay_seconds: @delay_seconds }
      batch_record[:message_group_id] = @message_group_id unless @message_group_id.nil?
      batch_records << batch_record
    end
  end

  until send_batches.length <= 0
    records = send_batches.shift
    until records.length <= 0
      queue.send_messages(entries: records.slice!(0..9))
    end
  end
end