Class: Fluent::Plugin::BosOutput

Inherits:
Output
  • Object
show all
Includes:
Baidubce
Defined in:
lib/fluent/plugin/out_bos.rb

Constant Summary collapse

DEFAULT_LINE_FORMAT_TYPE = 
'single_value'
DEFAULT_FORMAT_TYPE = 
'json'

Instance Attribute Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#formatter ⇒ Object

Returns the value of attribute formatter.



60
61
62
 
# File 'lib/fluent/plugin/out_bos.rb', line 60

def formatter
  @formatter
end

Instance Method Details

#configure(conf) ⇒ Object 



62
63
64
65
66
67
 
# File 'lib/fluent/plugin/out_bos.rb', line 62

def configure(conf)
  compat_parameters_convert(conf, :inject, :formatter) 
  super
  
  @formatter = formatter_create
end

#format(tag, time, record) ⇒ Object 



142
143
144
145
 
# File 'lib/fluent/plugin/out_bos.rb', line 142

def format(tag, time, record)
  record = inject_values_to_record(tag, time, record)
  @formatter.format(tag, time, record).chomp + "\n"
end

#multi_workers_ready? ⇒ Boolean

Returns:

  • (Boolean) 


56
57
58
 
# File 'lib/fluent/plugin/out_bos.rb', line 56

def multi_workers_ready?
  true
end

#prefer_buffered_processing ⇒ Object 



52
53
54
 
# File 'lib/fluent/plugin/out_bos.rb', line 52

def prefer_buffered_processing
  true
end

#send_tmpfile ⇒ Object 



89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
 
# File 'lib/fluent/plugin/out_bos.rb', line 89

def send_tmpfile
  while thread_current_running?
    if @part_number <= 100
        if @queue.size > 1
            to_send = @queue.pop
            tmpfile = to_send["file"]
            tmpfile.rewind
            s = tmpfile.read
            tmpfile.close
            tmpfile.unlink
            response = @client.upload_part(
                @bos_bucket, @part_prefix+@part_suffix.to_s, @upload_id, @part_number,s.size ,{})  do |buf_writer|
                       buf_writer << s
            end
            @part_list << {
              "partNumber" => @part_number,
              "eTag" => response['etag']
            }
            @part_number += 1 
        elsif @queue.size == 1
             to_send = @current_file
             if Time.now - to_send["time"] > 900
               @queue.pop
               tmpfile = to_send["file"]
               tmpfile.rewind
               s = tmpfile.read
               tmpfile.close
               tmpfile.unlink
               response = @client.upload_part(
                   @bos_bucket, @part_prefix+@part_suffix.to_s, @upload_id, @part_number,s.size ,{})  do |buf_writer|
                       buf_writer << s
               end
               @part_list << {
                 "partNumber" => @part_number,
                  "eTag" => response['etag']
               }
              @client.complete_multipart_upload(@bos_bucket, @part_prefix+@part_suffix.to_s, @upload_id, @part_list)
              @part_suffix += 1
              @upload_id = @client.initiate_multipart_upload(@bos_bucket, @part_prefix+@part_suffix.to_s)["uploadId"]
              @part_list = []
              @part_number = 1
             end
        end 
    elsif 
       @client.complete_multipart_upload(@bos_bucket, @part_prefix+@part_suffix.to_s, @upload_id, @part_list)
       @part_suffix += 1
       @upload_id = @client.initiate_multipart_upload(@bos_bucket, @part_prefix+@part_suffix.to_s)["uploadId"]
       @part_list = []
       @part_number = 1
    end 
  end
end

#start ⇒ Object 



69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
 
# File 'lib/fluent/plugin/out_bos.rb', line 69

def start 
  credentials = Auth::BceCredentials.new(
      @bce_key_id,
      @bce_sec_key
  )

  bosconf = BceClientConfiguration.new(
      credentials,
      @bos_endpoint
  )
  @client = Services::BosClient.new(bosconf)
  @part_suffix = 1
  @upload_id = @client.initiate_multipart_upload(@bos_bucket, @part_prefix+@part_suffix.to_s)["uploadId"]
  @part_number=1
  @part_list=[]
  @queue = Queue.new
  super
  thread_create(:send_tmpfile_to_bos, &method(:send_tmpfile))
end

#write(chunk) ⇒ Object 



147
148
149
150
151
152
153
154
155
156
157
158
159
 
# File 'lib/fluent/plugin/out_bos.rb', line 147

def write(chunk)
    if @current_tmpfile.nil? || @current_tmpfile.path.nil? || @current_tmpfile.size >= 5*1024*1024
    @current_tmpfile = Tempfile.new("bostmpfile")
    @current_file = {
      "time" => Time.now,
      "file" => @current_tmpfile
    }
    @queue.push(@current_file)
  end
  ss = chunk.read
  @current_tmpfile.write(ss)
  @current_tmpfile.size
end