Class: DatahubHttpClient

Inherits:
Object
  • Object
show all
Defined in:
lib/fluent/plugin/datahub/datahub-http-client.rb

Overview

@author pholy.ht ##

@time 2016-03-24  ##

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(endpoint, access_id, access_key) ⇒ DatahubHttpClient

Returns a new instance of DatahubHttpClient.



19
20
21
22
23
 
# File 'lib/fluent/plugin/datahub/datahub-http-client.rb', line 19

def initialize(endpoint, access_id, access_key)
    @endpoint = endpoint
    @access_id = access_id
    @access_key = access_key
end

Instance Attribute Details

#access_id ⇒ Object

Returns the value of attribute access_id.



15
16
17
 
# File 'lib/fluent/plugin/datahub/datahub-http-client.rb', line 15

def access_id
  @access_id
end

#access_key ⇒ Object

Returns the value of attribute access_key.



16
17
18
 
# File 'lib/fluent/plugin/datahub/datahub-http-client.rb', line 16

def access_key
  @access_key
end

#endpoint ⇒ Object

Returns the value of attribute endpoint.



17
18
19
 
# File 'lib/fluent/plugin/datahub/datahub-http-client.rb', line 17

def endpoint
  @endpoint
end

Instance Method Details

#create_project(project_name, comment) ⇒ Object 



112
113
114
115
116
117
 
# File 'lib/fluent/plugin/datahub/datahub-http-client.rb', line 112

def create_project(project_name, comment)
    path = "/projects/" + project_name
    params = {}
    params["Comment"] = comment
    return send_request("POST", path, params)
end

#create_topic(project_name, topic_name, shard_count, lifecycle, record_type, record_schema, comment) ⇒ Object 



142
143
144
145
146
147
148
149
150
151
 
# File 'lib/fluent/plugin/datahub/datahub-http-client.rb', line 142

def create_topic(project_name, topic_name, shard_count, lifecycle, record_type, record_schema, comment)
    path = "/projects/" + project_name + "/topics/" + topic_name
    params = {}
    params["ShardCount"] = shard_count
    params["Lifecycle"] = lifecycle
    params["RecordType"] = record_type
    params["RecordSchema"] = JSON.generate(record_schema)
    params["Comment"] = comment
    return send_request("POST", path, params)
end

#delete_project(project_name) ⇒ Object

datahub目前未实现update操作 def update_project(project_name, comment)

path = "/projects/" + project_name
params = {}
params["Comment"] = comment
return send_request("PUT", path, params)

end



127
128
129
130
 
# File 'lib/fluent/plugin/datahub/datahub-http-client.rb', line 127

def delete_project(project_name)
    path = "/projects/" + project_name
    return send_request("DELETE", path)
end

#delete_topic(project_name, topic_name) ⇒ Object 



161
162
163
164
 
# File 'lib/fluent/plugin/datahub/datahub-http-client.rb', line 161

def delete_topic(project_name, topic_name)
    path = "/projects/" + project_name + "/topics/" + topic_name
    return send_request("DELETE", path)
end

#get_project(project_name) ⇒ Object 



107
108
109
110
 
# File 'lib/fluent/plugin/datahub/datahub-http-client.rb', line 107

def get_project(project_name)
    path = "/projects/" + project_name
    return send_request("GET", path)
end

#get_shard_cursor(project_name, topic_name, shard_id, offset = DateTime.now.strftime('%Q'), type = "LATEST") ⇒ Object 



201
202
203
204
205
206
207
208
 
# File 'lib/fluent/plugin/datahub/datahub-http-client.rb', line 201

def get_shard_cursor(project_name, topic_name, shard_id, offset=DateTime.now.strftime('%Q'), type="LATEST")
    path = "/projects/" + project_name + "/topics/" + topic_name + "/shards/" + shard_id
    params = {}
    params["Action"] = "cursor"
    params["SystemTime"] = offset
    params["Type"] = type
    return send_request("POST", path, params)
end

#get_signature(params) ⇒ Object

Signature = base64(HmacSha1(AccessKeySecret, VERB + “n”

+ CONTENT-TYPE + "\n"
+ DATE + "\n"
+ CanonicalizedDatahubHeaders + "\n"
+ CanonicalizedResource))



31
32
33
34
35
36
37
38
39
 
# File 'lib/fluent/plugin/datahub/datahub-http-client.rb', line 31

def get_signature(params)
    verb = params["verb"]
    content_type = params["content-type"]
    resource = params["resource"]
    gmt_time = params["date"]
    data = verb + "\n" + content_type + "\n" + gmt_time + "\n" + "x-datahub-client-version:1\n" + resource
    # data = "GET\napplication/json\nFri, 06 May 2016 06:43:31 GMT\nx-datahub-client-version:1\n/projects/test_project/topics/datahub_fluentd_out_1"
    return "DATAHUB " + @access_id + ":" + Base64.encode64("#{OpenSSL::HMAC.digest('sha1', @access_key, data)}").chomp
end

#get_topic(project_name, topic_name) ⇒ Object 



137
138
139
140
 
# File 'lib/fluent/plugin/datahub/datahub-http-client.rb', line 137

def get_topic(project_name, topic_name)
    path = "/projects/" + project_name + "/topics/" + topic_name
    return send_request("GET", path)
end

#list_projects ⇒ Object 



102
103
104
105
 
# File 'lib/fluent/plugin/datahub/datahub-http-client.rb', line 102

def list_projects()
    path = "/projects"
    return send_request("GET", path)
end

#list_shards(project_name, topic_name) ⇒ Object 



166
167
168
169
 
# File 'lib/fluent/plugin/datahub/datahub-http-client.rb', line 166

def list_shards(project_name, topic_name)
    path = "/projects/" + project_name + "/topics/" + topic_name + "/shards"
    return send_request("GET", path)
end

#list_topics(project_name) ⇒ Object 



132
133
134
135
 
# File 'lib/fluent/plugin/datahub/datahub-http-client.rb', line 132

def list_topics(project_name)
    path = "/projects/" + project_name + "/topics"
    return send_request("GET", path)
end

#read_data_from_shard(project_name, topic_name, shard_id, count, offset = DateTime.now.strftime('%Q'), type = "LATEST") ⇒ Object 



219
220
221
222
223
224
225
226
227
 
# File 'lib/fluent/plugin/datahub/datahub-http-client.rb', line 219

def read_data_from_shard(project_name, topic_name, shard_id, count, offset=DateTime.now.strftime('%Q'), type="LATEST")
    path = "/projects/" + project_name + "/topics/" + topic_name + "/shards/" + shard_id
    cursor = get_shard_cursor(project_name, topic_name, shard_id, offset, type).to_hash["Cursor"]
    params = {}
    params["Action"] = "sub"
    params["Cursor"] = cursor
    params["Limit"] = count
    return send_request("POST", path, params)
end

#read_data_from_shard_with_cursor(project_name, topic_name, shard_id, cursor, count) ⇒ Object 



210
211
212
213
214
215
216
217
 
# File 'lib/fluent/plugin/datahub/datahub-http-client.rb', line 210

def read_data_from_shard_with_cursor(project_name, topic_name, shard_id, cursor, count)
    path = "/projects/" + project_name + "/topics/" + topic_name + "/shards/" + shard_id
    params = {}
    params["Action"] = "sub"
    params["Cursor"] = cursor
    params["Limit"] = count
    return send_request("POST", path, params)
end

#send_request(method, path, params = {}, headers = {}) ⇒ Object 



41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/fluent/plugin/datahub/datahub-http-client.rb', line 41

def send_request(method, path, params={}, headers={})
    uri = URI(@endpoint + path)
    http_req = nil
    if method == "GET"
        http_req = Net::HTTP::Get.new(uri.path)
    elsif method == "POST"
        http_req = Net::HTTP::Post.new(uri.path)
        # start_time = Time.now.to_f
        # puts "start_time :" + start_time.to_s
        http_req.body = JSON.generate(params)
        # end_time = Time.now.to_f
        # puts "end_time   :" + end_time.to_s
        # puts ((end_time - start_time)*1000).to_s
    elsif method == "PUT"
        http_req = Net::HTTP::Put.new(uri.path)
        http_req.body = JSON.generate(params)
    elsif method == "DELETE"
        http_req = Net::HTTP::Delete.new(uri.path)
    else
        raise "Unsupported method: " + method
    end
    
    sig_params = {}
    sig_params["verb"] = method
    sig_params["content-type"] = "application/json" 
    sig_params["resource"] = path
    sig_params["date"] = Time.now.utc.strftime("%a, %d %b %Y %T") + " GMT"
    
    http_req["Authorization"] = get_signature(sig_params)
    http_req["x-datahub-client-version"] = 1
    http_req["Date"] = sig_params["date"]
    http_req["Content-Type"] = sig_params["content-type"]
    http_req["ACCEPT-ENCODING"] = nil
    
    # puts http_req["Date"]
    # puts http_req["Authorization"]
    # puts http_req.to_hash
    
    headers.each do |k, v|
        http_req[k] = v
    end
    
    # puts path
    
    return Net::HTTP.start(uri.host, uri.port) {
        |client|
        http_resp = client.request(http_req)
        if http_resp.code != '200' and http_resp.code != '201'
            if http_resp.body.empty?
                raise "Send request failed, unknown response " 
            end
            raise http_resp.body
        end
        result = http_resp.body
        if result.empty?
            return nil
        end
        return JSON.parse(result)
    }
end

#update_topic(project_name, topic_name, lifecycle, comment) ⇒ Object 



153
154
155
156
157
158
159
 
# File 'lib/fluent/plugin/datahub/datahub-http-client.rb', line 153

def update_topic(project_name, topic_name, lifecycle, comment)
    path = "/projects/" + project_name + "/topics/" + topic_name
    params = {}
    params["Lifecycle"] = lifecycle
    params["Comment"] = comment
    return send_request("PUT", path, params)
end

#write_data_to_topic(project_name, topic_name, record_entities) ⇒ Object

def split_shard(project_name, topic_name, shard_id, split_key)

path = "/projects/" + project_name + "/topics/" + topic_name + "/shards"
params = {}
params["Action"] = "split"
params["ShardId"] = shard_id
params["SplitKey"] = split_key
return send_request("POST", path, params)

end



193
194
195
196
197
198
199
 
# File 'lib/fluent/plugin/datahub/datahub-http-client.rb', line 193

def write_data_to_topic(project_name, topic_name, record_entities)
    path = "/projects/" + project_name + "/topics/" + topic_name + "/shards"
    params = {}
    params["Action"] = "pub"
    params["Records"] = record_entities
    return send_request("POST", path, params)
end