Module: CrawlabRubySdk

Defined in:
lib/crawlab_ruby_sdk.rb,
lib/crawlab_ruby_sdk/version.rb

Defined Under Namespace

Classes: Error

Constant Summary collapse

VERSION = 
"0.2.6"

Class Method Summary collapse

Class Method Details

._save(sub_client, items = []) ⇒ Object 



102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
 
# File 'lib/crawlab_ruby_sdk.rb', line 102

def self._save(sub_client,items=[])
  records = []
  task_id = get_task_id
  if task_id == nil || task_id == ""
    return
  end

  items.each do |item|
    item["_tid"] = task_id
    records << item
  end

  data = {task_id: task_id,data:records}.to_json.b

  # puts "=====#{{task_id: task_id,data:records}.to_json}==="

  msg = Grpc::StreamMessage.new(code:3,data:data)

  sub_client.Send([msg]) 
end

.get_table_name(table_name = "") ⇒ Object 



139
140
141
142
143
144
145
146
147
 
# File 'lib/crawlab_ruby_sdk.rb', line 139

def self.get_table_name(table_name="")
  table_names = Verify.table_names
  if ENV["TABLE_NAME"] != nil && ENV["TABLE_NAME"] != "" && table_names.include?(ENV["TABLE_NAME"])
    table_name = ENV["TABLE_NAME"]
    return table_name
  end 

  return table_name
end

.get_task_id ⇒ Object 



123
124
125
126
127
128
129
 
# File 'lib/crawlab_ruby_sdk.rb', line 123

def self.get_task_id
  task_id = ENV["CRAWLAB_TASK_ID"]
  if task_id == nil || task_id == ""
    task_id = "6486e256fa1cb07a47c09adf"
  end
  return task_id
end

.save(sub_client, items = []) ⇒ Object 



85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/crawlab_ruby_sdk.rb', line 85

def self.save(sub_client,items = [])
  results = []

  items.each do |item|
    results << item

    if results.size >= 50
      _save(sub_client,results)
    end
    
  end

  if results.size > 0
    _save(sub_client,results)
  end
end

.save_file_stream_to_oss(oss_path, stream) ⇒ Object 



135
136
137
 
# File 'lib/crawlab_ruby_sdk.rb', line 135

def self.save_file_stream_to_oss(oss_path,stream)
  OssServerClient.new.send_stream(oss_path,stream)
end

.save_file_to_oss(oss_path, file_path) ⇒ Object 



131
132
133
 
# File 'lib/crawlab_ruby_sdk.rb', line 131

def self.save_file_to_oss(oss_path,file_path)
  OssServerClient.new.send(oss_path,file_path)
end

.save_item(item = {}, table_name = "", use_gzip = true) ⇒ Object 



39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/crawlab_ruby_sdk.rb', line 39

def self.save_item(item={},table_name="",use_gzip=true)
  address = ENV["CRAWLAB_GRPC_ADDRESS"]
  if address==nil || address == ""
    address = "localhost:9666"
  end  

  auth = ENV["CRAWLAB_GRPC_AUTH_KEY"]
  if auth==nil || auth == ""
    auth = "Crawlab2021!"
  end  

  table_name = get_table_name(table_name)
  if !Verify.IsVerified?([item],table_name)
    return
  end
  
  client = TaskServiceClient.new(address,auth,use_gzip)

  sub_client = client.subscribe

  save(sub_client,[item])
end

.save_items(items = [], table_name = "", use_gzip = true) ⇒ Object 



62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
 
# File 'lib/crawlab_ruby_sdk.rb', line 62

def self.save_items(items=[],table_name="",use_gzip=true)
  address = ENV["CRAWLAB_GRPC_ADDRESS"]
  if address==nil || address == ""
    address = "localhost:9666"
  end  

  auth = ENV["CRAWLAB_GRPC_AUTH_KEY"]
  if auth==nil || auth == ""
    auth = "Crawlab2021!"
  end  

  table_name = get_table_name(table_name)
  if !Verify.IsVerified?(items,table_name)
    return
  end    
  
  client = TaskServiceClient.new(address,auth,use_gzip)

  sub_client = client.subscribe

  save(sub_client,items)    
end