Class: Knj::Process

Inherits:
Object show all
Defined in:
lib/knj/process.rb

Defined Under Namespace

Classes: Resultobject

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(args = {}) ⇒ Process

Constructor. Sets in, out and various other needed variables.



8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'lib/knj/process.rb', line 8

def initialize(args = {})
  @args = args
  @in = @args[:in]
  @out = @args[:out]
  @on_rec = @args[:on_rec]
  @in_count = 0
  @out_count = 0
  @out_answers = {}
  @out_mutex = Mutex.new
  @debug = @args[:debug]
  @args[:sleep_answer] = 0.001 if !@args.key?(:sleep_answer)
  @thread_error = nil
  
  #Used when this process is trying to receive block-results from the subprocess.
  @blocks = {}
  
  #Used when the other process is trying to receive block-results from this object.
  @blocks_send = {}
  
  #Else the sockets might hang when waiting for results and stuff like that... - knj.
  @in.sync = true
  @out.sync = true
  
  if @args[:err]
    @err_thread = Knj::Thread.new do
      @args[:err].each_line do |str|
        if @args[:on_err]
          @args[:on_err].call(str)
        else
          $stderr.print "Process error: #{str}"
        end
      end
    end
  end
  
  if @args[:listen]
    require "#{$knjpath}/thread"
    @listen_thread = Knj::Thread.new do
      begin
        self.listen
      rescue Exception => e
        $stderr.print "#{Knj::Errors.error_str(e)}\n\n" if @debug
        @thread_error = e
      end
    end
  end
end

Instance Attribute Details

#blocks ⇒ Object (readonly)

Returns the value of attribute blocks.



5
6
7
 
# File 'lib/knj/process.rb', line 5

def blocks
  @blocks
end

#blocks_send ⇒ Object (readonly)

Returns the value of attribute blocks_send.



5
6
7
 
# File 'lib/knj/process.rb', line 5

def blocks_send
  @blocks_send
end

Instance Method Details

#answer(id, obj, type = "answer") ⇒ Object

Answers a send.



272
273
274
275
276
277
278
279
 
# File 'lib/knj/process.rb', line 272

def answer(id, obj, type = "answer")
  $stderr.print "Answering #{id} (#{obj}).\n" if @debug
  str = Marshal.dump(obj)
  
  @out_mutex.synchronize do
    @out.write("#{type}:#{id}:#{str.length}\n#{str}")
  end
end

#destroy ⇒ Object

Closes everything down…



418
419
420
421
 
# File 'lib/knj/process.rb', line 418

def destroy
  self.kill_listen
  @err_thread.kill if @err_thread
end

#exec_block_results(id) ⇒ Object

This is used to call the block with results and remove the results from the array.



357
358
359
360
361
362
363
364
365
366
367
368
369
370
 
# File 'lib/knj/process.rb', line 357

def exec_block_results(id)
  return nil if @blocks[id][:results].empty?
  
  removes = []
  @blocks[id][:mutex].synchronize do
    results = @blocks[id][:results]
    @blocks[id][:results] = []
    
    results.each do |res|
      removes << res
      @blocks[id][:block].call(res)
    end
  end
end

#has_answer?(id) ⇒ Boolean

Returns true if an answer with a certain ID has arrived.

Returns:

  • (Boolean) 


352
353
354
 
# File 'lib/knj/process.rb', line 352

def has_answer?(id)
  return @out_answers.key?(id)
end

#join ⇒ Object

Raises:

  • (@thread_error) 


60
61
62
63
64
 
# File 'lib/knj/process.rb', line 60

def join
  @listen_thread.join if @listen_thread
  sleep 0.5
  raise @thread_error if @thread_error
end

#kill_listen ⇒ Object 



56
57
58
 
# File 'lib/knj/process.rb', line 56

def kill_listen
  @listen_thread.kill if @listen_thread
end

#listen ⇒ Object

Listens for a new incoming object.



67
68
69
70
71
 
# File 'lib/knj/process.rb', line 67

def listen
  loop do
    self.listen_loop
  end
end

#listen_loop ⇒ Object 



73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
 
# File 'lib/knj/process.rb', line 73

def listen_loop
  str = @in.gets("\n")
  if str == nil
    raise "Socket closed." if @in.closed?
    sleep 0.1
    return nil
  end
  
  
  data = str.strip.split(":")
  raise "Expected length of 2 or 3 but got: '#{data.length}'.\n#{data}" if data.length != 2 and data.length != 3
  
  raise "Invalid ID: '#{data[1]}'." if data[1].to_s.strip.length <= 0
  id = data[1].to_i
  
  raise "Invalid length: '#{data[2]}' (#{str.to_s.strip})." if data[2].to_s.strip.length <= 0
  length = data[2].to_i
  
  $stderr.print "Received ID #{id}.\n" if @debug
  res = @in.read(length)
  
  begin
    obj = Marshal.load(res)
    $stderr.print "Got content for '#{id}' (#{data[0]}).\n" if @debug
    
    case data[0]
      when "answer"
        #raise "Already have answer for '#{id}'." if @out_answers.key?(id)
        @out_answers[id] = obj
      when "answer_block"
        @blocks[id][:mutex].synchronize do
          @blocks[id][:results] += obj
        end
      when "answer_block_end"
        $stderr.print "Answer-block-end received!\n" if @debug
        @blocks[id][:block_result] = obj
        @blocks[id][:finished] = true
      when "send"
        Knj::Thread.new do
          result_obj = Knj::Process::Resultobject.new(:process => self, :id => id, :obj => obj)
          
          begin
            @on_rec.call(result_obj)
          rescue SystemExit => e
            raise e
          rescue Exception => e
            #Error was raised - try to forward it to the server.
            result_obj.answer("type" => "process_error", "class" => e.class.name, "msg" => e.message, "backtrace" => e.backtrace)
          end
        end
      when "send_block"
        result_obj = Knj::Process::Resultobject.new(:process => self, :id => id, :obj => obj)
        @blocks_send[id] = {:result_obj => result_obj, :waiting_for_result => false}
        
        @blocks_send[id][:enum] = Enumerator.new do |y|
          @on_rec.call(result_obj) do |answer_block|
            $stderr.print "Running enum-block for #{answer_block}\n" if @debug
            
            break if !@blocks_send.key?(id)
            y << answer_block
            dobreak = false
            
            loop do
              if !@blocks_send.key?(id)
                dobreak = true
                break
              end
              
              break if @blocks_send[id][:waiting_for_result]
              sleep 0.01
            end
            
            break if dobreak
          end
        end
      when "send_block_res"
        begin
          @blocks_send[obj][:waiting_for_result] = true
          res = @blocks_send[obj][:enum].next
          self.answer(id, {"result" => res})
        rescue StopIteration
          self.answer(id, "StopIteration")
        end
      when "send_block_end"
        if @blocks_send.key?(obj)
          enum = @blocks_send[obj][:enum]
          @blocks_send.delete(obj)
          
          begin
            enum.next if enum #this has to be called to stop Enumerator from blocking...
          rescue StopIteration
            #do nothing.
          end
        end
        
        self.answer(id, "ok")
      when "send_block_buffer"
        buffer_use = true
        
        Knj::Thread.new do
          result_obj = Knj::Process::Resultobject.new(:process => self, :id => id, :obj => obj)
          block_res = nil
          
          begin
            if buffer_use
              buffer_answers = []
              buffer_done = false
              
              buffer_thread = Knj::Thread.new do
                loop do
                  arr = buffer_answers.shift(200)
                  
                  if !arr.empty?
                    $stderr.print "Sending: #{arr.length} results.\n" if @debug
                    self.answer(id, arr, "answer_block")
                  else
                    sleep 0.05
                  end
                  
                  break if buffer_done and buffer_answers.empty?
                end
              end
            end
            
            begin
              begin
                count = 0
                block_res = @on_rec.call(result_obj) do |answer_block|
                  if buffer_use
                    loop do
                      if buffer_answers.length > 1000
                        $stderr.print "Buffer is more than 1000 - sleeping and tries again in 0.05 sec.\n" if @debug
                        sleep 0.05
                      else
                        break
                      end
                    end
                  end
                  
                  count += 1
                  if buffer_use
                    buffer_answers << answer_block
                  else
                    self.answer(id, [answer_block], "answer_block")
                  end
                  
                  if count >= 100
                    count = 0
                    
                    loop do
                      answer = self.send("obj" => id, "type" => "send_block_count")
                      $stderr.print "Answer was: #{answer}\n" if @debug
                      
                      if answer >= 100
                        $stderr.print "More than 100 results are buffered on the other side - wait for them to be handeled before sending more.\n" if @debug
                        sleep 0.05
                      else
                        $stderr.print "Less than 100 results in buffer - send more.\n" if @debug
                        break
                      end
                    end
                  end
                end
              ensure
                buffer_done = true if buffer_use
              end
            ensure
              buffer_thread.join if buffer_use
            end
          rescue Exception => e
            $stderr.print Knj::Errors.error_str(e) if @debug
            #Error was raised - try to forward it to the server.
            result_obj.answer("type" => "process_error", "class" => e.class.name, "msg" => e.message, "backtrace" => e.backtrace)
          ensure
            $stderr.print "Answering with block-end.\n" if @debug
            self.answer(id, block_res, "answer_block_end")
          end
        end
      when "send_block_count"
        if @blocks.key?(obj)
          count = @blocks[obj][:results].length
        else
          count = 0
        end
        
        self.answer(id, count)
      else
        $stderr.print "Unknown command: '#{data[0]}'."
        raise "Unknown command: '#{data[0]}'."
    end
  rescue Exception => e
    $stderr.print Knj::Errors.error_str(e) if @debug
    #Error was raised - try to forward it to the server.
    result_obj = Knj::Process::Resultobject.new(:process => self, :id => id, :obj => obj)
    result_obj.answer("type" => "process_error", "class" => e.class.name, "msg" => e.message, "backtrace" => e.backtrace)
  end
end

#read_answer(id) ⇒ Object

Waits for data with a certain ID and returns it when it exists.



373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
 
# File 'lib/knj/process.rb', line 373

def read_answer(id)
  $stderr.print "Reading answer (#{id}).\n" if @debug
  block_res = @blocks[id]
  $stderr.print "Answer is block for #{id} #{block_res}\n" if @debug and block_res
  
  loop do
    if block_res
      self.exec_block_results(id)
      break if block_res and block_res[:finished]
    else
      break if @out_answers.key?(id)
    end
    
    sleep @args[:sleep_answer]
  end
  
  if block_res
    self.exec_block_results(id)
    @blocks.delete(id)
  end
  
  ret = @out_answers[id]
  @out_answers.delete(id)
  
  if ret.is_a?(Hash) and ret["type"] == "process_error"
    $stderr.print "Process-error - begin generating error...\n" if @debug
    err = RuntimeError.new(ret["msg"])
    bt = []
    ret["backtrace"].each do |subproc_bt|
      bt << "Subprocess: #{subproc_bt}"
    end
    
    caller.each do |proc_bt|
      bt << proc_bt
    end
    
    err.set_backtrace(bt)
    raise err
  end
  
  $stderr.print "Returning answer (#{id}).\n" if @debug
  return ret
end

#send(args, &block) ⇒ Object

Sends a command to the client.



282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
 
# File 'lib/knj/process.rb', line 282

def send(args, &block)
  args = {"obj" => args} if !args.is_a?(Hash)
  
  my_id = nil
  raise "No 'obj' was given." if !args["obj"]
  str = Marshal.dump(args["obj"])
  
  if args.key?("type")
    type = args["type"]
  else
    type = "send"
  end
  
  raise "Invalid type: '#{type}'." if type.to_s.strip.length <= 0
  args["wait_for_answer"] = true if !args.key?("wait_for_answer")
  
  @out_mutex.synchronize do
    my_id = @out_count
    @out_count += 1
    
    if block
      if type == "send"
        if args["buffer_use"]
          type = "send_block_buffer"
          @blocks[my_id] = {:block => block, :results => [], :finished => false, :buffer => args["buffer_use"], :mutex => Mutex.new}
        else
          type = "send_block"
        end
      end
    end
    
    $stderr.print "Writing #{type}:#{my_id}:#{args["obj"]} to socket.\n" if @debug
    @out.write("#{type}:#{my_id}:#{str.length}\n#{str}")
  end
  
  #If block is broken it might never give us control to return anything - thats why we use ensure.
  begin
    if type == "send_block"
      loop do
        res = self.send("obj" => my_id, "type" => "send_block_res")
        
        if res == "StopIteration"
          break
        elsif res.is_a?(Hash) and res.key?("result")
          #do nothing.
        else
          raise "Unknown result: '#{res}'."
        end
        
        block.call(res["result"])
      end
    end
  ensure
    #Tell the subprocess we are done with the block (if break, exceptions or anything else like that was used).
    if type == "send_block"
      res = self.send("obj" => my_id, "type" => "send_block_end")
      raise "Unknown result: '#{res}'." if res != "ok"
    end
    
    if args["wait_for_answer"]
      #Make very, very short sleep, if the result is almost instant this will heavily optimize the speed, because :sleep_answer-argument wont be used.
      sleep 0.00001
      return self.read_answer(my_id)
    end
    
    return {:id => my_id}
  end
end