Class: PcaprLocal::Xtractr

Inherits:
Object
  • Object
show all
Defined in:
lib/pcapr_local/xtractr.rb,
lib/pcapr_local/xtractr/instance.rb

Defined Under Namespace

Classes: Instance, XtractrError, XtractrIndexingException

Constant Summary collapse

EXE_PATH = 
File.join(ROOT, 'lib/exe/xtractr')
REAPER_INTERVAL =

Idle xtractr process reaper runs every REAPER_INTERVAL seconds.

10
RE_INDEXING_DONE =

Last line of normal xtractr indexing output.

/optimizing terms\.db\.\.\.done/

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(config) ⇒ Xtractr

Returns a new instance of Xtractr.



15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/pcapr_local/xtractr.rb', line 15

def initialize config
    @xtractr_path = EXE_PATH
    @idle_timeout = config.fetch("idle_timeout")
    @index_dir    = config.fetch("index_dir")
    @reaper_interval = config.fetch("reaper_interval", REAPER_INTERVAL)
    # Hash of index dir to xtractr instance.
    @xcache = {}
    # Lock to synchronize creation/destruction of xtractr instances.
    @xcache_lock = Mutex.new

    start_reaper
    at_exit do
        shutdown
    end 
end

Class Method Details

.index_dir?(path) ⇒ Boolean

Does path look like an xtractr index directory?

Returns:

  • (Boolean) 


76
77
78
 
# File 'lib/pcapr_local/xtractr.rb', line 76

def self.index_dir?(path)
    File.exist? File.join(path, 'packets.db')
end

.index_time(path) ⇒ Object

Returns timestamp (float) for xtractr index.



81
82
83
84
85
86
87
88
89
90
 
# File 'lib/pcapr_local/xtractr.rb', line 81

def self.index_time(path)
    db_file = File.join(path, 'packets.db')
    if File.exists? db_file
        File.mtime(db_file).to_f
    elsif File.exists? path
        File.mtime(path).to_f
    else
        0.0
    end
end

Instance Method Details

#get(index_dir, url) ⇒ Object

Forwards GET request to xtractr instance created for index_dir. A relative path will be expanded relative to the configured index_dir.



144
145
146
147
148
 
# File 'lib/pcapr_local/xtractr.rb', line 144

def get index_dir, url
    xtractr = nil
    xtractr = xtractr_for index_dir
    xtractr.get url
end

#get_summary(index_dir) ⇒ Object 



128
129
130
131
132
133
134
135
136
137
138
139
140
 
# File 'lib/pcapr_local/xtractr.rb', line 128

def get_summary index_dir
    # Start xtractr in browse mode and get summary data
    browser = Instance.new index_dir, @xtractr_path
    about = JSON.parse(browser.get('api/about')[2])
    services = JSON.parse(browser.get('api/services')[2])
    service_names = []
    services["rows"].each do |row|
        service_names << row['name'].downcase
    end
    return { :about => about, :services => service_names }
ensure
    browser.stop if browser
end

#index(pcap_path, index_dir) ⇒ Object

Indexes pcap it index_dir and returns hash containing xtractr summary data. Raises exception if indexing fails.



99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/pcapr_local/xtractr.rb', line 99

def index pcap_path, index_dir
    FileUtils.mkdir_p index_dir

    command = [@xtractr_path, 'index', index_dir, '--mode', 'forensics', pcap_path]
    Logger.debug "running: #{command.inspect}"
    xtractr_out = Tempfile.new "xtractr_out"
    pid = fork do
        # Xtractr forks a child process. Set process group so we
        # can send signal to all processes in a group.
        STDOUT.reopen xtractr_out
        STDERR.reopen xtractr_out
        exec *command 
    end
    #XXX enforce timeout.
    Process.wait pid 

    xtractr_out.rewind
    output = xtractr_out.read

    unless $?.exitstatus == 0 and Xtractr.index_dir?(index_dir) and output =~ RE_INDEXING_DONE
        Logger.error "Indexing failed with output:\n" + output
        raise XtractrIndexingException, "Indexing failed"
    end

    return get_summary(index_dir)
ensure
    xtractr_out.close! if xtractr_out
end

#post(index_dir, url, body) ⇒ Object

Forwards POST request to xtractr instance created for index_dir. A relative path will be expanded relative to the configured index_dir.



152
153
154
155
156
 
# File 'lib/pcapr_local/xtractr.rb', line 152

def post index_dir, url, body
    xtractr = nil
    xtractr = xtractr_for index_dir
    xtractr.post url, body
end

#reap ⇒ Object

Kills idle xtractr processes



57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/pcapr_local/xtractr.rb', line 57

def reap
    @xcache_lock.synchronize do 
        @xcache.to_a.each do |dir, xtractr|
            if xtractr.lock.try_lock
                # xtractr is not in use right now
                begin
                    if xtractr.last_use + @idle_timeout < Time.new.to_f
                        xtractr.stop
                        @xcache.delete dir
                    end
                ensure
                    xtractr.lock.unlock
                end
            end
        end
    end
end

#shutdown ⇒ Object

Kill xtractr instances at exit (idle or not).



49
50
51
52
53
54
 
# File 'lib/pcapr_local/xtractr.rb', line 49

def shutdown
    @xcache.each_value do |xtractr|
        $stderr.puts "stopping xtractr process"
        xtractr.stop rescue nil
    end
end

#start_reaper ⇒ Object

Start reaper thread.



35
36
37
38
39
40
41
42
43
44
45
46
 
# File 'lib/pcapr_local/xtractr.rb', line 35

def start_reaper
    Thread.new do
        loop do
            begin
                reap
            rescue Exception
                Logger.error "Exception while cleaning up idle processes: #{e.message}\n" + e.backtrace.join("\n")
            end
            sleep @reaper_interval
        end
    end
end

#xtractr_for(index_dir) ⇒ Object 



158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
 
# File 'lib/pcapr_local/xtractr.rb', line 158

def xtractr_for index_dir
    # Ensure index_dir path is absolute.
    if index_dir.slice(0,1) != '/'
        index_dir = File.expand_path(File.join(@index_dir, index_dir))
    end

    # Get or create xtractr instance.
    @xcache_lock.synchronize do
        xtractr = @xcache[index_dir]
        if not xtractr
            xtractr = Instance.new(index_dir, @xtractr_path)
            @xcache[index_dir] = xtractr
        end
        return xtractr
    end
end