Class: Arachni::Data::Framework

Inherits:
Object
  • Object
show all
Defined in:
lib/arachni/data/framework.rb,
lib/arachni/data/framework/rpc.rb

Overview

Data for Framework.

Author:

Defined Under Namespace

Classes: Error, RPC

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Framework

Returns a new instance of Framework.



46
47
48
49
50
51
52
53
54
55
56
57
58
 
# File 'lib/arachni/data/framework.rb', line 46

def initialize
    @rpc = RPC.new

    @sitemap = {}

    @page_queue            = Support::Database::Queue.new
    @page_queue.max_buffer_size = 10
    @page_queue_total_size = 0

    @url_queue                 = Support::Database::Queue.new
    @url_queue.max_buffer_size = Float::INFINITY
    @url_queue_total_size      = 0
end

Instance Attribute Details

#page_queue ⇒ Support::Database::Queue (readonly)

Returns:



35
36
37
 
# File 'lib/arachni/data/framework.rb', line 35

def page_queue
  @page_queue
end

#page_queue_total_size ⇒ Integer

Returns:

  • (Integer) 


38
39
40
 
# File 'lib/arachni/data/framework.rb', line 38

def page_queue_total_size
  @page_queue_total_size
end

#rpc ⇒ RPC

Returns:



28
29
30
 
# File 'lib/arachni/data/framework.rb', line 28

def rpc
  @rpc
end

#sitemap ⇒ Hash<String, Integer> (readonly)

Returns List of crawled URLs with their HTTP codes.

Returns:

  • (Hash<String, Integer>) —

    List of crawled URLs with their HTTP codes.



32
33
34
 
# File 'lib/arachni/data/framework.rb', line 32

def sitemap
  @sitemap
end

#url_queue ⇒ Support::Database::Queue (readonly)

Returns:



41
42
43
 
# File 'lib/arachni/data/framework.rb', line 41

def url_queue
  @url_queue
end

#url_queue_total_size ⇒ Integer

Returns:

  • (Integer) 


44
45
46
 
# File 'lib/arachni/data/framework.rb', line 44

def url_queue_total_size
  @url_queue_total_size
end

Class Method Details

.load(directory) ⇒ Object 



136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
 
# File 'lib/arachni/data/framework.rb', line 136

def self.load( directory )
    framework = new

    framework.rpc = RPC.load( "#{directory}/rpc/" )
    framework.sitemap.merge! Marshal.load( IO.binread( "#{directory}/sitemap" ) )

    Dir["#{directory}/page_queue/*"].each do |page_file|
        framework.page_queue.disk << page_file
    end

    Marshal.load( IO.binread( "#{directory}/url_queue" ) ).each do |url|
        framework.url_queue.buffer << url
    end

    framework.page_queue_total_size =
        Marshal.load( IO.binread( "#{directory}/page_queue_total_size" ) )
    framework.url_queue_total_size =
        Marshal.load( IO.binread( "#{directory}/url_queue_total_size" ) )

    framework
end

Instance Method Details

#add_page_to_sitemap(page) ⇒ Object

Parameters:



92
93
94
 
# File 'lib/arachni/data/framework.rb', line 92

def add_page_to_sitemap( page )
    update_sitemap( page.dom.url => page.code )
end

#clear ⇒ Object 



158
159
160
161
162
163
164
165
166
167
168
 
# File 'lib/arachni/data/framework.rb', line 158

def clear
    rpc.clear

    @sitemap.clear

    @page_queue.clear
    @page_queue_total_size = 0

    @url_queue.clear
    @url_queue_total_size = 0
end

#dump(directory) ⇒ Object 



108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
 
# File 'lib/arachni/data/framework.rb', line 108

def dump( directory )
    FileUtils.mkdir_p( directory )

    rpc.dump( "#{directory}/rpc/" )

    page_queue_directory = "#{directory}/page_queue/"

    FileUtils.rm_rf( page_queue_directory )
    FileUtils.mkdir_p( page_queue_directory )

    page_queue.buffer.each do |page|
        IO.binwrite(
            "#{page_queue_directory}/#{page.persistent_hash}",
            page_queue.serialize( page )
        )
    end

    page_queue.disk.each do |filepath|
        FileUtils.cp filepath, "#{page_queue_directory}/"
    end

    IO.binwrite( "#{directory}/url_queue", Marshal.dump( @url_queue.buffer ) )

    %w(sitemap page_queue_total_size url_queue_total_size).each do |attribute|
        IO.binwrite( "#{directory}/#{attribute}", Marshal.dump( send(attribute) ) )
    end
end

#push_to_page_queue(page) ⇒ Object

Note:

Increases the #page_queue_total_size.

Parameters:



75
76
77
78
79
 
# File 'lib/arachni/data/framework.rb', line 75

def push_to_page_queue( page )
    @page_queue << page.clear_cache
    add_page_to_sitemap( page )
    @page_queue_total_size += 1
end

#push_to_url_queue(url) ⇒ Object

Note:

Increases the #url_queue_total_size.

Parameters:



85
86
87
88
 
# File 'lib/arachni/data/framework.rb', line 85

def push_to_url_queue( url )
    @url_queue << url
    @url_queue_total_size += 1
end

#statistics ⇒ Object 



60
61
62
63
64
65
66
67
68
69
 
# File 'lib/arachni/data/framework.rb', line 60

def statistics
    {
        rpc:                   @rpc.statistics,
        sitemap:               @sitemap.size,
        page_queue:            @page_queue.size,
        page_queue_total_size: @page_queue_total_size,
        url_queue:             @url_queue.size,
        url_queue_total_size:  @url_queue_total_size
    }
end

#update_sitemap(entries) ⇒ Object 



96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/arachni/data/framework.rb', line 96

def update_sitemap( entries )
    entries.each do |url, code|
        # Feedback from the trainer or whatever, don't include it in the
        # sitemap, it'll just add noise.
        next if url.include?( Utilities.random_seed )

        @sitemap[url] = code
    end

    @sitemap
end