Class: Arachni::Data::Framework

Inherits:
Object
  • Object
show all
Defined in:
lib/arachni/data/framework.rb,
lib/arachni/data/framework/rpc.rb

Overview

Data for Framework.

Author:

Defined Under Namespace

Classes: Error, RPC

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initializeFramework

Returns a new instance of Framework.


46
47
48
49
50
51
52
53
54
55
56
57
58
# File 'lib/arachni/data/framework.rb', line 46

def initialize
    @rpc = RPC.new

    @sitemap = {}

    @page_queue            = Support::Database::Queue.new
    @page_queue.max_buffer_size = 10
    @page_queue_total_size = 0

    @url_queue                 = Support::Database::Queue.new
    @url_queue.max_buffer_size = Float::INFINITY
    @url_queue_total_size      = 0
end

Instance Attribute Details

#page_queueSupport::Database::Queue (readonly)


35
36
37
# File 'lib/arachni/data/framework.rb', line 35

def page_queue
  @page_queue
end

#page_queue_total_sizeInteger

Returns:

  • (Integer)

38
39
40
# File 'lib/arachni/data/framework.rb', line 38

def page_queue_total_size
  @page_queue_total_size
end

#rpcRPC

Returns:


28
29
30
# File 'lib/arachni/data/framework.rb', line 28

def rpc
  @rpc
end

#sitemapHash<String, Integer> (readonly)

Returns List of crawled URLs with their HTTP codes.

Returns:

  • (Hash<String, Integer>)

    List of crawled URLs with their HTTP codes.


32
33
34
# File 'lib/arachni/data/framework.rb', line 32

def sitemap
  @sitemap
end

#url_queueSupport::Database::Queue (readonly)


41
42
43
# File 'lib/arachni/data/framework.rb', line 41

def url_queue
  @url_queue
end

#url_queue_total_sizeInteger

Returns:

  • (Integer)

44
45
46
# File 'lib/arachni/data/framework.rb', line 44

def url_queue_total_size
  @url_queue_total_size
end

Class Method Details

.load(directory) ⇒ Object


136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
# File 'lib/arachni/data/framework.rb', line 136

def self.load( directory )
    framework = new

    framework.rpc = RPC.load( "#{directory}/rpc/" )
    framework.sitemap.merge! Marshal.load( IO.binread( "#{directory}/sitemap" ) )

    Dir["#{directory}/page_queue/*"].each do |page_file|
        framework.page_queue.disk << page_file
    end

    Marshal.load( IO.binread( "#{directory}/url_queue" ) ).each do |url|
        framework.url_queue.buffer << url
    end

    framework.page_queue_total_size =
        Marshal.load( IO.binread( "#{directory}/page_queue_total_size" ) )
    framework.url_queue_total_size =
        Marshal.load( IO.binread( "#{directory}/url_queue_total_size" ) )

    framework
end

Instance Method Details

#add_page_to_sitemap(page) ⇒ Object

Parameters:


92
93
94
# File 'lib/arachni/data/framework.rb', line 92

def add_page_to_sitemap( page )
    update_sitemap( page.dom.url => page.code )
end

#clearObject


158
159
160
161
162
163
164
165
166
167
168
# File 'lib/arachni/data/framework.rb', line 158

def clear
    rpc.clear

    @sitemap.clear

    @page_queue.clear
    @page_queue_total_size = 0

    @url_queue.clear
    @url_queue_total_size = 0
end

#dump(directory) ⇒ Object


108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
# File 'lib/arachni/data/framework.rb', line 108

def dump( directory )
    FileUtils.mkdir_p( directory )

    rpc.dump( "#{directory}/rpc/" )

    page_queue_directory = "#{directory}/page_queue/"

    FileUtils.rm_rf( page_queue_directory )
    FileUtils.mkdir_p( page_queue_directory )

    page_queue.buffer.each do |page|
        IO.binwrite(
            "#{page_queue_directory}/#{page.persistent_hash}",
             Marshal.dump( page )
        )
    end

    page_queue.disk.each do |filepath|
        FileUtils.cp filepath, "#{page_queue_directory}/"
    end

    IO.binwrite( "#{directory}/url_queue", Marshal.dump( @url_queue.buffer ) )

    %w(sitemap page_queue_total_size url_queue_total_size).each do |attribute|
        IO.binwrite( "#{directory}/#{attribute}", Marshal.dump( send(attribute) ) )
    end
end

#push_to_page_queue(page) ⇒ Object

Note:

Increases the #page_queue_total_size.

Parameters:


75
76
77
78
79
# File 'lib/arachni/data/framework.rb', line 75

def push_to_page_queue( page )
    @page_queue << page.clear_cache
    add_page_to_sitemap( page )
    @page_queue_total_size += 1
end

#push_to_url_queue(url) ⇒ Object

Note:

Increases the #url_queue_total_size.

Parameters:


85
86
87
88
# File 'lib/arachni/data/framework.rb', line 85

def push_to_url_queue( url )
    @url_queue << url
    @url_queue_total_size += 1
end

#statisticsObject


60
61
62
63
64
65
66
67
68
69
# File 'lib/arachni/data/framework.rb', line 60

def statistics
    {
        rpc:                   @rpc.statistics,
        sitemap:               @sitemap.size,
        page_queue:            @page_queue.size,
        page_queue_total_size: @page_queue_total_size,
        url_queue:             @url_queue.size,
        url_queue_total_size:  @url_queue_total_size
    }
end

#update_sitemap(entries) ⇒ Object


96
97
98
99
100
101
102
103
104
105
106
# File 'lib/arachni/data/framework.rb', line 96

def update_sitemap( entries )
    entries.each do |url, code|
        # Feedback from the trainer or whatever, don't include it in the
        # sitemap, it'll just add noise.
        next if url.include?( Utilities.random_seed )

        @sitemap[url] = code
    end

    @sitemap
end