Top Level Namespace

Defined Under Namespace

Classes: DeadFinder, DeadFinderRunner, Logger

Constant Summary collapse

Channel = 
Concurrent::Channel
CACHE_SET = 
Concurrent::Map.new
CACHE_QUE = 
Concurrent::Map.new
OUTPUT = 
{}
VERSION = 
'1.5.0'

Instance Method Summary collapse

Instance Method Details

#extract_directory(uri) ⇒ Object 



30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/deadfinder/utils.rb', line 30

def extract_directory(uri)
  return "#{uri.scheme}://#{uri.host}#{uri.path}" if uri.path.end_with?('/')

  path_components = uri.path.split('/')
  path_components.last
  path_components.pop

  directory_path = path_components.join('/')

  if directory_path.start_with?('/')
    "#{uri.scheme}://#{uri.host}#{directory_path}/"
  else
    "#{uri.scheme}://#{uri.host}/#{directory_path}/"
  end
end

#gen_output(options) ⇒ Object 



190
191
192
193
 
# File 'lib/deadfinder.rb', line 190

def gen_output(options)
  output_data = OUTPUT.to_h
  File.write(options['output'], JSON.pretty_generate(output_data)) unless options['output'].empty?
end

#generate_url(text, base_url) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
 
# File 'lib/deadfinder/utils.rb', line 5

def generate_url(text, base_url)
  node = text.to_s
  begin
    unless node.start_with?('http://', 'https://')
      uri = URI(base_url)
      if node.start_with? '//'
        return "#{uri.scheme}:#{node}"
      elsif node.start_with? '/'
        return "#{uri.scheme}://#{uri.host}#{node}"
      elsif ignore_scheme? node
        return nil
      else
        return "#{extract_directory(uri)}#{node}"
      end
    end
  rescue StandardError
    # puts e
  end
  node
end

#ignore_scheme?(url) ⇒ Boolean

Returns:

  • (Boolean) 


26
27
28
 
# File 'lib/deadfinder/utils.rb', line 26

def ignore_scheme?(url)
  url.start_with?('mailto:', 'tel:', 'sms:', 'data:', 'file:')
end

#run_file(filename, options) ⇒ Object 



154
155
156
157
158
159
160
161
162
163
164
165
 
# File 'lib/deadfinder.rb', line 154

def run_file(filename, options)
  Logger.set_silent if options['silent']

  Logger.info "Reading: #{filename}"
  app = DeadFinderRunner.new
  File.foreach(filename) do |line|
    target = line.chomp
    Logger.target "Checking: #{target}"
    app.run target, options
  end
  gen_output(options)
end

#run_pipe(options) ⇒ Object 



141
142
143
144
145
146
147
148
149
150
151
152
 
# File 'lib/deadfinder.rb', line 141

def run_pipe(options)
  Logger.set_silent if options['silent']

  Logger.info 'Reading from STDIN'
  app = DeadFinderRunner.new
  while $stdin.gets
    target = $LAST_READ_LINE.chomp
    Logger.target "Checking: #{target}"
    app.run target, options
  end
  gen_output(options)
end

#run_sitemap(sitemap_url, options) ⇒ Object 



176
177
178
179
180
181
182
183
184
185
186
187
188
 
# File 'lib/deadfinder.rb', line 176

def run_sitemap(sitemap_url, options)
  Logger.set_silent if options['silent']
  Logger.info "Parsing sitemap: #{sitemap_url}"
  app = DeadFinderRunner.new
  base_uri = URI(sitemap_url)
  sitemap = SitemapParser.new sitemap_url, { recurse: true }
  sitemap.to_a.each do |url|
    turl = generate_url url, base_uri
    Logger.target "Checking: #{turl}"
    app.run turl, options
  end
  gen_output(options)
end

#run_url(url, options) ⇒ Object 



167
168
169
170
171
172
173
174
 
# File 'lib/deadfinder.rb', line 167

def run_url(url, options)
  Logger.set_silent if options['silent']

  Logger.target "Checking: #{url}"
  app = DeadFinderRunner.new
  app.run url, options
  gen_output(options)
end