Class: HTML::Proofer
- Inherits:
-
Object
- Object
- HTML::Proofer
- Includes:
- Yell::Loggable
- Defined in:
- lib/html/proofer.rb,
lib/html/proofer/checks.rb,
lib/html/proofer/version.rb,
lib/html/proofer/checkable.rb
Defined Under Namespace
Constant Summary collapse
- VERSION =
"1.5.4"
Instance Attribute Summary collapse
-
#options ⇒ Object
readonly
Returns the value of attribute options.
-
#parallel_opts ⇒ Object
readonly
Returns the value of attribute parallel_opts.
-
#typhoeus_opts ⇒ Object
readonly
Returns the value of attribute typhoeus_opts.
Class Method Summary collapse
Instance Method Summary collapse
- #add_failed_tests(filenames, desc, status = nil) ⇒ Object
-
#external_link_checker(external_urls) ⇒ Object
the hypothesis is that Proofer runs way faster if we pull out all the external URLs and run the checks at the end.
- #failed_tests ⇒ Object
- #files ⇒ Object
- #get_checks ⇒ Object
- #has_hash?(url) ⇒ Boolean
- #hydra ⇒ Object
-
#initialize(src, opts = {}) ⇒ Proofer
constructor
A new instance of Proofer.
- #log_level ⇒ Object
- #queue_request(method, href, filenames) ⇒ Object
- #response_handler(response, filenames) ⇒ Object
- #run ⇒ Object
Constructor Details
#initialize(src, opts = {}) ⇒ Proofer
Returns a new instance of Proofer.
32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 |
# File 'lib/html/proofer.rb', line 32 def initialize(src, opts={}) @src = src @proofer_opts = { :ext => ".html", :favicon => false, :href_swap => [], :href_ignore => [], :check_external_hash => false, :alt_ignore => [], :disable_external => false, :verbose => false, :only_4xx => false, :directory_index_file => "index.html", :validate_html => false, :error_sort => :path } @typhoeus_opts = { :followlocation => true } # fall back to parallel defaults @parallel_opts = opts[:parallel] || {} opts.delete(:parallel) # Typhoeus won't let you pass in any non-Typhoeus option; if the option is not # a proofer_opt, it must be for Typhoeus opts.keys.each do |key| if @proofer_opts[key].nil? @typhoeus_opts[key] = opts[key] end end @options = @proofer_opts.merge(@typhoeus_opts).merge(opts) @failed_tests = [] Yell.new({ :format => false, :name => "HTML::Proofer", :level => "gte.#{log_level}" }) do |l| l.adapter :stdout, level: [:debug, :info, :warn] l.adapter :stderr, level: [:error, :fatal] end end |
Instance Attribute Details
#options ⇒ Object (readonly)
Returns the value of attribute options.
30 31 32 |
# File 'lib/html/proofer.rb', line 30 def @options end |
#parallel_opts ⇒ Object (readonly)
Returns the value of attribute parallel_opts.
30 31 32 |
# File 'lib/html/proofer.rb', line 30 def parallel_opts @parallel_opts end |
#typhoeus_opts ⇒ Object (readonly)
Returns the value of attribute typhoeus_opts.
30 31 32 |
# File 'lib/html/proofer.rb', line 30 def typhoeus_opts @typhoeus_opts end |
Class Method Details
.create_nokogiri(path) ⇒ Object
231 232 233 234 |
# File 'lib/html/proofer.rb', line 231 def self.create_nokogiri(path) content = File.open(path).read Nokogiri::HTML(content) end |
Instance Method Details
#add_failed_tests(filenames, desc, status = nil) ⇒ Object
255 256 257 258 259 260 261 262 263 |
# File 'lib/html/proofer.rb', line 255 def add_failed_tests(filenames, desc, status = nil) if filenames.nil? @failed_tests << Checks::Issue.new("", desc, status) elsif filenames.each { |f| @failed_tests << Checks::Issue.new(f, desc, status) } end end |
#external_link_checker(external_urls) ⇒ Object
the hypothesis is that Proofer runs way faster if we pull out all the external URLs and run the checks at the end. Otherwise, we’re halting the consuming process for every file. In addition, sorting the list lets libcurl keep connections to hosts alive. Finally, we’ll make a HEAD request, rather than GETing all the contents
152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 |
# File 'lib/html/proofer.rb', line 152 def external_link_checker(external_urls) external_urls = Hash[external_urls.sort] logger.info HTML::colorize :yellow, "Checking #{external_urls.length} external links..." Ethon.logger = logger # log from Typhoeus/Ethon external_urls.each_pair do |href, filenames| if has_hash? href && @options[:check_external_hash] queue_request(:get, href, filenames) else queue_request(:head, href, filenames) end end logger.debug HTML::colorize :yellow, "Running requests for all #{hydra.queued_requests.size} external URLs..." hydra.run end |
#failed_tests ⇒ Object
265 266 267 268 269 270 |
# File 'lib/html/proofer.rb', line 265 def failed_tests return [] if @failed_tests.empty? result = [] @failed_tests.each { |f| result << f.to_s } result end |
#files ⇒ Object
220 221 222 223 224 225 226 227 228 229 |
# File 'lib/html/proofer.rb', line 220 def files if File.directory? @src pattern = File.join @src, "**", "*#{@options[:ext]}" Dir.glob(pattern).select { |fn| File.file? fn } elsif File.extname(@src) == @options[:ext] [@src] else [] end end |
#get_checks ⇒ Object
236 237 238 239 240 241 |
# File 'lib/html/proofer.rb', line 236 def get_checks checks = HTML::Proofer::Checks::Check.subclasses.map { |c| c.name } checks.delete("Favicons") unless @options[:favicon] checks.delete("Html") unless @options[:validate_html] checks end |
#has_hash?(url) ⇒ Boolean
243 244 245 246 247 248 249 |
# File 'lib/html/proofer.rb', line 243 def has_hash?(url) begin URI.parse(url).fragment rescue URI::InvalidURIError nil end end |
#hydra ⇒ Object
216 217 218 |
# File 'lib/html/proofer.rb', line 216 def hydra @hydra ||= Typhoeus::Hydra.new end |
#log_level ⇒ Object
251 252 253 |
# File 'lib/html/proofer.rb', line 251 def log_level @options[:verbose] ? :debug : :info end |
#queue_request(method, href, filenames) ⇒ Object
170 171 172 173 174 |
# File 'lib/html/proofer.rb', line 170 def queue_request(method, href, filenames) request = Typhoeus::Request.new(href, @typhoeus_opts.merge({:method => method})) request.on_complete { |response| response_handler(response, filenames) } hydra.queue request end |
#response_handler(response, filenames) ⇒ Object
176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 |
# File 'lib/html/proofer.rb', line 176 def response_handler(response, filenames) effective_url = response.[:effective_url] href = response.request.base_url method = response.request.[:method] response_code = response.code debug_msg = "Received a #{response_code} for #{href}" debug_msg << " in #{filenames.join(' ')}" unless filenames.nil? logger.debug debug_msg if response_code.between?(200, 299) return if @options[:only_4xx] return unless @options[:check_external_hash] if hash = has_hash?(href) body_doc = Nokogiri::HTML(response.body) # user-content is a special addition by GitHub. xpath = %$//*[@name="#{hash}"]|//*[@id="#{hash}"]$ xpath << %$|//*[@name="user-content-#{hash}"]|//*[@id="user-content-#{hash}"]$ if URI.parse(href).host.match(/github\.com/i) if body_doc.xpath(xpath).empty? add_failed_tests filenames, "External link #{href} failed: #{effective_url} exists, but the hash '#{hash}' does not", response_code end end elsif response.timed_out? return if @options[:only_4xx] add_failed_tests filenames, "External link #{href} failed: got a time out", response_code elsif (response_code == 405 || response_code == 420 || response_code == 503) && method == :head # 420s usually come from rate limiting; let's ignore the query and try just the path with a GET uri = URI(href) queue_request(:get, uri.scheme + "://" + uri.host + uri.path, filenames) # just be lazy; perform an explicit get request. some servers are apparently not configured to # intercept HTTP HEAD elsif method == :head queue_request(:get, effective_url, filenames) else return if @options[:only_4xx] && !response_code.between?(400, 499) # Received a non-successful http response. add_failed_tests filenames, "External link #{href} failed: #{response_code} #{response.}", response_code end end |
#run ⇒ Object
76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 |
# File 'lib/html/proofer.rb', line 76 def run unless @src.is_a? Array external_urls = {} logger.info HTML::colorize :white, "Running #{get_checks} checks on #{@src} on *#{@options[:ext]}... \n\n" results = Parallel.map(files, @parallel_opts) do |path| html = HTML::Proofer.create_nokogiri(path) result = {:external_urls => {}, :failed_tests => []} get_checks.each do |klass| logger.debug HTML::colorize :blue, "Checking #{klass.to_s.downcase} on #{path} ..." check = Object.const_get(klass).new(@src, path, html, @options) check.run result[:external_urls].merge!(check.external_urls) result[:failed_tests].concat(check.issues) if check.issues.length > 0 end result end results.each do |item| external_urls.merge!(item[:external_urls]) @failed_tests.concat(item[:failed_tests]) end external_link_checker(external_urls) unless @options[:disable_external] logger.info HTML::colorize :green, "Ran on #{files.length} files!\n\n" else external_urls = Hash[*@src.map{ |s| [s, nil] }.flatten] external_link_checker(external_urls) unless @options[:disable_external] end if @failed_tests.empty? logger.info HTML::colorize :green, "HTML-Proofer finished successfully." else matcher = nil # always sort by the actual option, then path, to ensure consistent alphabetical (by filename) results @failed_tests = @failed_tests.sort do |a,b| comp = (a.send(@options[:error_sort]) <=> b.send(@options[:error_sort])) comp.zero? ? (a.path <=> b.path) : comp end @failed_tests.each do |issue| case @options[:error_sort] when :path if matcher != issue.path logger.error HTML::colorize :blue, "- #{issue.path}" matcher = issue.path end logger.error HTML::colorize :red, " * #{issue.desc}" when :desc if matcher != issue.desc logger.error HTML::colorize :blue, "- #{issue.desc}" matcher = issue.desc end logger.error HTML::colorize :red, " * #{issue.path}" when :status if matcher != issue.status logger.error HTML::colorize :blue, "- #{issue.status}" matcher = issue.status end logger.error HTML::colorize :red, " * #{issue.to_s}" end end raise HTML::colorize :red, "HTML-Proofer found #{@failed_tests.length} failures!" end end |