Class: Webget::Mirror

Inherits:
Object
  • Object
show all
Defined in:
lib/webget/mirror.rb,
lib/webget/mirror/utils.rb,
lib/webget/mirror/mirror.rb,
lib/webget/mirror/version.rb,
lib/webget/mirror/website.rb,
lib/webget/mirror/find_links.rb,
lib/webget/mirror/download_page.rb,
lib/webget/mirror/collect_page_info.rb

Defined Under Namespace

Classes: Website

Constant Summary collapse

MAJOR =

todo: namespace inside version or something - why? why not??

0
MINOR = 
0
PATCH = 
1
VERSION = 
[MAJOR,MINOR,PATCH].join('.')
HTML_CHARSET_RE = 
%r{
   <meta [ ]+
       [^<>]*?        ## note - use non-greedy (shortest) match
  \bcharset
        [ ]*=[ ]*
          ['"]?       ## optional opening quote
        (?<charset>[a-z0-9_-]+)
}ix
HTML_DOCTYPE_RE = 
%r{
   <!DOCTYPE [ ]+
        (?<doctype> [^<>]+?)  ## note - use non-greedy (shortest) match
                               ## do NOT allow opening/closing brackets for now
                               ##  ever possible? double check
            [ ]*
   >
}ix

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

version string for generator meta tag (includes ruby version)



14
15
16
 
# File 'lib/webget/mirror/version.rb', line 14

def self.banner
  "webget-mirror/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}] in (#{root})"
end

.root ⇒ Object 



18
19
20
 
# File 'lib/webget/mirror/version.rb', line 18

def self.root
  File.expand_path( File.dirname(File.dirname(File.dirname(File.dirname(__FILE__)))) )
end

.version ⇒ Object 



9
10
11
 
# File 'lib/webget/mirror/version.rb', line 9

def self.version
  VERSION
end

Instance Method Details

#_broken_path?(path) ⇒ Boolean

and does NOT end_with / pages

Returns:



14
15
16
17
18
 
# File 'lib/webget/mirror/find_links.rb', line 14

def _broken_path?( path )
      path.start_with?( '//' ) ||
      path.end_with?( '/' ) ||
     !path.start_with?( '/' )  ## note - MUST start with single slash (/)

end

#_collect_page_info(doc, html:) ⇒ Object 



54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
 
# File 'lib/webget/mirror/collect_page_info.rb', line 54

def _collect_page_info( doc, html: )

           ## use collect_page_stat( doc: )

           ##   or   collect_page_info  ( pass in nokogiri doc !!)

           ##     PageInfo   (or Page::Info), PageStat

           ##       - title

           ##       - html_doctype

           ##       - html_charset

           ##       - tabs

           ##       ...


           ### try to find page title

           ##    not - title might be missing (nil)!!

             title_el =  doc.at_css('title')
             title =  title_el ? title_el.text.strip :  nil

    ##

             ##  note - use "plain-old" regex

             ##     to get "raw" doctype/charset  from html source

             ##

             ## record doctype

             ##  e.g

             ##  <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">

             ##  <!DOCTYPE HTML>

             ## and html charset (inside meta)

             ##  e.g.

             ## <meta http-equiv="Content-Type" content="text/html; charset=Windows-1252">

             ## <meta charset="Windows-1252">


             ###

             ## note -   limit search to 1024 ( or allow double 2048)


             html_doctype =  (m=HTML_DOCTYPE_RE.match( html[0,1024] )) ? m[:doctype] : nil
             html_charset =  (m=HTML_CHARSET_RE.match( html[0,1024] )) ? m[:charset] : nil


             ## check for tabs  - make it nil if no tabs found otherwise use count

             tabs = html.scan( "\t" )
             tabs =  tabs.size == 0 ? nil : tabs.size


   meta = {
     title:        title,
     html_doctype: html_doctype,
     html_charset: html_charset,
     tabs:         tabs,
   }

   meta
end

#_download_page(url, encoding: nil, force: false) ⇒ Object 



7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
 
# File 'lib/webget/mirror/download_page.rb', line 7

def _download_page( url,
                    encoding: nil,
                    force: false )

##

## note - on windows cached will be CASE-INSENSITIVE

##       e.g. usadave and USAdave will match

##    make sure match is CASE-SENSITIVE!!!


          if force == false && Webcache.cached?( url )
              puts "   CACHE HIT - #{url}"
              html  = Webcache.read( url )

              return [html,nil]
          end


        puts "==> download #{url} (encoding: #{encoding})..."


    ## note: assume plain 7-bit ascii for now

    ##  -- assume rsssf uses ISO_8859_15 (updated version of ISO_8859_1)

    ###-- does NOT use utf-8 character encoding!!!

    response = Webget.page( url, encoding: encoding )  ## fetch (and cache) html page (via HTTP GET)


    ## note: exit on get / fetch error - do NOT continue for now - why? why not?

    ## note -    allow 404 not found to pass through



    ## note - status.code is an integer number (not a string!!)

    if response.status.code == 404

        meta = {
           http_status:     response.status.code,
        }

        ["404 NOT FOUND",meta]

    elsif response.status.code == 200
      puts "html:"
      html =  response.text
      pp html[0..200]

      ## note - use "hacky" undocument internal response._text_encoding

      ##                   to get "upstream" encoding used from convert to utf-8

      ##                         unicode boms may override user supplied encoding!!!

      ##  or change upstream

      ##   and    use/add response.text_with_encoding( ) - why? why not?

      ##   yes, upstream now uses

      ##    text( encoding: _encoding_user )!!!



      meta = {
          encoding:         response._text_encoding,
          encoding_source:  response._text_encoding_source,  ## bom|http|html|user|fallback

          encoding_valid:   response._text_encoding_valid,

          ascii7bit:         response._text_ascii_only,
          chars_8bit:        response._text_8bit,
          utf8_replace:      response._text_utf8_replace,

          http_content_type:     response.content_type,
          http_content_length:   response.content_length,
          http_status:           response.status.code,
      }

        [html,meta]

    else
       puts "unexpected http status (code) - #{response.status}"
       exit 1
    end
end

#_find_links(site:, doc:, url:, verbose: true) ⇒ Object

get all links ignore anchor links and split into internal and external



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
 
# File 'lib/webget/mirror/find_links.rb', line 25

def _find_links( site:,
                 doc:,
                 url:,
                 verbose: true )


       ## note - base_url is  the doc(ument) url

       ##                            e.g.  https://rsssf.org/curtour.html

       base_url = URI( url )


           if _broken_path?( base_url.path )
              puts "!! normalized base_url.path expected  - got:"
              pp url
              pp base_url
              exit 1
           end

    ##

    ## note: Array#compact removes all nil values from an array.

    ##    if no href in a - nokigiri return nil

    ##

    ##     might still incl. empty string ("") - remove too - why? why not?


    ##

    ##  fix - change to css('a[href]') or such ??

    ###     document.css("a[href]").each do |a|

    ## links = doc.css('a').map { |a| a['href'] }.compact



    links = doc.css( 'a[href]' ).map do |a|
                    ## strip leading & trailing spaces e.g.

                    ##   "http://www.danskfodbold.dk "

                    ##    is invalid url!!!

                    a['href'].strip
                end.reject do |href|
                    # skip

                    #  - empty strings,

                    #  - page anchors

                    href.empty? || href.start_with?('#') ||

                    ##  skip mailto links/javascript snippets

                    href.match?( /\A(?:mailto|javascript)/i ) ||

                     ## also skip broken mailto links

                     ##   that is, missing mailto

                     ##  e.g.

                     href.include?( '@' )
                end


    ## split into internal & external

    ## make links absolute

    ##   ignore anchor links (see above)


    pages     = []
    externals = []

    links.each do |href|

        ##

        ## auto-fix ("site-wide") known quirks:

        href = site.autofix_href.call( href )    if  site.autofix_href.is_a?( Proc )


                      page_url = nil
                      begin

                        ## special case

                        ##  check for protocol-relative  //  e.g. //hello.html

                        ##    NOT handled by URI

                        ##       URI makes hello.html into host !!!

                        ##                host is hello.html and path is nil

                        ##        only works properly with triple ///

                        ##             e.g. ///hello.html

                        ##              now host is nil, and path is /hello.html


      ##   URI.join(URI("https://example.com/page.html"), "//cdn.example.com/file.js")

      ##  # => #<URI::HTTPS https://cdn.example.com/file.js>  ✓ Works!

      ##

      ##   But with just the string:

      ##    URI("//cdn.example.com/file.js")

      ##     Parses incorrectly—no scheme, treats cdn.example.com as host  !!!!!



       ##  The browser breaks down //path/page.html like this:

       ##  - Protocol: Inherited from the current page (e.g., https:).

       ##  - Domain (Authority): path

       ##  - File Path: /page.html

       ##

       ## If your website is hosted on https://example.com and

       ## a user clicks <a href="//path/page.html">, the browser will try

       ## to navigate to https://path/page.html.

       ## Unless you own a domain name that is literally just path,

       ##  this will result in a "Site cannot be reached" error.

       ###

       ###  "legacy" protocol relative is "//://" !!!!

       ##

       ##   move notes from here to dedicated notes page!!




                        if href.start_with?("//")
                           puts "!!! debug break on href starting with //:"
                           pp  href
                           pp  url
                           pp  base_url
                           exit 1
                        end



                        ## check if href is absolute?

                        href_url = URI( href )

                        ## assume already absolute

                        if href_url.scheme && href_url.host
                          page_url = href_url
                        else
                          ## try to make absolute (relative to base_url)

                          page_url = URI.join(base_url, href_url)
                        end

                      rescue => ex
                         ## skip bad urls and log


                         msg = "bad url in #{base_url.path}:\n#{href}\nex:#{ex}\n"

                         ## note - only report in verbose mode (fresh download or such)!!!

                         if verbose
                           log( msg )
                           puts "!! " + msg
                         end

                         next
                      end

                      ###

                      ##  fix-fix-fix

                      ##    check for  optional www too

                      ##          assume same for now ??

                      ##    or better add to autofix

                      ##            if www.rsssf.org  change to  rsssf.org


                      if page_url.host == site.host    ## e.g. 'rsssf.org'

                          if page_url.path == base_url.path
                                 puts "   anchor  #{href}  =>  #{page_url.fragment}"     if verbose
                          else
                               puts "   internal page  #{href}  =>  #{page_url.path}"     if verbose

                               ## note - for internal pages

                               ##  for now no SUPPORT for query

                               ##    e.g. foo=1&bar=2

                               if page_url.query
                                   ## change to ValueError or such - why? why not?

                                   ## raise ArgumentError, "query in internal page links not yet supported, sorry - got #{page_url}"

                                   msg = "query in internal page links not yet supported, sorry - got #{page_url}"
                                   puts "!! WARN - #{msg}"
                                   log( msg )
                                   next
                               end

        if _broken_path?( page_url.path )
            puts "!! normalized page_url.path expected - got:"
            pp page_url.path
            pp page_url
            puts "base_url:"
            pp url
            pp base_url
            exit 1
          end

                               pages << page_url.path
                          end
                      else
                         puts "!! external  #{href}  =>  #{page_url}"      if verbose
                         externals << page_url.to_s
                      end
                    end

     ## make uniq

     pages     = pages.uniq
     externals = externals.uniq

      if verbose
      puts "   #{pages.size} internal & #{externals.size} external link(s) found in #{base_url.path}:"


    pp pages
    pp externals
      end

    [pages, externals]
end

#_mirror_pages(site:, force: false, batch: 1000) ⇒ Object

use limit for batch - why? why not? start of / try a batch of a hundred



17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
 
# File 'lib/webget/mirror/mirror.rb', line 17

def _mirror_pages( site:,
                   force: false,
                   batch: 1000 )

    visited    = 0
    downloaded = 0

    time_start = Time.now

    loop do

       ## (i)  prioritize main pages (e.g. use start_pages_path)

       ## /curdom.html

       ## /curtour.html

       ## /histdom.html

       ## /intclub.html

       ## /intland.html

      page_recs =  MirrorDb::Model::Page.where( cached: false,
                                                path:   site.start_pages_path
                                               ).limit( batch )


      ##  (ii)   prefer pages   (e.g. use boost_pages_path_like)

      ##  starting with /tables,/tables[a-z]/

      if page_recs.size == 0 && site.boost_pages_path_like?
        page_recs =  MirrorDb::Model::Page.where( cached: false ).
                                           where( 'path LIKE ?',
                                                  site.boost_pages_path_like ).limit( batch )
      end

      ##  (iii)  retry "unconstrained"  if nothing found matching  (i & ii)

      if page_recs.size == 0
        page_recs =  MirrorDb::Model::Page.where( cached: false ).limit( batch )
      end


      ### no more pages - done - break out of loop and say goodbye

      break   if page_recs.size == 0




       page_recs.each_with_index do |page_rec,i|

        ##

        ##  fix-fix-fix  - change to mime type - why? why not?

        ##           allow pages with no extensions!!!


         ### special case for non .html/.htm pages (e.g. .pdf others too??)

         ##    do NOT download / mirror / cache for now

         if page_rec.not_html?
            page_rec.update!( cached: true )
            next
         end


         ## note - on download (not if cached)

         ##        encoding

         ##           might be get changed

         ##        ALWAYS use updated encoding!!


         ##

         ## note - workaround for windows

         ##     on windows File.exist? (and Webcache.cached?)

         ##          is case-insensitive

         ##    e.g. /USAdave/ is the same as /usadave/

         ##

         ##   as a workaround ALWAYS hardcode 404

         ##    for /USAdave/    to get (and record) 404  (and not CACHE HITS!!)

         ##   e.g. try https://rsssf.org/USAdave/cncc.html  => 404 (NOT FOUND)

         ##            https://rsssf.org/usadave/cncc.html  => 200 (OK)



         ##  note - url e.g. https://rsssf.org

         ##         path MUST start with /  e.g.  /curtour.html

         ##  resulting in   https://rsssf.org/curtour.html


         url = site.base_url+page_rec.path

        ## if %r{/USAdave/}.match?(page_rec.path)

        ##                                 ['', {status: 404}]




  ## check if not in cache

  ##   note - use force == true  to always (force) download


          html, response_meta = _download_page( url,
                                                encoding: page_rec.encoding,
                                                force:    force  )

          if response_meta
              downloaded += 1
              puts " ---  " + fmt_time_diff( time_start,  count: downloaded )

              ###

              ## special case

              ##  check for 404 NOT FOUND

              if response_meta[:http_status] == 404
                      page_rec.update!( http_status: 404,
                                        cached:      true )

                next   ### note - skip further processing on 404 (no links etc.)!!

              end
          end



          html = site.errata( html, url: url )    if site.errata?




         ## Standard HTML4-style parsing (default)

         ## doc = Nokogiri::HTML(malformed_html)

         ##  -or-

         ## More robust HTML5 parsing

         ##doc = Nokogiri::HTML5(malformed_html)


           doc = Nokogiri::HTML( html )


           ## get (page meta info)

           ##   title, tabs (count), html_doctype, html_charset

            page_info = _collect_page_info( doc, html: html )



          ##  note - if response meta data present than fresh download (not cached)

          ##  cached  = response_meta ? false : true


          ## turn on verbose mode only if page downloaded (not on cache hit)

           verbose = response_meta ? true : false
          ## verbose = true


           internals, _ = _find_links( site: site,
                                       doc: doc,
                                       url: url,
                                       verbose: verbose
                                     )


            ## add links to db

            internals.each do |path|
               internal_rec = MirrorDb::Model::Page.find_or_create_by!(
                                                            path: path ) do |rec|
                                    puts "     add linked page #{rec.path}"

                                    rec.encoding = site.page_encoding( rec.path )
                                    rec.cached   = false
                                 end

               ## puts "  add link from #{page_rec.path} to #{internal_rec.path} to mirror.db"

               ###  note allow - find (may happen after "crash" or interrupt)

               link_rec = MirrorDb::Model::Link.find_or_create_by!(
                                                         from_page_id: page_rec.id,
                                                         to_page_id:   internal_rec.id )
            end

            puts "  [#{i+1}/#{page_recs.size}] update page #{page_rec.path} w/ #{internals.size} page(s) linked - >#{page_info[:title] || 'n/a'}<"


            ###

            ##  note - remove cached (flag) and replace with http_status => nil|200|404|etc?

            ##         that is, cached = false  => nil

            ##                  cached = true   =>  200|404|etc - why? why not??

            attribs = {
                cached: true
            }
            ## add (optional) page_info attribus

            more_attribs = {
              title:        page_info[:title], ## note - might be missing (nil) in some pages

              html_doctype: page_info[:html_doctype],
              html_charset: page_info[:html_charset],
              tabs:         page_info[:tabs]
            }
            attribs = attribs.merge( more_attribs )


            ## check for encoding when fresh download (via response meta data)

            if response_meta
               more_attribs = {
                  encoding:        response_meta[:encoding] ? response_meta[:encoding].downcase : nil,
                  encoding_source: response_meta[:encoding_source],   ## bom|html|http|user|fallback

                  encoding_valid:  response_meta[:encoding_valid],    ## nil|true|false


                   ascii7bit:         response_meta[:ascii7bit],      ## nil|true|false

                   ## note - convert 10 - 212=>8, 233=>2  (use only first total count; no details)

                   chars_8bit:        response_meta[:chars_8bit] ? response_meta[:chars_8bit].to_i(10) : nil,
                   utf8_replace:      response_meta[:utf8_replace],

                   http_content_type:     response_meta[:http_content_type],
                   http_content_length:   response_meta[:http_content_length],
                   http_status:           response_meta[:http_status]
                }
               attribs = attribs.merge( more_attribs )
            end


            page_rec.update!( **attribs )



            visited += 1

           if visited % 100 == 0
              puts "\n visited: #{visited} (downloaded: #{downloaded}) - " +
                 " #{MirrorDb::Model::Page.count} page(s) indexed " +
                 "(#{MirrorDb::Model::Page.cached.count} cached, " +
                 "#{MirrorDb::Model::Page.not_cached.count} missing)"

             puts "  " + fmt_time_diff( time_start,  step: downloaded,
                                                      count: downloaded+MirrorDb::Model::Page.not_cached.count )


           end
       end
    end



            puts "\n visited: #{visited} (downloaded: #{downloaded}) - " +
                 " #{MirrorDb::Model::Page.count} page(s) indexed " +
                 "(#{MirrorDb::Model::Page.cached.count} cached, " +
                 "#{MirrorDb::Model::Page.not_cached.count} missing)"
end

#fmt_time_diff(time_start, time_end = Time.now, count:, step: nil) ⇒ Object 



8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/webget/mirror/utils.rb', line 8

def fmt_time_diff( time_start, time_end=Time.now, count:, step: nil )
   time_diff  = time_end - time_start
   buf = String.new

     if count == 0 || step == 0
       buf +=  "  %d:%02d mins" % [time_diff/60, time_diff%60]
     elsif step
       buf +=  "  [#{step}/#{count} - %5.2f%%]" % [step*100/count]

       buf +=  "  %d:%02d mins" % [time_diff/60, time_diff%60]
       buf +=  " - %5.2f secs/page" % [time_diff/step]

       time_estimate = (time_diff/step) * count
       buf +=  ", estimate: %d:%02d mins" % [time_estimate/60, time_estimate%60]
    else
      buf +=  "  %d:%02d mins" % [time_diff/60, time_diff%60]
      buf +=  " - %5.2f secs/page  (#{count} pages)" % [time_diff/count]
   end

   buf
end

#log(msg) ⇒ Object

auto log errors (append to logs.txt)



39
40
41
42
43
44
45
46
 
# File 'lib/webget/mirror.rb', line 39

def log( msg )
   ## append msg to ./logs.txt

   ##     use ./errors.txt - why? why not?

   File.open( './mirror_logs.txt', 'a:utf-8' ) do |f|
     f.write( msg )
     f.write( "\n" )
   end
end