Class: Webget::Mirror::Website

Inherits:
Object
  • Object
show all
Defined in:
lib/webget/mirror/website.rb

Instance Method Summary collapse

Instance Method Details

#_default_page_encoding ⇒ Object 



76
77
78
 
# File 'lib/webget/mirror/website.rb', line 76

def _default_page_encoding
     defined?( @default_page_encoding ) ?  @default_page_encoding : 'UTF-8'
end

#autofix_href ⇒ Object 



135
 
# File 'lib/webget/mirror/website.rb', line 135

def autofix_href()  defined?( @autofix_href ) ? @autofix_href : nil; end

#autofix_href=(m) ⇒ Object

Raises:



129
130
131
132
133
134
 
# File 'lib/webget/mirror/website.rb', line 129

def autofix_href=( m )
     raise ArgumentError,
             "proc expected for autofix_href=; got #{m} #{m.class.name}"   unless m.is_a?( Proc )

     @autofix_href = m
end

#base_url ⇒ Object

Raises:



15
16
17
18
 
# File 'lib/webget/mirror/website.rb', line 15

def base_url
    raise ArgumentError, "website.base_url not set"   if @base_url.nil?
    @base_url.to_s     ## note - returns string NOT uri object!!!
end

#base_url=(url) ⇒ Object 



12
13
14
 
# File 'lib/webget/mirror/website.rb', line 12

def base_url=( url )
    @base_url = URI( url )   ## note URI() same as URI.parse()
end

#boost_pages_path_like ⇒ Object Also known as: boost_path_like 



63
 
# File 'lib/webget/mirror/website.rb', line 63

def boost_pages_path_like()  defined?( @boost ) ? @boost : nil; end

#boost_pages_path_like=(str) ⇒ Object Also known as: boost_path_like= 



61
 
# File 'lib/webget/mirror/website.rb', line 61

def boost_pages_path_like=(str)  @boost = str; end

#boost_pages_path_like? ⇒ Boolean Also known as: boost_path_like?

Returns:



62
 
# File 'lib/webget/mirror/website.rb', line 62

def boost_pages_path_like?() defined?( @boost ); end

#default_page_encoding=(encoding) ⇒ Object 



73
74
75
 
# File 'lib/webget/mirror/website.rb', line 73

def default_page_encoding=( encoding )
     @default_page_encoding = encoding
end

#errata(html, url:) ⇒ Object 



105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/webget/mirror/website.rb', line 105

def errata( html, url: )
     if defined?( @errata_edits )

       ## lookup edits by path e.g. /tablesp/poland-satrip77.html
       ##                       or  /miscellaneous/torre-madrid.html

       page_url = URI( url )

       edits = @errata_edits[page_url.path]

       ## note - for now always use gsub (not sub)
       ##   maybe add option later
       if edits
         edits.each do |search,replace|
                       html = html.gsub( search, replace )
                    end
       end
       html
     else   ## pass along as is (1:1)
       html
     end
end

#errata? ⇒ Boolean

quick fix html w/ search & replace default: do nothing

Returns:



104
 
# File 'lib/webget/mirror/website.rb', line 104

def errata?()   defined?( @errata_edits ); end

#errata_edits=(edits) ⇒ Object 



100
 
# File 'lib/webget/mirror/website.rb', line 100

def errata_edits=(edits)  @errata_edits = edits; end

#host ⇒ Object

Raises:



7
8
9
10
 
# File 'lib/webget/mirror/website.rb', line 7

def host
   raise ArgumentError, "website.base_url not set"   if @base_url.nil?
   @base_url.host
end

#mirror ⇒ Object

kick-off mirror (pages) operation/run



140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
 
# File 'lib/webget/mirror/website.rb', line 140

def mirror


  ## add seed/start pages
  start_pages.each do |config|
    path     = config['page']
    encoding = config['encoding']
    encoding = page_encoding( path )   if config['encoding'].nil? || config['encoding'].empty?

    page_rec = MirrorDb::Model::Page.find_or_create_by!( path: path ) do |rec|
                    ## note - block only called on create (NOT find!!)
                    puts "  add page #{rec.path} (cached: false) to mirror.db"

                    rec.encoding = encoding
                    rec.cached   = false
              end
    pp page_rec



    ## fix-fix-fix use Mirror.new( self )
    ##         pass in site config on new!!!
    mirror = Mirror.new
    mirror._mirror_pages( site: self )
  end
end

#page_encoding(path) ⇒ Object

default page encoding (lookup by path); change to windows-1256 if needed



90
91
92
93
94
95
96
 
# File 'lib/webget/mirror/website.rb', line 90

def page_encoding( path )
   if defined?( @page_encodings )
      @page_encodings[ path ]
   else
     _default_page_encoding
   end
end

#page_encodings=(encodings) ⇒ Object

use hash with custom (individual) defaults e.g. Hash.new { |h,key| h = 'windows-1252' }



84
85
86
 
# File 'lib/webget/mirror/website.rb', line 84

def page_encodings=( encodings )  ## hash (path => encoding)
    @page_encodings = encodings
end

#start_pages ⇒ Object

Raises:



28
29
30
31
 
# File 'lib/webget/mirror/website.rb', line 28

def start_pages
    raise ArgumentError, "website.start_pages not set"   if @start_pages.nil?
    @start_pages
end

#start_pages=(config) ⇒ Object

array of hash (table) records e.g. page, encoding /index.html, windows-1252



25
26
27
 
# File 'lib/webget/mirror/website.rb', line 25

def start_pages=( config )
  @start_pages = config
end

#start_pages_path ⇒ Object Also known as: start_path 



34
35
36
37
38
39
40
41
42
43
44
45
46
 
# File 'lib/webget/mirror/website.rb', line 34

def start_pages_path
    ## collect/build path for all start pages
    ## e.g.  path: ['/curdom.html',
    ##               '/curtour.html',
    ##               '/histdom.html',
    ##               '/intclub.html',
    ##               '/intland.html']
    path = []
    start_pages.each do |config|
        path << config['page']
    end
    path
end