Class: Bookshark::Extractor

Inherits:
Object
  • Object
show all
Includes:
FileManager
Defined in:
lib/bookshark.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from FileManager

#list_directories, #list_files, #save_to

Constructor Details

#initialize(options = {}) ⇒ Extractor

Returns a new instance of Extractor.



37
38
39
40
41
 
# File 'lib/bookshark.rb', line 37

def initialize(options = {})
  options = DEFAULTS.merge(options)
  @site   = options[:site]
  @format = options[:format]
end

Instance Attribute Details

#format ⇒ Object

Returns the value of attribute format.



35
36
37
 
# File 'lib/bookshark.rb', line 35

def format
  @format
end

#site ⇒ Object

Returns the value of attribute site.



35
36
37
 
# File 'lib/bookshark.rb', line 35

def site
  @site
end

Instance Method Details

#author(options = {}) ⇒ Object 



43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'lib/bookshark.rb', line 43

def author(options = {})
  uri = process_options(options, __method__)
  options[:format] ||= @format

  author_extractor = Biblionet::Extractors::AuthorExtractor.new
  author = author_extractor.load_and_extract_author(uri)

  response = {}
  response[:author] = !author.nil? ? [author] : []
  response = change_format(response, options[:format])
  return response
end

#authors_from_storage ⇒ Object

def books_from_storage

extract_from_storage_and_save('book', 'html_book_pages', 'json_book_pages')

end



171
172
173
 
# File 'lib/bookshark.rb', line 171

def authors_from_storage
  extract_from_storage_and_save('author', 'html_author_pages', 'json_author_pages')
end

#book(options = {}) ⇒ Object 



72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
 
# File 'lib/bookshark.rb', line 72

def book(options = {})
  options[:site] ||= @site

  if options[:site] == 'biblionet'
    book_extractor = Biblionet::Extractors::BookExtractor.new

    if book_extractor.present?(options[:isbn])
      search_engine = Biblionet::Extractors::Search.new
      options[:id]  = search_engine.search_by_isbn(options[:isbn])
    end

    uri = process_options(options, __method__)
    options[:format]  ||= @format
    options[:eager]   ||= false
    options[:nilify]  ||= false

    if options[:eager]
      book = eager_extract_book(uri)
    else
      book = book_extractor.load_and_extract_book(uri)
    end

    response = {}
    response[:book] = !book.nil? ? [book] : []

    return nil if response[:book].empty? and options[:nilify]

    response = change_format(response, options[:format])

    response = book_extractor.decode_text(response) if response.class == "String"

    return response
  elsif options[:site] == 'nlg'
    book_extractor = Nlg::Extractors::BookExtractor.new

    options[:format] ||= @format

    # if !options[:uri].nil?
    #   uri = "#{options[:uri]}/Export?style=MARCXML"
    # elsif !options[:id].nil?
    #   uri = "http://nbib.nlg.gr/Record/#{options[:id]}/Export?style=MARCXML"
    # end

    book = book_extractor.load_and_extract_book(options[:id])

    response = {}
    response[:book] = !book.nil? ? [book] : []
  end
end

#categories_from_storage ⇒ Object 



179
180
181
 
# File 'lib/bookshark.rb', line 179

def categories_from_storage
  extract_from_storage_and_save('category', 'html_category_pages', 'json_category_pages')
end

#category(options = {}) ⇒ Object

puts Bookshark::Extractor.new(format: ‘pretty_json’).bibliographical_book(id: 103788)



139
140
141
142
143
144
145
146
147
148
149
150
151
 
# File 'lib/bookshark.rb', line 139

def category(options = {})
  uri = process_options(options, __method__)
  options[:format] ||= @format

  category_extractor = Biblionet::Extractors::CategoryExtractor.new
  category = category_extractor.extract_categories_from(uri)

  response = {}
  response[:category] = !category.nil? ? [category] : []
  response = change_format(response, options[:format])

  return response
end

#extract_books_from_storage_and_save(start_id, finish_id, format = 'pretty_json') ⇒ Object 



183
184
185
186
187
188
189
190
191
 
# File 'lib/bookshark.rb', line 183

def extract_books_from_storage_and_save(start_id, finish_id, format = 'pretty_json')
  start_id.upto(finish_id) do |book_id|
    record = book(id: book_id, local: true, format: format, nilify: true)

    dir_to_save = Bookshark.path_to_storage + '/' + 'json_book_records/' + "#{((book_id-1)/1000)}/" + "book_#{book_id}.json"

    save_to(dir_to_save, record) unless record.nil?
  end
end

#extract_from_storage_and_save(metadata_type, source_dir, target_dir) ⇒ Object 



194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
 
# File 'lib/bookshark.rb', line 194

def extract_from_storage_and_save(metadata_type, source_dir, target_dir)
  list_directories(path: Bookshark.path_to_storage + '/' + source_dir).each do |dir|
    dir_to_save = dir.gsub(source_dir, target_dir)

    list_files(path: dir, extension: 'html', all:true).each do |file|
      puts "Extracting from file: " + file.to_s

      # Extract publisher metadata form local file.
      options = {uri: file, format: 'pretty_json', local: true}

      case metadata_type
      when 'author'
        record = author(options)
      when 'publisher'
        record = publisher(options)
      # when 'book'
      #   record = book(options)
      when 'category'
        record = category(options)
      end

      # Prepare a path to save the new file.
      filename  = File.basename(file,".*")
      path_to_save = "#{dir_to_save}#{filename}.json"

      # Save to file.
      save_to("#{path_to_save}", record)

    end # unless File.directory?(dir_to_save) # if dir.end_with? '/195/'
  end
end

#parse_all_books ⇒ Object 



246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
 
# File 'lib/bookshark.rb', line 246

def parse_all_books
  bp = Biblionet::Extractors::BookExtractor.new

  list_directories(path: 'storage/raw_html_pages').each do |dir|
    dir_to_save = dir.gsub(/raw_html_pages/, 'books')

    list_files(path: dir, extension: 'html', all:true).each do |file|

      # Load the book from html file and parse the data.
      # pp "Parsing book: #{file}"
      pp file
      book = bp.load_and_extract_book(file)

      # Prepare a path to save the new file.
      filename  = File.basename(file,".*")
      path_to_save = "#{dir_to_save}#{filename}.json"

      # Save to file.
      bp.save_to("#{path_to_save}", JSON.pretty_generate(book))
      # pp "Book #{file} saved!"
    end unless File.directory?(dir_to_save) # if dir.end_with? '/195/'
  end
end

#parse_all_categories(will_save = false) ⇒ Object 



226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
 
# File 'lib/bookshark.rb', line 226

def parse_all_categories(will_save=false)
  # list_directories('raw_ddc_pages').each do |dir|
    # p dir
  # end
  category_extractor = Biblionet::Extractors::CategoryExtractor.new
  all_categories = Hash.new

  list_files(path: 'storage/raw_ddc_pages', extension: 'html', all:true).each do |file|
    categories = category_extractor.extract_categories_from(file)
    all_categories.merge!(categories) unless categories.nil? or categories.empty?
  end

  if will_save
    all_categories_json = all_categories.to_json
    save_to('storage/all_categories.json',all_categories_json)
  end

  all_categories
end

#publisher(options = {}) ⇒ Object 



56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/bookshark.rb', line 56

def publisher(options = {})
  uri = process_options(options, __method__)
  options[:format] ||= @format

  publisher_extractor = Biblionet::Extractors::PublisherExtractor.new
  publisher = publisher_extractor.load_and_extract_publisher(uri)

  response = {}
  response[:publisher] = !publisher.nil? ? [publisher] : []
  response = change_format(response, options[:format])
  response = publisher_extractor.decode_text(response)

  return response
  # return uri
end

#publishers_from_storage ⇒ Object 



175
176
177
 
# File 'lib/bookshark.rb', line 175

def publishers_from_storage
  extract_from_storage_and_save('publisher', 'html_publisher_pages', 'json_publisher_pages')
end

#search(options = {}) ⇒ Object 



153
154
155
156
157
158
159
160
161
162
163
164
165
 
# File 'lib/bookshark.rb', line 153

def search(options = {})
  options[:format]        ||= @format
  options[:results_type]  ||= 'metadata'

  search_engine  = Biblionet::Extractors::Search.new
  search_results = search_engine.perform_search(options)

  response = {}
  response[:book] = search_results
  response = change_format(response, options[:format])

  return response
end