Class: Wiki

Inherits:
Object
  • Object
show all
Defined in:
lib/wp/wiki.rb,
lib/wp/old/wiki.rb

Constant Summary collapse

@@index_url =

@@index_url = “dumps.wikimedia.org/enwiki/20130604/enwiki-20130604-pages-articles-multistream-index.txt.bz2”

"/d/wiki/enwiki-20130604-pages-articles-multistream-index.txt.bz2"
@@articles_url = 
"/d/wiki/enwiki-20130604-pages-articles-multistream.xml.bz2"

Instance Method Summary collapse

Constructor Details

#initialize(path = "enwiki") ⇒ Wiki

Returns a new instance of Wiki.



82
83
 
# File 'lib/wp/wiki.rb', line 82

def initialize
end

Instance Method Details

#[](key) ⇒ Object 



11
12
13
 
# File 'lib/wp/old/wiki.rb', line 11

def [](key)
  Zlib.inflate @db.get(key)
end

#article(title, redirected_from = nil) ⇒ Object 



107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
 
# File 'lib/wp/wiki.rb', line 107

def article(title, redirected_from=nil)
  if title =~ %r{^title/(.+)}
    title = $1
  end

  return nil unless offset = db["title/#{title}"]
    
  offset = offset.to_i
  length = db["length/#{offset}"].to_i

  bz2_stream(@@articles_url, offset, length) do |io|
    xml = XMLReader.new io.read

    xml.each do |article|

      if title == article.title
        if article.redirect?
          return article(article.redirect, title)
        else
          article.redirected_from = redirected_from
          return article
        end
      end

    end
  end
end

#db ⇒ Object 



86
87
88
 
# File 'lib/wp/wiki.rb', line 86

def db
  @db ||= LevelDB::DB.new File.expand_path("~/.cache/wp/enwiki-index")
end

#html(title) ⇒ Object 



23
24
25
 
# File 'lib/wp/old/wiki.rb', line 23

def html(title)
  WikiCloth::Parser.new(data: self[title]).to_html
end

#import_index(index_url = @@index_url) ⇒ Object 



135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
 
# File 'lib/wp/wiki.rb', line 135

def import_index(index_url=@@index_url)
  bz2_stream(index_url) do |io|

    last_offset = nil

    io.each_line.with_index do |line,i|
      line.chomp!

      offset, n, title = line.split(":", 3)
      db["title/#{title}"] = offset

      offset = offset.to_i

      last_offset = offset if last_offset.nil?

      if last_offset != offset
        length = offset - last_offset
        db["length/#{last_offset}"] = length.to_s

        last_offset = offset
      end

      print "\e[1G#{commatize i} - #{title}\e[J" if i % 11337 == 0

    end

  end
end

#random(amount = 30) ⇒ Object 



97
98
99
100
101
102
103
104
105
 
# File 'lib/wp/wiki.rb', line 97

def random(amount=30)
  letters = [*'A'..'Z'] + [*'0'..'9']

  (1..30).map do 
    prefix = letters.sample + letters.sample.downcase
    key, val = db.each(from: prefix).first
    key.split("/", 2).last
  end
end

#search(prefix, max = 30) ⇒ Object 



91
92
93
94
95
 
# File 'lib/wp/wiki.rb', line 91

def search(title, amount=40)
  db.each(from: "title/#{title}").take(amount).map do |key, val|
    key.split("/", 2).last
  end
end

#titles(max = 30) ⇒ Object 



15
16
17
 
# File 'lib/wp/old/wiki.rb', line 15

def titles(max=30)
  search("", max)
end