Class: NHKore::Sifter
Overview
Constant Summary collapse
- DEFAULT_DIR =
Util::CORE_DIR
- DEFAULT_FUTSUU_FILENAME =
'sift_nhk_news_web_regular'
- DEFAULT_YASASHII_FILENAME =
'sift_nhk_news_web_easy'
- DEFAULT_FUTSUU_FILE =
build_file(DEFAULT_FUTSUU_FILENAME)
- DEFAULT_YASASHII_FILE =
build_file(DEFAULT_YASASHII_FILENAME)
Instance Attribute Summary collapse
Class Method Summary collapse
Instance Method Summary collapse
- #build_header ⇒ Object
- #build_rows(words) ⇒ Object
- #build_word_row(word) ⇒ Object
- #compare_empty_str(str1, str2) ⇒ Object
- #filter?(article) ⇒ Boolean
- #filter_by_datetime(datetime_filter = nil, from: nil, to: nil) ⇒ Object
- #filter_by_title(title_filter, uncase: true, unspace: true) ⇒ Object
- #filter_by_url(url_filter, uncase: true, unspace: true) ⇒ Object
- #ignore(key) ⇒ Object
-
#initialize(news) ⇒ Sifter
constructor
A new instance of Sifter.
-
#put_csv! ⇒ Object
This does not output #caption.
- #put_html! ⇒ Object
- #put_json! ⇒ Object
- #put_yaml! ⇒ Object
- #sift ⇒ Object
- #to_s ⇒ Object
Methods included from Fileable
Constructor Details
#initialize(news) ⇒ Sifter
Returns a new instance of Sifter.
43 44 45 46 47 48 49 |
# File 'lib/nhkore/sifter.rb', line 43 def initialize(news) @articles = news.articles.values.dup @caption = nil @filters = {} @ignores = {} @output = nil end |
Instance Attribute Details
#articles ⇒ Object
37 38 39 |
# File 'lib/nhkore/sifter.rb', line 37 def articles @articles end |
#caption ⇒ Object
38 39 40 |
# File 'lib/nhkore/sifter.rb', line 38 def caption @caption end |
#filters ⇒ Object
39 40 41 |
# File 'lib/nhkore/sifter.rb', line 39 def filters @filters end |
#ignores ⇒ Object
40 41 42 |
# File 'lib/nhkore/sifter.rb', line 40 def ignores @ignores end |
#output ⇒ Object
41 42 43 |
# File 'lib/nhkore/sifter.rb', line 41 def output @output end |
Class Method Details
.build_file(filename) ⇒ Object
30 31 32 |
# File 'lib/nhkore/sifter.rb', line 30 def self.build_file(filename) return File.join(DEFAULT_DIR,filename) end |
Instance Method Details
#build_header ⇒ Object
51 52 53 54 55 56 57 58 59 60 61 |
# File 'lib/nhkore/sifter.rb', line 51 def build_header header = [] header << 'Frequency' unless @ignores[:freq] header << 'Word' unless @ignores[:word] header << 'Kana' unless @ignores[:kana] header << 'English' unless @ignores[:eng] header << 'Definition' unless @ignores[:defn] return header end |
#build_rows(words) ⇒ Object
63 64 65 66 67 68 69 70 71 |
# File 'lib/nhkore/sifter.rb', line 63 def build_rows(words) rows = [] words.each do |word| rows << build_word_row(word) end return rows end |
#build_word_row(word) ⇒ Object
73 74 75 76 77 78 79 80 81 82 83 |
# File 'lib/nhkore/sifter.rb', line 73 def build_word_row(word) row = [] row << word.freq unless @ignores[:freq] row << word.word unless @ignores[:word] row << word.kana unless @ignores[:kana] row << word.eng unless @ignores[:eng] row << word.defn unless @ignores[:defn] return row end |
#compare_empty_str(str1, str2) ⇒ Object
363 364 365 366 367 368 369 370 371 372 373 374 |
# File 'lib/nhkore/sifter.rb', line 363 def compare_empty_str(str1,str2) has_str1 = !Util.empty_web_str?(str1) has_str2 = !Util.empty_web_str?(str2) if has_str1 && !has_str2 return -1 # Bubble word1 to top elsif !has_str1 && has_str2 return 1 # Bubble word2 to top end return 0 # Further comparison needed end |
#filter?(article) ⇒ Boolean
85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 |
# File 'lib/nhkore/sifter.rb', line 85 def filter?(article) return false if @filters.empty? datetime_filter = @filters[:datetime] title_filter = @filters[:title] url_filter = @filters[:url] if !datetime_filter.nil? datetime = article.datetime return true if datetime.nil? || datetime < datetime_filter[:from] || datetime > datetime_filter[:to] end if !title_filter.nil? title = article.title.to_s title = Util.unspace_web_str(title) if title_filter[:unspace] title = title.downcase if title_filter[:uncase] return true unless title.include?(title_filter[:filter]) end if !url_filter.nil? url = article.url.to_s url = Util.unspace_web_str(url) if url_filter[:unspace] url = url.downcase if url_filter[:uncase] return true unless url.include?(url_filter[:filter]) end return false end |
#filter_by_datetime(datetime_filter = nil, from: nil, to: nil) ⇒ Object
118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 |
# File 'lib/nhkore/sifter.rb', line 118 def filter_by_datetime(datetime_filter=nil,from: nil,to: nil) if !datetime_filter.nil? if datetime_filter.respond_to?(:[]) # If out-of-bounds, just nil. from = datetime_filter[0] if from.nil? to = datetime_filter[1] if to.nil? else from = datetime_filter if from.nil? to = datetime_filter if to.nil? end end from = to if from.nil? to = from if to.nil? from = Util.jst_time(from) unless from.nil? to = Util.jst_time(to) unless to.nil? datetime_filter = [from,to] return self if datetime_filter.flatten.compact.empty? @filters[:datetime] = {from: from,to: to} return self end |
#filter_by_title(title_filter, uncase: true, unspace: true) ⇒ Object
145 146 147 148 149 150 151 152 |
# File 'lib/nhkore/sifter.rb', line 145 def filter_by_title(title_filter,uncase: true,unspace: true) title_filter = Util.unspace_web_str(title_filter) if unspace title_filter = title_filter.downcase if uncase @filters[:title] = {filter: title_filter,uncase: uncase,unspace: unspace} return self end |
#filter_by_url(url_filter, uncase: true, unspace: true) ⇒ Object
154 155 156 157 158 159 160 161 |
# File 'lib/nhkore/sifter.rb', line 154 def filter_by_url(url_filter,uncase: true,unspace: true) url_filter = Util.unspace_web_str(url_filter) if unspace url_filter = url_filter.downcase if uncase @filters[:url] = {filter: url_filter,uncase: uncase,unspace: unspace} return self end |
#ignore(key) ⇒ Object
163 164 165 166 167 |
# File 'lib/nhkore/sifter.rb', line 163 def ignore(key) @ignores[key] = true return self end |
#put_csv! ⇒ Object
This does not output #caption.
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 |
# File 'lib/nhkore/sifter.rb', line 170 def put_csv! require 'csv' words = sift @output = CSV.generate(headers: :first_row,write_headers: true) do |csv| csv << build_header words.each do |word| csv << build_word_row(word) end end return @output end |
#put_html! ⇒ Object
186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 |
# File 'lib/nhkore/sifter.rb', line 186 def put_html! words = sift @output = ''.dup @output << <<~HTML <!DOCTYPE html> <html lang="ja"> <head> <meta charset="utf-8"> <title>NHKore</title> <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Noto+Serif+JP&display=fallback"> <style> body { background-color: #FCFBF9; color: #333333; font-family: 'Noto Serif JP',Verdana,sans-serif; } h1 { color: #737373; } table { border-collapse: collapse; table-layout: fixed; width: 100%; } tr:nth-child(even) { background-color: #A5C7ED; } tr:hover { background-color: #FFDDCA; } td,th { border: 1px solid #333333; padding: 8px; text-align: left; } th { background-color: #082A8E; color: #FCFBF9; } td { vertical-align: top; } td:nth-child(1) { padding-right: 1em; text-align: right; } </style> </head> <body> <h1>NHKore</h1> <h2>#{@caption}</h2> <table> HTML # If have too few or too many '<col>', invalid HTML. @output << %Q(<col style="width:6em;">\n) unless @ignores[:freq] @output << %Q(<col style="width:17em;">\n) unless @ignores[:word] @output << %Q(<col style="width:17em;">\n) unless @ignores[:kana] @output << %Q(<col style="width:5em;">\n) unless @ignores[:eng] @output << "<col>\n" unless @ignores[:defn] # No width for defn, fills rest of page @output << '<tr>' build_header.each do |h| @output << "<th>#{h}</th>" end @output << "</tr>\n" words.each do |word| @output << '<tr>' build_word_row(word).each do |w| @output << "<td>#{Util.escape_html(w.to_s)}</td>" end @output << "</tr>\n" end @output << <<~HTML </table> </body> </html> HTML return @output end |
#put_json! ⇒ Object
276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 |
# File 'lib/nhkore/sifter.rb', line 276 def put_json! require 'json' words = sift @output = ''.dup @output << <<~JSON { "caption": #{JSON.generate(@caption)}, "header": #{JSON.generate(build_header)}, "words": [ JSON if !words.empty? 0.upto(words.length - 2) do |i| @output << " #{JSON.generate(build_word_row(words[i]))},\n" end @output << " #{JSON.generate(build_word_row(words[-1]))}\n" end @output << "]\n}\n" return @output end |
#put_yaml! ⇒ Object
303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 |
# File 'lib/nhkore/sifter.rb', line 303 def put_yaml! require 'psychgus' words = sift yaml = { caption: @caption, header: build_header, words: build_rows(words), } header_styler = Class.new do include Psychgus::Styler def style_sequence(sniffer,node) parent = sniffer.parent if !parent.nil? && parent.node.respond_to?(:value) && parent.value == 'header' node.style = Psychgus::SEQUENCE_FLOW end end end # Put each Word on one line (flow/inline style). @output = Util.dump_yaml(yaml,flow_level: 4,stylers: header_styler.new) return @output end |
#sift ⇒ Object
332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 |
# File 'lib/nhkore/sifter.rb', line 332 def sift master_article = Article.new @articles.each do |article| next if filter?(article) article.words.each_value do |word| master_article.add_word(word,use_freq: true) end end words = master_article.words.values words.sort! do |word1,word2| # Order by freq DESC (most frequent words to top). i = (word2.freq <=> word1.freq) # Order by !defn.empty, word ASC, !kana.empty, kana ASC, defn.len DESC, defn ASC. i = compare_empty_str(word1.defn,word2.defn) if i == 0 # Favor words that have definitions i = (word1.word.to_s <=> word2.word.to_s) if i == 0 i = compare_empty_str(word1.kana,word2.kana) if i == 0 # Favor words that have kana i = (word1.kana.to_s <=> word2.kana.to_s) if i == 0 i = (word2.defn.to_s.length <=> word1.defn.to_s.length) if i == 0 # Favor longer definitions i = (word1.defn.to_s <=> word2.defn.to_s) if i == 0 i end return words end |
#to_s ⇒ Object
376 377 378 |
# File 'lib/nhkore/sifter.rb', line 376 def to_s return @output.to_s end |