Class: XMLTV::Grabber

Inherits:
Object
  • Object
show all
Defined in:
lib/xmltv/xmltv.rb

Constant Summary collapse

Dag = 
24 * 60 * 60
Vandaag = 
Date.today
MythTV = 
Hash.new

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Grabber

Returns a new instance of Grabber.



339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
 
# File 'lib/xmltv/xmltv.rb', line 339

def initialize
  @hits = 0
#      @myname = File.basename($PROGRAM_NAME).gsub(/\..*/, '')
  @myname = self.class.to_s[7..-8].downcase
  @chnbasedir = "#{XmltvOptions.basedir}/#{myname}"
  @config_file_name = XmltvOptions.config || "#{chnbasedir}/config"
  @reject_file_name = "#{chnbasedir}/rejects"
  @config = load_config_file
  @lang = 'nl'
  @generator = "#{myname}.#{lang}"
  @base_url = "http://www.#{generator}"
  @spooldir = config['spooldir'] || "#{chnbasedir}/spool"
  [ chnbasedir, spooldir ].each do |dir|
    FileUtils.mkdir_p(dir) unless test(?d, dir)
  end
  @channel_list = "#{chnbasedir}/channel_list"
  @all_channels = load_channel_file(channel_list)
end

Instance Attribute Details

#all_channels ⇒ Object

Returns the value of attribute all_channels.



316
317
318
 
# File 'lib/xmltv/xmltv.rb', line 316

def all_channels
  @all_channels
end

#base_url ⇒ Object

Returns the value of attribute base_url.



315
316
317
 
# File 'lib/xmltv/xmltv.rb', line 315

def base_url
  @base_url
end

#channel_list ⇒ Object

Returns the value of attribute channel_list.



315
316
317
 
# File 'lib/xmltv/xmltv.rb', line 315

def channel_list
  @channel_list
end

#chnbasedir ⇒ Object

Returns the value of attribute chnbasedir.



315
316
317
 
# File 'lib/xmltv/xmltv.rb', line 315

def chnbasedir
  @chnbasedir
end

#config ⇒ Object

Returns the value of attribute config.



316
317
318
 
# File 'lib/xmltv/xmltv.rb', line 316

def config
  @config
end

#config_file_name ⇒ Object

Returns the value of attribute config_file_name.



315
316
317
 
# File 'lib/xmltv/xmltv.rb', line 315

def config_file_name
  @config_file_name
end

#generator ⇒ Object

Returns the value of attribute generator.



315
316
317
 
# File 'lib/xmltv/xmltv.rb', line 315

def generator
  @generator
end

#lang ⇒ Object

Returns the value of attribute lang.



315
316
317
 
# File 'lib/xmltv/xmltv.rb', line 315

def lang
  @lang
end

#myname ⇒ Object

Returns the value of attribute myname.



315
316
317
 
# File 'lib/xmltv/xmltv.rb', line 315

def myname
  @myname
end

#reject_file_name ⇒ Object

Returns the value of attribute reject_file_name.



316
317
318
 
# File 'lib/xmltv/xmltv.rb', line 316

def reject_file_name
  @reject_file_name
end

#spooldir ⇒ Object

Returns the value of attribute spooldir.



315
316
317
 
# File 'lib/xmltv/xmltv.rb', line 315

def spooldir
  @spooldir
end

Instance Method Details

#add_channels_to_config ⇒ Object 



594
595
596
597
598
599
600
601
602
603
604
605
 
# File 'lib/xmltv/xmltv.rb', line 594

def add_channels_to_config
  XmltvOptions.add.each do |item|
    if (f = all_channels.has_key?(item)) && ! config['channels'].include?(item)
      config['channels'] << item
      STDERR.puts("added #{item} #{channel_display(item)}") if XmltvOptions.verbose
    else
      reason = f ? "already included" : "not in channel list"
      STDERR.puts "#{item} #{reason}" 
    end
  end
  save_config
end

#cachefile(chan_id) ⇒ Object 



370
371
372
 
# File 'lib/xmltv/xmltv.rb', line 370

def cachefile(chan_id)
  "#{chnbasedir}/#{channel_name(chan_id)}.yaml"
end

#channel_display(chan_id) ⇒ Object 



367
368
369
 
# File 'lib/xmltv/xmltv.rb', line 367

def channel_display(chan_id)
  all_channels[chan_id]
end

#channel_name(chan_id) ⇒ Object 



364
365
366
 
# File 'lib/xmltv/xmltv.rb', line 364

def channel_name(chan_id)
  "#{chan_id}.#{myname}.#{lang}"
end

#check_argv(args) ⇒ Object 



380
381
382
383
384
385
386
387
388
389
390
 
# File 'lib/xmltv/xmltv.rb', line 380

def check_argv(args)
  case args[0]
    when 'all'
      all_channels.keys
    when 'new'
      XmltvOptions.only_new = true
      all_channels.keys
    else
      args
  end
end

#check_channel(chan_id) ⇒ Object 



590
591
592
 
# File 'lib/xmltv/xmltv.rb', line 590

def check_channel(chan_id)
  all_channels.has_key?(chan_id)
end

#clean_cache_dir ⇒ Object 



457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
 
# File 'lib/xmltv/xmltv.rb', line 457

def clean_cache_dir
  clean_spool_dir
  last_time = config['cleaned']
  if last_time.is_a?(Date) && last_time >= Vandaag
    return
  end
  count = 0
  channels = config_channels.map{|x| cachefile(x) }
  Dir["#{chnbasedir}/*.yaml"].each do |fn|
    unless channels.include? fn
      STDERR.puts("removed #{fn} -- not configured")
      File.unlink fn
      next
    end
    cache = YAML.load_file(fn)
    dels = clean_cache(cache)
    File.open(fn, 'w') {|h| h.puts cache.to_yaml} if dels > 0
    count += dels
    config['cleaned'] = Vandaag
  end
  STDERR.puts("#{myname}: Removed #{count} entries") if count > 0

end

#clean_spool_dir ⇒ Object 



446
447
448
449
450
451
452
453
454
 
# File 'lib/xmltv/xmltv.rb', line 446

def clean_spool_dir
  xmlfiles = config_channels.map{|x| outputfile(x) }
  Dir["#{spooldir}/*.xml"].each do |fn|
    unless xmlfiles.include? fn
      STDERR.puts("removed #{fn} -- not configured")
      File.unlink fn
    end
  end
end

#config_channels ⇒ Object 



438
439
440
441
442
443
444
445
 
# File 'lib/xmltv/xmltv.rb', line 438

def config_channels
  rsl = config['channels']
  if rsl.empty?
    all_channels.keys
  else
    rsl
  end
end

#date_stats(chan, time) ⇒ Object 



587
588
589
 
# File 'lib/xmltv/xmltv.rb', line 587

def date_stats(chan, time)
  @days_av[chan].add(time.to_date)
end

#delete_channels_from_config ⇒ Object 



607
608
609
610
611
612
613
614
615
616
617
 
# File 'lib/xmltv/xmltv.rb', line 607

def delete_channels_from_config
  XmltvOptions.del.each do |item|
    if config['channels'].include?(item)
      config['channels'].delete(item)
      STDERR.puts("deleted #{item} #{channel_display(item)}") if XmltvOptions.verbose
    else
      STDERR.puts "#{item} not in config list" 
    end
  end
  save_config
end

#do_list(array) ⇒ Object 



621
622
623
624
625
 
# File 'lib/xmltv/xmltv.rb', line 621

def do_list(array)
  array.sort_by{|x| channel_display(x)}.each do |item|
    printline item
  end
end

#do_options ⇒ Object 



639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
 
# File 'lib/xmltv/xmltv.rb', line 639

def do_options
  if XmltvOptions.list
    list_config
  elsif XmltvOptions.available   
    list_all
  else
    if XmltvOptions.add
      add_channels_to_config
    end
    if XmltvOptions.del
      delete_channels_from_config
    end
    list_config
  end
  exit
end

#dump(*args) ⇒ Object 



582
583
584
585
586
 
# File 'lib/xmltv/xmltv.rb', line 582

def dump(*args)
  args.each do |arg|
    PP.pp(arg,STDERR)
  end
end

#fetch(url) ⇒ Object 



555
556
557
558
559
560
561
562
563
564
565
566
 
# File 'lib/xmltv/xmltv.rb', line 555

def fetch(url)
  @hits += 1
  tries = 0
  begin
    open(url) { |h| Hpricot(h) }
  rescue Errno::ECONNREFUSED, ::Timeout::Error, EOFError, Errno::ETIMEDOUT
    tries += 1
    sleep 2
    retry if tries <= 3
    raise BadSiteError.new
  end
end

#fix_times(pda) ⇒ Object 



515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
 
# File 'lib/xmltv/xmltv.rb', line 515

def fix_times(pda)
  errors = 0
  startwith = pda.size
  pda.delete_if do |x| 
    rsl = x.has_key?('stop') && x['stop'] <= x['start']
    reject('invalid', x) if rsl
    rsl
  end
  if pda.empty?
    return 0
  end
  starttimes = pda.sort do |x, y| 
    if (a = x['start']) != (b = y['start'])
      a <=> b
    else
      a = x.has_key?('stop') ? x['stop'] : x['start']
      b = y.has_key?('stop') ? y['stop'] : y['start']
      b <=> a
    end
  end
  starttimes.each_with_index do |entry, idx|
  #      dump(entry, '++++++++++++++') if entry.has_key?('title') && entry['title'] == 'NOS Studio Voetbal'
    nxt = starttimes[idx + 1]
    next unless nxt
    unless entry.has_key?('stop') && entry['stop'] <= nxt['start']
      errors += 1
      entry['stop'] = nxt['start'].dup
    end
  end
  unless starttimes[-1].has_key?('stop')
    starttimes[-1]['start'] = nil
  end
  pda.delete_if do |x| 
    rsl = x['start'].nil? ||  x['stop'] <= x['start']
    reject('removed', x) if rsl
    rsl
  end
  @rejects = startwith - pda.size
  errors
end

#get_channels ⇒ Object 



392
393
394
395
396
397
398
399
 
# File 'lib/xmltv/xmltv.rb', line 392

def get_channels
  clean_cache_dir
  if ! ARGV.empty?
    ARGV.replace(check_argv(ARGV))
  else
    config_channels
  end
end

#grab_channel(chan_id) ⇒ Object 



330
331
332
333
 
# File 'lib/xmltv/xmltv.rb', line 330

def grab_channel(chan_id)
  STDERR.puts "Grabber must implement grab_channel(chan_id)"
  exit
end

#list_all ⇒ Object 



635
636
637
638
 
# File 'lib/xmltv/xmltv.rb', line 635

def list_all
  puts "Available channels for #{myname}"
  do_list(all_channels.keys)
end

#list_config ⇒ Object 



626
627
628
629
630
631
632
633
634
 
# File 'lib/xmltv/xmltv.rb', line 626

def list_config
  if config['channels'].empty?
    explan = channel_list ? 'yet' : 'needed'
    puts "No configuration #{explan} for #{myname}"
  else
    puts "Configured channels for #{myname}"
    do_list(config['channels'])
  end
end

#load_cachefile(chan_id) ⇒ Object 



373
374
375
376
377
378
379
 
# File 'lib/xmltv/xmltv.rb', line 373

def load_cachefile(chan_id)
  begin
    YAML.load_file(cachefile(chan_id))
  rescue Errno::ENOENT
    Hash.new
  end
end

#load_channel_file(fn) ⇒ Object 



411
412
413
414
415
416
417
418
419
 
# File 'lib/xmltv/xmltv.rb', line 411

def load_channel_file(fn)
  if (fn)
    begin
      YAML.load_file(fn)
    rescue  Errno::ENOENT
      fetch_all_channels
    end
  end
end

#load_config_file ⇒ Object 



401
402
403
404
405
406
407
408
409
 
# File 'lib/xmltv/xmltv.rb', line 401

def load_config_file
  begin
    YAML.load_file(config_file_name)
  rescue
    h = Hash.new
    h['channels'] = []
    h
  end
end

#outputfile(chan_id) ⇒ Object 



361
362
363
 
# File 'lib/xmltv/xmltv.rb', line 361

def outputfile(chan_id)
  "#{spooldir}/#{channel_name(chan_id)}.xml"
end

#printline(it) ⇒ Object 



618
619
620
 
# File 'lib/xmltv/xmltv.rb', line 618

def printline(it)
  printf("%-5s %s\n", it, channel_display(it))
end

#proghash(entry, chan_id) ⇒ Object 



484
485
486
487
488
489
490
491
492
493
494
495
 
# File 'lib/xmltv/xmltv.rb', line 484

def proghash(entry, chan_id)
  progdata = Progdata.new
  entry.keys.each do |k|
    dtd = XmlWriter::Programme_dtd.assoc(k)
    if dtd && dtd.size == 1
      progdata[k] = entry[k]
    end
  end
  cnm = channel_name(chan_id)
  progdata['channel'] = MythTV[cnm] || cnm
  progdata
end

#reject(*args) ⇒ Object 



497
498
499
500
501
502
503
504
505
506
507
508
509
510
 
# File 'lib/xmltv/xmltv.rb', line 497

def reject(*args)
  File.open(reject_file_name, 'a') do |h|
    h.puts Time.now
    args.each do |arg|
      case arg
        when String, Time, Integer
          h.puts arg
        else
          PP.pp(arg, h)
      end
    end
    h.puts '===='
  end
end

#report(channel, nprogs, errors) ⇒ Object 



656
657
658
659
660
661
662
663
664
665
666
667
668
669
 
# File 'lib/xmltv/xmltv.rb', line 656

def report(channel, nprogs, errors)
  STDERR.printf("  %s %-6.6s %-20.20s %5d %5d %5d %5d %8d %5d\n",
    Time.now.strftime("%H:%M:%S"),
    channel,
    channel_display(channel),
    nprogs,
    @hits,
    @days_av[channel].size,
    errors,
    test(?s, outputfile(channel)),
    @rejects)
    
  @hits = 0
end

#run ⇒ Object 



671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
 
# File 'lib/xmltv/xmltv.rb', line 671

def run
  if XmltvOptions.version
    puts version
    exit
  end
  do_options if XmltvOptions.action
  if channel_list && ! test(?f, config_file_name)   # We don't want to grab _all_ channels, surely ?
    list_all
    puts "\n Do #{$PROGRAM_NAME} --add chn1,chn2,chn3... to create a configuration file"
    exit
  end
  ARGV.delete_if do |arg|
    begin
      send(arg)
      true
    rescue NoMethodError
      false
    end
  end
  trap('INT') do
    STDERR.puts "\nInterrupted\n"
    raise ArgumentError.new('interrupt')
#        exit!
  end
  
  begin
    channels = get_channels
    STDERR.printf("\n%-40s prgs   hts  days  errs   bytes    rej\n", "#{Date.today}: #{myname}")
    channels.each do |channel|
      next if XmltvOptions.only_new && test(?f, outputfile(channel))
      @days_av = Hash.new { |h,k| h[k] = Set.new }
      @rejects = 0
      writer = XmlWriter.new(self)
      unless check_channel(channel)
        STDERR.puts "No such channel #{channel}"
        next
      end
      begin  
        nprogs = grab_channel(channel)
        pda = transform(channel)
        errors = fix_times(pda)
      rescue Timeout::Error, Errno::ECONNRESET
        STDERR.puts "Timeout grab_channel#{channel}"
        next
      rescue BadChannelError
        STDERR.puts "Zie #{reject_file_name}"
        next
      end
      writer.write_file(pda, channel)
      report(channel, nprogs, errors)
    end
  rescue SystemExit
  rescue ValidateError => exc
    STDERR.puts exc.class, exc.message, exc.backtrace
  rescue ::Exception => exc
    STDERR.puts exc.class, exc.message, exc.backtrace
    if XmltvOptions.mailto
      MailSender.sendmail(XmltvOptions.mailto, "Xmltv exception", [exc.class, exc.message, exc.backtrace].join("\n"))
    end
    raise
  ensure
  end
  save_config
end

#save(*args) ⇒ Object 



572
573
574
575
576
577
578
579
580
 
# File 'lib/xmltv/xmltv.rb', line 572

def save(*args)
  file = "/tmp/xmltv_exception.#{$$}"
  args.each_with_index do |el, i|
    File.open("#{file}-#{i}", 'w') do |h|
      h.puts el
    end
  end
  STDERR.puts "See #{file}-*"
end

#save_config ⇒ Object 



420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
 
# File 'lib/xmltv/xmltv.rb', line 420

def save_config
  loop do
    if test(?f, config_file_name)
      File.open(config_file_name, 'w') { |h| h.puts config.to_yaml }
      return
    end
    dir = File.dirname(config_file_name)
    if test(?d, dir)
      FileUtils.touch config_file_name
    elsif test(?e, dir)
      raise "#{dir} exists, but isn't a directory"
    else
      FileUtils.mkdir_p dir
    end
  end
end

#save_object(obj, filename) ⇒ Object 



567
568
569
570
571
 
# File 'lib/xmltv/xmltv.rb', line 567

def save_object(obj, filename)
  File.open(filename, 'w') do |h|
    h.puts obj.to_yaml
  end
end

#transform(chan_id) ⇒ Object 



334
335
336
337
 
# File 'lib/xmltv/xmltv.rb', line 334

def transform(chan_id)
  STDERR.puts "Grabber must implement transform(chan_id)"
  exit
end

#version ⇒ Object 



357
358
359
 
# File 'lib/xmltv/xmltv.rb', line 357

def version
  "XMLTV::Grabber version 0.8"
end