Class: Geoptima::Data

Inherits:
Object
  • Object
show all
Includes:
ErrorCounter
Defined in:
lib/geoptima/data.rb

Overview

The Geoptima::Data is an entire JSON file of events

Instance Attribute Summary collapse

Attributes included from ErrorCounter

#errors

Class Method Summary collapse

Instance Method Summary collapse

Methods included from ErrorCounter

#combine_errors, #report_errors

Constructor Details

#initialize(path) ⇒ Data

Returns a new instance of Data.



672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
 
# File 'lib/geoptima/data.rb', line 672

def initialize(path)
  @path = path
  @name = File.basename(path)
#      @json = JSON.parse(File.read(path))
  @json = MultiJson.decode(File.read(path))
  @fields = {}
  @errors = {}
  if $debug
    puts "Read Geoptima: #{geoptima.to_json}"
    puts "\tSubscriber: #{subscriber.to_json}"
    puts "\tIMSI: #{self['imsi']}"
    puts "\tIMEI: #{self['imei']}"
    puts "\tMCC: #{self['MCC']}"
    puts "\tMNC: #{self['MNC']}"
    puts "\tStart: #{start}"
  end
end

Instance Attribute Details

#count ⇒ Object (readonly)

Returns the value of attribute count.



671
672
673
 
# File 'lib/geoptima/data.rb', line 671

def count
  @count
end

#json ⇒ Object (readonly)

Returns the value of attribute json.



671
672
673
 
# File 'lib/geoptima/data.rb', line 671

def json
  @json
end

#name ⇒ Object (readonly)

Returns the value of attribute name.



671
672
673
 
# File 'lib/geoptima/data.rb', line 671

def name
  @name
end

#path ⇒ Object (readonly)

Returns the value of attribute path.



671
672
673
 
# File 'lib/geoptima/data.rb', line 671

def path
  @path
end

Class Method Details

.max_start ⇒ Object 



726
727
728
 
# File 'lib/geoptima/data.rb', line 726

def self.max_start
  @@max_start ||= MAX_VALID_DATETIME
end

.min_start ⇒ Object 



723
724
725
 
# File 'lib/geoptima/data.rb', line 723

def self.min_start
  @@min_start ||= MIN_VALID_DATETIME
end

Instance Method Details

#[](key) ⇒ Object 



714
715
716
 
# File 'lib/geoptima/data.rb', line 714

def [](key)
  @fields[key] ||= subscriber[key] || subscriber[key.downcase]
end

#events ⇒ Object 



729
730
731
 
# File 'lib/geoptima/data.rb', line 729

def events
  @events ||= make_events
end

#events_names ⇒ Object 



738
739
740
 
# File 'lib/geoptima/data.rb', line 738

def events_names
  events.keys.sort
end

#find_first_and_last(events_data) ⇒ Object 



832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
 
# File 'lib/geoptima/data.rb', line 832

def find_first_and_last(events_data)
  @first = nil
  @last = nil
  events_data.each do |event_type,data|
    if data.length > 0
      @first ||= data[0]
      @last ||= data[-1]
      @first = data[0] if(@first && @first.time > data[0].time)
      @last = data[-1] if(@last && @last.time < data[-1].time)
    end
  end
  if $debug
    puts "For data: #{self}"
    puts "\tFirst event: #{@first}"
    puts "\tLast event:  #{@last}"
  end
end

#first ⇒ Object 



732
733
734
 
# File 'lib/geoptima/data.rb', line 732

def first
  events && @first
end

#geoptima ⇒ Object 



699
700
701
 
# File 'lib/geoptima/data.rb', line 699

def geoptima
  @geoptima ||= json['geoptima']
end

#id ⇒ Object 



711
712
713
 
# File 'lib/geoptima/data.rb', line 711

def id
  @id ||= self['id'] || self['udid'] || self['imei']
end

#imei ⇒ Object 



708
709
710
 
# File 'lib/geoptima/data.rb', line 708

def imei
  @imei ||= self['imei']
end

#incr_error(name) ⇒ Object 



689
690
691
692
 
# File 'lib/geoptima/data.rb', line 689

def incr_error(name)
  @errors[name] ||= 0
  @errors[name] += 1
end

#last ⇒ Object 



735
736
737
 
# File 'lib/geoptima/data.rb', line 735

def last
  events && @last
end

#make_events ⇒ Object 



753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
 
# File 'lib/geoptima/data.rb', line 753

def make_events
  @count = 0
  @events_metadata = make_hash('events-metadata')
  events_data = {}
  geoptima['events'].each do |data|
    events = data['values']
    event_type = data.keys.reject{|k| k=~/values/}[0]
    event_count = data[event_type].to_i
    header = @events_metadata[event_type]
    # If the JSON is broken (known bug on some releases of the iPhone app)
    # Then get the header information from a list of known headers
    unless header
      puts "No header found for '#{event_type}', trying known Geoptima headers"
      header = Event::KNOWN_HEADERS[event_type]
      puts "Found known header '#{event_type}' => #{header.inspect}" if(header)
    end
    # Double-check the header length matches a multiple of the data length
    if header
      mismatch_records = events.length - header.length * event_count
      if mismatch_records != 0
        puts "'#{event_type}' header length #{header.length} incompatible with data length #{events.length} and record count #{event_count}"
        proposed_header = header
        header = nil
        incr_error "Metadata mismatch"
        if events.length == proposed_header.length * event_count * 2 && event_type == 'roundtrip'
          incr_error "#4593 iPhone roundtrip event counts"
          event_count *= 2
          header = proposed_header
        elsif Event::ALT_HEADERS.keys.grep(event_type).length>0
          incr_error "#{Event::HEADER_BUGS[event_type]} #{event_type}"
          [Event::KNOWN_HEADERS[event_type],*(Event::ALT_HEADERS[event_type])].each do |alt_header|
            puts "Trying alternative header: #{alt_header.inspect}" if($debug)
            if alt_header && (events.length == alt_header.length * event_count)
              puts "\tAlternative header length matches: #{alt_header.inspect}" if($debug)
              records_valid = (0...[10,event_count].min).inject(true) do |vt,ri|
                timeoffset = events[ri*alt_header.length]
                vt &&= timeoffset.is_a?(Fixnum)
              end
              if records_valid
                header = alt_header
                puts "Found alternative header that matches #{event_type}: #{header.join(',')}"
                break
              end
            end
          end
        end
      end
    else
      puts "No header found for event type: #{event_type}"
    end
    # Now process the single long data array into a list of events with timestamps
    if header
      events_data[event_type] = (0...event_count).inject([]) do |a,block|
        index = header.length * block
        record = events[index...(index+header.length)]
        if record && record.length == header.length
          @count += 1
          event = Event.new(self,start,event_type,header,record,a[-1])
          combine_errors event
          puts "About to add new event #{event} to existing list of #{a.length} events (previous: #{a[-1] && a[-1].time})" if($debug)
          a << event
        else
          puts "Invalid '#{event_type}' data block #{block}: #{record.inspect}"
          incr_error "Invalid data block"
          break a
        end
      end
      if $debug
        puts "Have '#{event_type}' event data:"
        puts "\t#{header.join("\t")}"
        events_data[event_type].each do |d|
          puts "\t#{d.data.join("\t")}"
        end
      end
    end
  end
  find_first_and_last(events_data)
  events_data
end

#make_hash(name) ⇒ Object 



741
742
743
744
745
746
747
748
749
750
751
752
 
# File 'lib/geoptima/data.rb', line 741

def make_hash(name)
  puts "About to process json hash: #{geoptima[name].to_json}" if($debug)
  geoptima[name].inject({}) do |a,md|
    if md.respond_to? 'keys'
      key = md.keys[0]
      a[key]=md[key]
    else
      puts "Invalid hash format for '#{name}': #{md.to_json[0..70]}..."
    end
    a
  end
end

#puts(line) ⇒ Object 



696
697
698
 
# File 'lib/geoptima/data.rb', line 696

def puts line
  Kernel.puts "#{name}: #{line}"
end

#start ⇒ Object 



717
718
719
 
# File 'lib/geoptima/data.rb', line 717

def start
  @start ||= subscriber['start'] && DateTime.parse(subscriber['start'].gsub(/Asia\/Bangkok/,'GMT+7'))#.gsub(/Mar 17 2044/,'Feb 14 2012'))
end

#subscriber ⇒ Object 



705
706
707
 
# File 'lib/geoptima/data.rb', line 705

def subscriber
  @subscriber ||= geoptima['subscriber']
end

#to_s ⇒ Object 



693
694
695
 
# File 'lib/geoptima/data.rb', line 693

def to_s
  json.to_json[0..100]
end

#valid? ⇒ Boolean

Returns:

  • (Boolean) 


720
721
722
 
# File 'lib/geoptima/data.rb', line 720

def valid?
  start && start >= (Data.min_start-1) && start < Data.max_start
end

#version ⇒ Object 



702
703
704
 
# File 'lib/geoptima/data.rb', line 702

def version
  @version ||= geoptima['Version'] || geoptima['version']
end