Module: FeedParser

Defined in:

lib/rfeedparser.rb,
lib/rfeedparser/parsers.rb

Defined Under Namespace

Classes: CharacterEncodingOverride, CharacterEncodingUnknown, LooseFeedParser, NonXMLContentType, StrictFeedParser, ThingsNobodyCaresAboutButMe, UndeclaredNamespace

Constant Summary

Version =

"0.9.932"

License =

"""Copyright (c) 2002-2006, Mark Pilgrim, All rights reserved.

Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS'
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE."""

Author =

"Jeff Hodges <http://somethingsimilar.com>"

Copyright_Holder =

"Mark Pilgrim <http://diveintomark.org/>"

Contributors =

[  "Jason Diamond <http://injektilo.org/>",
  "John Beimler <http://john.beimler.org/>",
  "Fazal Majid <http://www.majid.info/mylos/weblog/>",
  "Aaron Swartz <http://aaronsw.com/>",
  "Kevin Marks <http://epeus.blogspot.com/>"
]

USER_AGENT =

HTTP “User-Agent” header to send to servers when downloading feeds. If you are embedding feedparser in a larger application, you should change this to your application name and URL.

"UniversalFeedParser/%s +http://feedparser.org/" % @version

ACCEPT_HEADER =

HTTP “Accept” header to send to servers when downloading feeds. If you don’t want to send an Accept header, set this to None.

"application/atom+xml,application/rdf+xml,application/rss+xml,application/x-netcdf,application/xml;q=0.9,text/xml;q=0.2,*/*;q=0.1"

SUPPORTED_VERSIONS =

{'' => 'unknown',
  'rss090' => 'RSS 0.90',
  'rss091n' => 'RSS 0.91 (Netscape)',
  'rss091u' => 'RSS 0.91 (Userland)',
  'rss092' => 'RSS 0.92',
  'rss093' => 'RSS 0.93',
  'rss094' => 'RSS 0.94',
  'rss20' => 'RSS 2.0',
  'rss10' => 'RSS 1.0',
  'rss' => 'RSS (unknown version)',
  'atom01' => 'Atom 0.1',
  'atom02' => 'Atom 0.2',
  'atom03' => 'Atom 0.3',
  'atom10' => 'Atom 1.0',
  'atom' => 'Atom (unknown version)',
  'cdf' => 'CDF',
  'hotrss' => 'Hot RSS'
}

Class Method Summary

• .parse(furi, options = {}) ⇒ Object

Class Method Details

.parse(furi, options = {}) ⇒ Object

# File 'lib/rfeedparser.rb', line 144

def parse(furi, options = {})
  furi.strip!
  # Parse a feed from a URL, file, stream or string
  $compatible = options[:compatible].nil? ? $compatible : options[:compatible]# Use the default compatibility if compatible is nil
  strictklass = options[:strict] || StrictFeedParser
  looseklass = options[:loose] || LooseFeedParser
  result = FeedParserDict.new
  result['feed'] = FeedParserDict.new
  result['entries'] = []
  if options[:modified]
    options[:modified] = Time.parse(options[:modified]).utc.rfc2822 
    # FIXME this ignores all of our time parsing work.  Does it matter?
  end
  result['bozo'] = false
  handlers = options[:handlers]
  if handlers.class != Array # FIXME why does this happen?
    handlers = [handlers]
  end

  begin
    parsed_furi = ForgivingURI.parse(furi)
    if [nil, "file"].include? parsed_furi.scheme
      $stderr << "Opening local file #{furi}\n" if $debug
      f = open(parsed_furi.path) # OpenURI doesn't behave well when passing HTTP options to a file.
    else
      # And when you do pass them, make sure they aren't just nil (this still true?)
      newd = {}
      newd["If-None-Match"] = options[:etag] unless options[:etag].nil?
      newd["If-Modified-Since"] = options[:modified] unless options[:modified].nil?
      newd["User-Agent"] = (options[:agent] || USER_AGENT).to_s 
      newd["Referer"] = options[:referrer] unless options[:referrer].nil?
      newd["Content-Location"] = options[:content_location] unless options[:content_location].nil?
      newd["Content-Language"] = options[:content_language] unless options[:content_language].nil?                    
      newd["Content-type"] = options[:content_type] unless options[:content_type].nil?
      
      f = open(furi, newd)
      
    end

    data = f.read
    f.close 
    
  rescue OpenURI::HTTPError => e
    unless e.to_s[0..2].to_i == 0
      result['status'] = e.to_s[0..2].to_i
      data = ''
      f = nil
    end
    
  rescue => e
    $stderr << "Rescued in parse: "+e.to_s+"\n" if $debug # My addition
    result['bozo'] = true
    result['bozo_exception'] = e
    data = ''
    f = nil
  end
  if f.respond_to?(:meta)
    result['etag'] = f.meta['etag']
    result['modified'] = f.meta['modified']
    result['url'] = f.base_uri.to_s
    result['status'] ||= f.status[0].to_i
    result['headers'] = f.meta
  end


  # there are four encodings to keep track of:
  # - http_encoding is the encoding declared in the Content-Type HTTP header
  # - xml_encoding is the encoding declared in the <?xml declaration
  # - sniffed_encoding is the encoding sniffed from the first 4 bytes of the XML data
  # - result['encoding'] is the actual encoding, as per RFC 3023 and a variety of other conflicting specifications
  http_headers = result['headers'] || {}
  result['encoding'], http_encoding, xml_encoding, sniffed_xml_encoding, acceptable_content_type =
  self.getCharacterEncoding(f,data)

  if not http_headers.blank? and not acceptable_content_type
    unless http_headers['content-type'].nil?
      bozo_message = "#{http_headers['content-type']} is not an XML media type"
    else
      bozo_message = 'no Content-type specified'
    end
    result['bozo'] = true
    result['bozo_exception'] = NonXMLContentType.new(bozo_message) # I get to care about this, cuz Mark says I should.
  end
  result['version'], data = self.stripDoctype(data)
  
  baseuri = http_headers['content-location'] || result['href']
  baselang = http_headers['content-language']

  # if server sent 304, we're done
  if result['status'] == 304
    result['version'] = ''
    result['debug_message'] = "The feed has not changed since you last checked, " +
    "so the server sent no data. This is a feature, not a bug!"
    return result
  end

  # if there was a problem downloading, we're done
  if data.nil? or data.empty?
    return result
  end

  # determine character encoding
  use_strict_parser = false
  known_encoding = false
  tried_encodings = []
  proposed_encoding = nil
  # try: HTTP encoding, declared XML encoding, encoding sniffed from BOM
  [result['encoding'], xml_encoding, sniffed_xml_encoding].each do |proposed_encoding|
    next if proposed_encoding.nil? or proposed_encoding.empty?
    next if tried_encodings.include? proposed_encoding
    tried_encodings << proposed_encoding
    begin
      data = self.toUTF8(data, proposed_encoding)
      known_encoding = use_strict_parser = true
      break
    rescue
    end
  end
  # if no luck and we have auto-detection library, try that
  if not known_encoding and $chardet
    begin 
      proposed_encoding = CharDet.detect(data)['encoding']
      if proposed_encoding and not tried_encodings.include?proposed_encoding
        tried_encodings << proposed_encoding
        data = self.toUTF8(data, proposed_encoding)
        known_encoding = use_strict_parser = true
      end
    rescue
    end
  end



  # if still no luck and we haven't tried utf-8 yet, try that
  if not known_encoding and not tried_encodings.include?'utf-8'
    begin
      proposed_encoding = 'utf-8'
      tried_encodings << proposed_encoding
      data = self.toUTF8(data, proposed_encoding)
      known_encoding = use_strict_parser = true
    rescue
    end
  end
  # if still no luck and we haven't tried windows-1252 yet, try that
  if not known_encoding and not tried_encodings.include?'windows-1252'
    begin
      proposed_encoding = 'windows-1252'
      tried_encodings << proposed_encoding
      data = self.toUTF8(data, proposed_encoding)
      known_encoding = use_strict_parser = true
    rescue
    end
  end
  
  # NOTE this isn't in FeedParser.py 4.1
  # if still no luck and we haven't tried iso-8859-2 yet, try that.
  #if not known_encoding and not tried_encodings.include?'iso-8859-2'
  #  begin
  #    proposed_encoding = 'iso-8859-2'
  #    tried_encodings << proposed_encoding
  #    data = self.toUTF8(data, proposed_encoding)
  #    known_encoding = use_strict_parser = true
  #  rescue
  #  end
  #end


  # if still no luck, give up
  if not known_encoding
    result['bozo'] = true
    result['bozo_exception'] = CharacterEncodingUnknown.new("document encoding unknown, I tried #{result['encoding']}, #{xml_encoding}, utf-8 and windows-1252 but nothing worked")
    result['encoding'] = ''
  elsif proposed_encoding != result['encoding']
    result['bozo'] = true
    result['bozo_exception'] = CharacterEncodingOverride.new("documented declared as #{result['encoding']}, but parsed as #{proposed_encoding}")
    result['encoding'] = proposed_encoding
  end

  if use_strict_parser
    # initialize the SAX parser
    saxparser = XML::SAX::Helpers::ParserFactory.makeParser("XML::Parser::SAXDriver")
    feedparser = strictklass.new(baseuri, baselang, 'utf-8')
    saxparser.setDocumentHandler(feedparser)
    saxparser.setDTDHandler(feedparser)
    saxparser.setEntityResolver(feedparser)
    saxparser.setErrorHandler(feedparser)

    inputdata = XML::SAX::InputSource.new('parsedfeed')
    inputdata.setByteStream(StringIO.new(data))
    begin
      saxparser.parse(inputdata)
    rescue Exception => parseerr # resparse
      if $debug
        $stderr << "xml parsing failed\n"
        $stderr << parseerr.to_s+"\n" # Hrmph.
      end
      result['bozo'] = true
      result['bozo_exception'] = feedparser.exc || e 
      use_strict_parser = false
    end
  end
  if not use_strict_parser
    feedparser = looseklass.new(baseuri, baselang, (known_encoding and 'utf-8' or ''))
    feedparser.parse(data)
    $stderr << "Using LooseFeed\n\n" if $debug
  end
  result['feed'] = feedparser.feeddata
  result['entries'] = feedparser.entries
  result['version'] = result['version'] || feedparser.version
  result['namespaces'] = feedparser.namespacesInUse
  return result
end