Class: CharDet::HebrewProber

Inherits:

CharSetProber

• Object
• CharSetProber
• CharDet::HebrewProber

Defined in:

lib/rchardet/hebrewprober.rb

Instance Attribute Summary

Attributes inherited from CharSetProber

#active

Instance Method Summary

• #charset_name ⇒ Object
• #feed(aBuf) ⇒ Object
• #initialize ⇒ HebrewProber constructor

  A new instance of HebrewProber.

• #is_final(c) ⇒ Object
• #is_non_final(c) ⇒ Object
• #reset ⇒ Object
• #set_model_probers(logicalProber, visualProber) ⇒ Object
• #state ⇒ Object

Methods inherited from CharSetProber

#confidence, #filter_high_bit_only, #filter_with_english_letters, #filter_without_english_letters, #result

Constructor Details

#initialize ⇒ HebrewProber

Returns a new instance of HebrewProber.

# File 'lib/rchardet/hebrewprober.rb', line 151

def initialize
  super()
  @_mLogicalProber = nil
  @_mVisualProber = nil
  reset()
end

Instance Method Details

#charset_name ⇒ Object

# File 'lib/rchardet/hebrewprober.rb', line 252

def charset_name
  # Make the decision: is it Logical or Visual?
  # If the final letter score distance is dominant enough, rely on it.
  finalsub = @_mFinalCharLogicalScore - @_mFinalCharVisualScore
  if finalsub >= MIN_FINAL_CHAR_DISTANCE
    return LOGICAL_HEBREW_NAME
  end
  if finalsub <= -MIN_FINAL_CHAR_DISTANCE
    return VISUAL_HEBREW_NAME
  end

  # It's not dominant enough, try to rely on the model scores instead.
  modelsub = @_mLogicalProber.confidence() - @_mVisualProber.confidence()
  if modelsub > MIN_MODEL_DISTANCE
    return LOGICAL_HEBREW_NAME
  end
  if modelsub < -MIN_MODEL_DISTANCE
    return VISUAL_HEBREW_NAME
  end

  # Still no good, back to final letter distance, maybe it'll save the day.
  if finalsub < 0.0
    return VISUAL_HEBREW_NAME
  end

  # (finalsub > 0 - Logical) or (don't know what to do) default to Logical.
  return LOGICAL_HEBREW_NAME
end

#feed(aBuf) ⇒ Object

# File 'lib/rchardet/hebrewprober.rb', line 192

def feed(aBuf)
  # Final letter analysis for logical-visual decision.
  # Look for evidence that the received buffer is either logical Hebrew or
  # visual Hebrew.
  # The following cases are checked:
  # 1) A word longer than 1 letter, ending with a final letter. This is an
  #    indication that the text is laid out "naturally" since the final letter
  #    really appears at the end. +1 for logical score.
  # 2) A word longer than 1 letter, ending with a Non-Final letter. In normal
  #    Hebrew, words ending with Kaf, Mem, Nun, Pe or Tsadi, should not end with
  #    the Non-Final form of that letter. Exceptions to this rule are mentioned
  #    above in isNonFinal(). This is an indication that the text is laid out
  #    backwards. +1 for visual score
  # 3) A word longer than 1 letter, starting with a final letter. Final letters
  #    should not appear at the beginning of a word. This is an indication that
  #    the text is laid out backwards. +1 for visual score.
  #
  # The visual score and logical score are accumulated throughout the text and
  # are finally checked against each other in GetCharSetName().
  # No checking for final letters in the middle of words is done since that case
  # is not an indication for either Logical or Visual text.
  #
  # We automatically filter out all 7-bit characters (replace them with spaces)
  # so the word boundary detection works properly. [MAP]

  if state() == ENotMe
    # Both model probers say it's not them. No reason to continue.
    return ENotMe
  end

  aBuf = filter_high_bit_only(aBuf)

  for cur in aBuf.split(' ')
    if cur == ' '
      # We stand on a space - a word just ended
      if @_mBeforePrev != ' '
        # next-to-last char was not a space so self._mPrev is not a 1 letter word
        if is_final(@_mPrev)
          # case (1) [-2:not space][-1:final letter][cur:space]
          @_mFinalCharLogicalScore += 1
        elsif is_non_final(@_mPrev)
          # case (2) [-2:not space][-1:Non-Final letter][cur:space]
          @_mFinalCharVisualScore += 1
        end
      end
    else
      # Not standing on a space
      if (@_mBeforePrev == ' ') and (is_final(@_mPrev)) and (cur != ' ')
        # case (3) [-2:space][-1:final letter][cur:not space]
        @_mFinalCharVisualScore += 1
      end
    end
    @_mBeforePrev = @_mPrev
    @_mPrev = cur
  end

  # Forever detecting, till the end or until both model probers return eNotMe (handled above)
  return EDetecting
end

#is_final(c) ⇒ Object

# File 'lib/rchardet/hebrewprober.rb', line 174

def is_final(c)
  return [FINAL_KAF, FINAL_MEM, FINAL_NUN, FINAL_PE, FINAL_TSADI].include?(c)
end

#is_non_final(c) ⇒ Object

# File 'lib/rchardet/hebrewprober.rb', line 178

def is_non_final(c)
  # The normal Tsadi is not a good Non-Final letter due to words like
  # 'lechotet' (to chat) containing an apostrophe after the tsadi. This
  # apostrophe is converted to a space in FilterWithoutEnglishLetters causing
  # the Non-Final tsadi to appear at an end of a word even though this is not
  # the case in the original text.
  # The letters Pe and Kaf rarely display a related behavior of not being a
  # good Non-Final letter. Words like 'Pop', 'Winamp' and 'Mubarak' for
  # example legally end with a Non-Final Pe or Kaf. However, the benefit of
  # these letters as Non-Final letters outweighs the damage since these words
  # are quite rare.
  return [NORMAL_KAF, NORMAL_MEM, NORMAL_NUN, NORMAL_PE].include?(c)
end

#reset ⇒ Object

# File 'lib/rchardet/hebrewprober.rb', line 158

def reset
  @_mFinalCharLogicalScore = 0
  @_mFinalCharVisualScore = 0
  # The two last characters seen in the previous buffer,
  # mPrev and mBeforePrev are initialized to space in order to simulate a word
  # delimiter at the beginning of the data
  @_mPrev = ' '
  @_mBeforePrev = ' '
  # These probers are owned by the group prober.
end

#set_model_probers(logicalProber, visualProber) ⇒ Object

# File 'lib/rchardet/hebrewprober.rb', line 169

def set_model_probers(logicalProber, visualProber)
  @_mLogicalProber = logicalProber
  @_mVisualProber = visualProber
end

#state ⇒ Object

# File 'lib/rchardet/hebrewprober.rb', line 281

def state
  # Remain active as long as any of the model probers are active.
  if (@_mLogicalProber.state() == ENotMe) and (@_mVisualProber.state() == ENotMe)
    return ENotMe
  end
  return EDetecting
end