Class: MediaWikiLexer

Inherits:

Object

• Object
• MediaWikiLexer

Defined in:

lib/mediacloth/mediawikilexer.rb

Defined Under Namespace

Classes: LexerTable

Constant Summary

INLINE_ELEMENTS =

[:LINK, :INTLINK, :BOLD, :ITALIC]

BLOCK_ELEMENTS =

[:PARA, :PRE, :PREINDENT, :UL, :OL, :DL, :LI, :SECTION, :TABLE, :ROW, :CELL, :HEAD]

PARA_BREAK_ELEMENTS =

[:UL, :OL, :DL, :PRE, :PREINDENT, :PASTE_START, :SECTION, :TABLE, :HLINE, :KEYWORD]

NAME_CHAR_TABLE =

(0 .. 255).collect{|n| n.chr =~ /[a-zA-Z0-9_\-]/ ? true : false}

TOKEN_CHAR_TABLE =

(0 .. 255).collect{|n| n.chr =~ /[a-zA-Z0-9_\-.;:?&@~=#%\/]/ ? true : false}

PUNCTUATION_CHAR_TABLE =

(0 .. 255).collect{|n| n.chr =~ /[\.,;:\-?]/ ? true : false}

HTML_TAGS =

%w{ a abbr acronym address applet area b base basefont bdo big blockquote body br
button caption center cite code col colgroup dd del dir div dfn dl dt em fieldset font form frame
frameset h1 h2 h3 h4 h5 h6 head hr html i iframe img input ins isindex kbd label legend li link map
menu meta noframes noscript object ol optgroup option p param pre q s samp script select small span
strike strong style sub sup table tbody td textarea tfoot th thead title tr tt u ul var xmp }

WIKI_TAGS =

%w{ nowiki math paste }

TAGS_WITHOUT_CLOSE_TAG =

%w{ br hr img }

Instance Attribute Summary

• #cursor ⇒ Object readonly

  Returns the value of attribute cursor.

• #tokens ⇒ Object readonly

  Returns the value of attribute tokens.

Instance Method Summary

• #initialize ⇒ MediaWikiLexer constructor

  A new instance of MediaWikiLexer.

• #lex ⇒ Object

  Returns the next token from the stream.

• #tokenize(input) ⇒ Object

Constructor Details

#initialize ⇒ MediaWikiLexer

Returns a new instance of MediaWikiLexer.

# File 'lib/mediacloth/mediawikilexer.rb', line 84

def initialize
  # Current position in token list
  @position = 0
  
  # Lexer table of methods that handle only formatting, e.g. bold or italicized
  # text; or spans of XHTML, or wiki-escape, markup
  @formatting_lexer_table = {}
  @formatting_lexer_table["'"] = method(:match_quote)
  @formatting_lexer_table["<"] = method(:match_left_angle)
  @formatting_lexer_table["&"] = method(:match_ampersand)
  @formatting_lexer_table["{"] = method(:match_left_curly)
  
  # Lexer table of methods that handle everything that may occur in-line in
  # addition to formatting, i.e. links and signatures
  @inline_lexer_table = @formatting_lexer_table.dup
  @inline_lexer_table["["] = method(:match_left_square)
  @inline_lexer_table["~"] = method(:match_tilde)
  @inline_lexer_table["h"] = method(:match_h_char)
      
  # Default lexer table, which includes all in-line formatting and links, plus
  # methods that handle constructs that begin on a newline
  @default_lexer_table = @inline_lexer_table.dup
  @default_lexer_table[" "] = method(:match_space)
  @default_lexer_table["="] = method(:match_equal)
  @default_lexer_table["*"] = method(:match_star)
  @default_lexer_table["#"] = method(:match_hash)
  @default_lexer_table[":"] = method(:match_colon)
  @default_lexer_table[";"] = method(:match_semicolon)
  @default_lexer_table["-"] = method(:match_dash)
  @default_lexer_table["_"] = method(:match_underscore)
  @default_lexer_table["\n"] = method(:match_newline)
  @default_lexer_table["\r"] = method(:match_newline)
  
  # Lexer table used inside spans of markup, wherein spans of newlines are not
  # automatically treated as paragraphs.
  @markup_lexer_table = @default_lexer_table.dup
  @markup_lexer_table["\n"] = nil
  @markup_lexer_table["\r"] = nil
  
  # Lexer table used inside of headings
  @heading_lexer_table = @inline_lexer_table.dup
  @heading_lexer_table["="] = method(:match_equal_in_heading)
  @heading_lexer_table["\n"] = method(:match_newline_in_heading)
  
  # Lexer table used inside the left half of an external link
  @link_lexer_table = {}
  @link_lexer_table["]"] = method(:match_right_square_in_link)
  @link_lexer_table["\n"] = method(:match_newline_in_link)
  @link_lexer_table["\r"] = method(:match_newline_in_link)
  @link_lexer_table[" "] = method(:match_space_in_link)
  
  # Lexer table used inside the right half of an external link, or the right
  # half of an internal link
  @link_opt_lexer_table = @inline_lexer_table.dup
  @link_opt_lexer_table["]"] = method(:match_right_square_in_link)
  @link_opt_lexer_table["\n"] = method(:match_newline_in_link)
  @link_opt_lexer_table["\r"] = method(:match_newline_in_link)
  
  # Lexer table used inside the left half of an internal link or internal
  # resource link
  @intlink_lexer_table = {}
  @intlink_lexer_table["]"] = method(:match_right_square_in_intlink)
  @intlink_lexer_table["\r"] = method(:match_newline_in_intlink)
  @intlink_lexer_table["\n"] = method(:match_newline_in_intlink)
  @intlink_lexer_table[":"] = method(:match_colon_in_intlink)
  @intlink_lexer_table["|"] = method(:match_pipe_in_intlink)
  @intlink_lexer_table["C"] = method(:match_c_char_in_intlink)
  
  # Lexer table used inside the category name of the left half of an 
  # internal link
  @intlink_cat_lexer_table = {}
  @intlink_cat_lexer_table["]"] = method(:match_right_square_in_intlink)
  @intlink_cat_lexer_table["\r"] = method(:match_newline_in_intlink)
  @intlink_cat_lexer_table["\n"] = method(:match_newline_in_intlink)
  @intlink_cat_lexer_table["|"] = method(:match_pipe_in_intlink)    
  
  # Lexer table used inside the right half of an internal link
  @intlink_opt_lexer_table = @formatting_lexer_table.dup
  @intlink_opt_lexer_table["]"] = method(:match_right_square_in_intlink)
  @intlink_opt_lexer_table["\n"] = method(:match_newline_in_intlink)
  @intlink_opt_lexer_table["\r"] = method(:match_newline_in_intlink)
  
  # Lexer table used inside the right half of an internal resource link
  @resourcelink_opt_lexer_table = @inline_lexer_table.dup
  @resourcelink_opt_lexer_table["]"] = method(:match_right_square_in_intlink)
  @resourcelink_opt_lexer_table["\n"] = method(:match_newline_in_intlink)
  @resourcelink_opt_lexer_table["\r"] = method(:match_newline_in_intlink)
  @resourcelink_opt_lexer_table["|"] = method(:match_pipe_in_intlink)
  
  # Lexer table used to parse tables
  @table_lexer_table = @inline_lexer_table.dup
  @table_lexer_table["*"] = method(:match_star)
  @table_lexer_table["#"] = method(:match_hash)
  @table_lexer_table["|"] = method(:match_pipe_in_table)
  @table_lexer_table["!"] = method(:match_bang_in_table)
  @table_lexer_table["{"] = method(:match_left_curly)
  @table_lexer_table[" "] = method(:match_space)
  
  # Lexer table used to parse ordered and unordered list items (which may nest)
  @items_lexer_table = @inline_lexer_table.dup
  @items_lexer_table["\n"] = method(:match_newline_in_items)
  
  # Lexer table used to parse entries in a definition list (which may not nest)
  @entries_lexer_table = @inline_lexer_table.dup
  @entries_lexer_table["\n"] = method(:match_newline_in_entries)
  @entries_lexer_table[":"] = method(:match_colon_in_entries)
  
  # Lexer table used inside spans of indented text
  @indent_lexer_table = @inline_lexer_table.dup
  @indent_lexer_table["\n"] = method(:match_newline_in_indent)
  
  # Lexer table used inside spans of pre-formatted text
  @pre_lexer_table = {}
  @pre_lexer_table["<"] = method(:match_left_angle_in_pre)
      
  # Lexer table used inside spans of <code>
  @code_lexer_table = @inline_lexer_table.dup
  @code_lexer_table[" "] = method(:match_space_in_code)
  @code_lexer_table["<"] = method(:match_left_angle_in_code)

  # Lexer table used when inside spans of wiki-escaped text
  @nowiki_lexer_table = {}
  @nowiki_lexer_table["<"] = method(:match_left_angle_in_nowiki)

  @paste_lexer_table = {}
  @paste_lexer_table["<"] = method(:match_left_angle_in_paste)
  @paste_lexer_table["\n"] = method(:match_newline_in_paste)
  @paste_lexer_table["\r"] = method(:match_newline_in_paste)

  # Lexer table used when inside spans of math
  @math_lexer_table = {}
  @math_lexer_table["<"] = method(:match_left_angle_in_math)
      
  # Lexer table used when inside a wiki template inclusion
  @template_lexer_table = {}
  @template_lexer_table["{"] = method(:match_left_curly_in_template)
  @template_lexer_table["|"] = method(:match_pipe_in_template)
  @template_lexer_table["}"] = method(:match_right_curly_in_template)
      
  @template_param_lexer_table = {}
  @template_param_lexer_table["{"] = method(:match_left_curly_in_template)
  @template_param_lexer_table["}"] = method(:match_right_curly_in_template)
  @template_param_lexer_table["|"] = method(:match_pipe_in_template)
      
  # Begin lexing in default state
  @lexer_table = LexerTable.new
  @lexer_table.push(@default_lexer_table)
end

Instance Attribute Details

#cursor ⇒ Object (readonly)

Returns the value of attribute cursor.

# File 'lib/mediacloth/mediawikilexer.rb', line 80

def cursor
  @cursor
end

#tokens ⇒ Object (readonly)

Returns the value of attribute tokens.

# File 'lib/mediacloth/mediawikilexer.rb', line 81

def tokens
  @tokens
end

Instance Method Details

#lex ⇒ Object

Returns the next token from the stream. Useful for RACC parsers.

# File 'lib/mediacloth/mediawikilexer.rb', line 277

def lex
  token = @tokens[@position]
  @position += 1
  return token
end

#tokenize(input) ⇒ Object

# File 'lib/mediacloth/mediawikilexer.rb', line 234

def tokenize(input)
  @text = input
  # Current position in the input text
  @cursor = 0
  # Tokens to be returned
  @tokens = TokenArray.new(self)
  # Stack of open token spans
  @context = []
  # Already lexed character data, not yet added to a TEXT token
  @pending = TokenString.new(self)
  # List symbols from the most recent line item of a list, e.g. '***'
  @list = ''
      
  start_span(:PARA)
      
  while (@cursor < @text.length)
    @char = @text[@cursor, 1]
    if @lexer_table[@char]
      @lexer_table[@char].call
    else
      @pending << @char
      @cursor += 1
    end
  end
  
  if @pending.is_empty_token?
    if @context.size > 0 and @tokens.last[0] == :PARA_START
      @context.pop
      @tokens.pop 
    end
  else
    @tokens.append_pending(@pending)
    @pending = TokenString.new(self)
  end
  while(@context.size > 0) do
    @tokens << [(@context.pop.to_s + '_END').to_sym, '']
  end
  @tokens << [false, false, 0, 0]
  @tokens
  
end