Module: RubyLexer::RubyLexer1_9

Includes:
NestedContexts
Defined in:
lib/rubylexer.rb

Constant Summary collapse

FUNCLIKE_KEYWORDLIST = 
RubyLexer::FUNCLIKE_KEYWORDLIST+FUNCLIKE_KEYWORDLIST_1_9
VARLIKE_KEYWORDLIST = 
RubyLexer::VARLIKE_KEYWORDLIST+VARLIKE_KEYWORDLIST_1_9
FUNCLIKE_KEYWORDS = 
/^(?:#{FUNCLIKE_KEYWORDLIST.join '|'})$/
VARLIKE_KEYWORDS = 
/^(?:#{VARLIKE_KEYWORDLIST.join '|'})$/
RE_UTF8 =

RE_* shamelessly stolen from jcode.rb

/[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf][\x80-\xbf]|[\xf0-\xf7][\x80-\xbf]{3}/n
RE_EUC =

is this complete?

/[\xa1-\xfe][\xa1-\xfe]/n
RE_SJIS =

is this complete? windows31j?

/[\x81-\x9f\xe0-\xef][\x40-\x7e\x80-\xfc]/n
ENCODING2EXTCHAR = 
{
  :utf8=>RE_UTF8,
  :euc=>RE_EUC,
  :sjis=>RE_SJIS,
  :binary=>/[\x80-\xFF]/n,
  :ascii=>nil
}

Instance Method Summary collapse

Instance Method Details

#assign_encoding!(str) ⇒ Object 



1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
 
# File 'lib/rubylexer.rb', line 1774

def assign_encoding! str
  #search for nonascii bytes
  #either directly or via hex (\xXX) or octal (\NNN) escapes
  #and \u escapes also 
  utf8=nonascii=false
  str.elems.grep(String).each do|frag|
    frag.scan(/#{EVEN_BS_S}(?:\\u|\\2[0-7][0-7]|\\x[89a-fA-F][0-9a-fA-F])|[^\x00-\x7F]/o) do |match|
      if match[-1]==?u
        utf8=true
        break if nonascii
      else
        nonascii=true
        break if utf8
      end
    end or break
  end

  lexerror(str,"utf8 and nonascii intermixed") if utf8 and nonascii and @encoding!=:utf8
  
  #encoding is source encoding unless \u escape is found 
  str.utf8! if utf8

  #maybe assign string fragments encodings if running under >=1.9?

  return str
end

#build_method_operators ⇒ Object 



1849
1850
1851
 
# File 'lib/rubylexer.rb', line 1849

def build_method_operators
  /#{RUBYSYMOPERATORREX}|\A![=~@]?|\A`/o
end

#callsite_symbol(x) ⇒ Object 



1656
1657
1658
1659
 
# File 'lib/rubylexer.rb', line 1656

def callsite_symbol(x)
  return if nextchar==?(
  super
end

#char_literal_or_op(ch) ⇒ Object

handle ? in ruby code. is it part of ?..: or a character literal?



1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
 
# File 'lib/rubylexer.rb', line 1711

def char_literal_or_op(ch) #unicode char literals, etc
  if colon_quote_expected? ch
    #char literal
    pos=input_position
    getchar
    extchar= ENCODING2EXTCHAR[@encoding]
    result=
    if extchar and extchar=@file.scan( extchar )
      assign_encoding!(StringToken.new('"', extchar))
    else
      getchar_maybe_escape
      assign_encoding!(StringToken.new('"', @file[pos+1...input_position]))
    end
    result.offset=pos
    result.bs_handler=:dquote19_esc_seq
    result.open='?'
    result.close=''
    return result
  else #(ternary) operator
    super
  end
end

#colon_operator(tok) ⇒ Object 



1865
1866
1867
1868
1869
1870
 
# File 'lib/rubylexer.rb', line 1865

def colon_operator tok
   if TernaryContext===@parsestack.last
     tok.ternary=true
     @parsestack.pop #should be in the context's see handler
   end 
end

#dquote_handle(ch) ⇒ Object 



1636
1637
1638
 
# File 'lib/rubylexer.rb', line 1636

def dquote_handle(ch)
  dquote19_esc_seq(ch,'"','"')
end

#dquote_handler_name ⇒ Object 



1640
1641
1642
 
# File 'lib/rubylexer.rb', line 1640

def dquote_handler_name
  :dquote19_esc_seq
end

#equals(ch) ⇒ Object

match /=(>|~|==?)?/ (= or == or =~ or === or =>)



1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
 
# File 'lib/rubylexer.rb', line 1759

def equals(ch) # /(?<foo>bar)/=~'bar'; declares foo lvar
  if readahead(2)=='=~' # =~... after regex, maybe?
    last=last_operative_token
   
    if StringToken===last and last.lvars
      #ruby delays adding lvars from regexps to known lvars table
      #for several tokens in some cases. not sure why or if on purpose
      #i'm just going to add them right away
      last.lvars.each{|lvar| localvars[lvar]=true }
    end
  end
  return super
end

#FUNCLIKE_KEYWORDS(orig = nil) ⇒ Object 



1621
1622
1623
 
# File 'lib/rubylexer.rb', line 1621

def FUNCLIKE_KEYWORDS orig=nil
  /(?:#{orig||super()}|^(?:#{FUNCLIKE_KEYWORDLIST_1_9.join '|'})$)/
end

#is__ENCODING__keyword?(name) ⇒ Boolean

Returns:

  • (Boolean) 


1860
1861
1862
 
# File 'lib/rubylexer.rb', line 1860

def is__ENCODING__keyword?(name)
  "__ENCODING__"==name
end

#keyword___ENCODING__(str, offset, result) ⇒ Object 



1674
1675
1676
1677
 
# File 'lib/rubylexer.rb', line 1674

def keyword___ENCODING__(str,offset,result)
  #result.last.value=huh
  return result
end

#keyword_not(*args, &block) ⇒ Object 



1680
 
# File 'lib/rubylexer.rb', line 1680

def keyword_not(*args,&block) _keyword_funclike(*args,&block) end

#maybe_end_stabby_block_param_list(tokch) ⇒ Object 



1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
 
# File 'lib/rubylexer.rb', line 1872

def maybe_end_stabby_block_param_list(tokch)
     stabby_params_just_ended=false
     (@parsestack.size-1).downto(1){|i|
       case @parsestack[i]
       when ParamListContextNoParen,AssignmentRhsContext
         #do nothing yet... see if inside a UnparenedParamListLhsContext
       when UnparenedParamListLhsContext #stabby proc
         @moretokens<<tokch
         (@parsestack.size-1).downto(i){|j|
           @moretokens.unshift @parsestack[j].endtoken(input_position-1)
         }
         @parsestack[i..-1]=[]
         tokch=@moretokens.shift
         stabby_params_just_ended=true
         break
       else break
       end
     }
     return stabby_params_just_ended,tokch
end

#method_params? ⇒ Boolean

Returns:

  • (Boolean) 


1649
1650
1651
1652
1653
 
# File 'lib/rubylexer.rb', line 1649

def method_params?  # .()
  lasttok=last_token_maybe_implicit #last_operative_token
  super or 
    (lasttok and lasttok.ident=='.') 
end

#plusminus(ch) ⇒ Object 



1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
 
# File 'lib/rubylexer.rb', line 1735

def plusminus(ch) #->
  pos=input_position
  assert(/^[+\-]$/===ch)
  if unary_op_expected?(ch) or
    KeywordToken===@last_operative_token &&
    /^(return|break|next)$/===@last_operative_token.ident
    if '->' == readahead(2) #stabby proc
        @file.pos+=2
        #push down block context
        localvars.start_block
        @parsestack.push ctx=RubyLexer::BlockContext.new(@linenum)
        ctx.wanting_stabby_block_body=true
        #read optional proc params
        block_param_list_lookahead ?(, RubyLexer::ParenedParamListLhsContext
        result=RubyLexer::KeywordToken.new('->',pos)
        result.offset=pos
        return result
    end
  end
  super
end

#read_encoding_line ⇒ Object 



1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
 
# File 'lib/rubylexer.rb', line 1662

def read_encoding_line
  if line=@file.scan(
       /\A#{WSNONLCHARS}*#[\x00-\x7F]*?(?:en)?coding#{WSNONLCHARS}*[:=]#{WSNONLCHARS}*([a-z0-9_-]+)[\x00-\x7F]*$/io
     )
    name=@file.last_match[1]
    name=encoding_name_normalize name
    @encoding=name.to_sym if ENCODINGS.include? name
    return line
  end
end

#regex(ch = nil) ⇒ Object 



1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
 
# File 'lib/rubylexer.rb', line 1802

def regex(ch=nil)
 result=super
   named_brs=[]
   if result.elems.size==1 and String===result.elems.first
       elem=result.elems.first
       index=0
       while index=elem.index(/(#{EVEN_BS_S})( \(\?[<'] | \(\?\# | \[ )/xo,index)
         index+=$1.size
         case $2
         when "(?<"
           index=elem.index(/\G...(#{LCLETTER}#{LETTER_DIGIT}+)>/o,index)
           break lexerror(result, "malformed named backreference") unless index
           index+=$&.size
           named_brs<<$1
         when "(?'"
           index=elem.index(/\G...(#{LCLETTER}#{LETTER_DIGIT}+)'/o,index)
           break lexerror(result, "malformed named backreference") unless index
           index+=$&.size
           named_brs<<$1
         when "(?#"
           index+=3
           index=elem.index(/#{EVEN_BS_S}\)/o,index)
           break lexerror(result, "unterminated regexp comment") unless index
           index+=$&.size
         when "["
           index+=1
           paren_ctr=1
           loop do
             index=elem.index(/#{EVEN_BS_S}(&&\[\^|\])/o,index)
             break lexerror(result, "unterminated character class") unless index
             index+=$&.size
             if $1==']'
               paren_ctr-=1
               break if paren_ctr==0
             else
               paren_ctr+=1
             end
           end
           break unless index

         end
       end
       result.lvars= named_brs unless named_brs.empty?
   end
 return result
end

#rubylexer_modules_init ⇒ Object 



1629
1630
1631
1632
1633
 
# File 'lib/rubylexer.rb', line 1629

def rubylexer_modules_init
  super
  @FUNCLIKE_KEYWORDS=FUNCLIKE_KEYWORDS @FUNCLIKE_KEYWORDS unless @FUNCLIKE_KEYWORDS==="->"
  @VARLIKE_KEYWORDS=VARLIKE_KEYWORDS @VARLIKE_KEYWORDS unless @VARLIKE_KEYWORDS==="__ENCODING__"
end

#semicolon_in_block_param_list? ⇒ Boolean

Returns:

  • (Boolean) 


1855
1856
1857
1858
 
# File 'lib/rubylexer.rb', line 1855

def semicolon_in_block_param_list?
  ParenedParamListLhsContext===@parsestack.last || 
    BlockParamListLhsContext===@parsestack.last
end

#special_identifier?(str, oldpos) ⇒ Boolean

Returns:

  • (Boolean) 


1683
1684
1685
1686
1687
1688
1689
 
# File 'lib/rubylexer.rb', line 1683

def special_identifier?(str,oldpos)
  if @parsestack.last.wantarrow and @file.skip ":"
    return SymbolToken.new(str,oldpos), KeywordToken.new(":",input_position-1,:as=>"=>")
  else
    return super
  end
end

#VARLIKE_KEYWORDS(orig = nil) ⇒ Object 



1625
1626
1627
 
# File 'lib/rubylexer.rb', line 1625

def VARLIKE_KEYWORDS orig=nil
  /(?:#{orig||super()}|^(?:#{VARLIKE_KEYWORDLIST_1_9.join '|'})$)/
end

#want_hard_nl? ⇒ Boolean

Returns:

  • (Boolean) 


1692
1693
1694
1695
 
# File 'lib/rubylexer.rb', line 1692

def want_hard_nl?
  return false if @file.check( /\A\n(?:#{WSTOKS})?[.:][^.:]/o )
  super
end

#Wquote_handler_name ⇒ Object 



1644
1645
1646
 
# File 'lib/rubylexer.rb', line 1644

def Wquote_handler_name
  :Wquote19_esc_seq
end