Class: SgfTokenizer
- Inherits:
-
Racc::Parser
- Object
- Racc::Parser
- SgfTokenizer
- Defined in:
- lib/sgf_tools/tokenizer.rb
Defined Under Namespace
Classes: ScanError
Instance Attribute Summary collapse
-
#filename ⇒ Object
readonly
Returns the value of attribute filename.
-
#lineno ⇒ Object
readonly
Returns the value of attribute lineno.
Instance Method Summary collapse
- #action(&block) ⇒ Object
- #load_file(filename) ⇒ Object
- #next_token ⇒ Object
- #scan_evaluate(str) ⇒ Object
- #scan_file(filename) ⇒ Object
- #scan_setup ⇒ Object
- #scan_str(str) ⇒ Object
Instance Attribute Details
#filename ⇒ Object (readonly)
Returns the value of attribute filename.
22 23 24 |
# File 'lib/sgf_tools/tokenizer.rb', line 22 def filename @filename end |
#lineno ⇒ Object (readonly)
Returns the value of attribute lineno.
21 22 23 |
# File 'lib/sgf_tools/tokenizer.rb', line 21 def lineno @lineno end |
Instance Method Details
#action(&block) ⇒ Object
26 27 28 |
# File 'lib/sgf_tools/tokenizer.rb', line 26 def action &block yield end |
#load_file(filename) ⇒ Object
35 36 37 38 39 40 |
# File 'lib/sgf_tools/tokenizer.rb', line 35 def load_file( filename ) @filename = filename open(filename, "r") do |f| scan_evaluate f.read end end |
#next_token ⇒ Object
47 48 49 |
# File 'lib/sgf_tools/tokenizer.rb', line 47 def next_token @rex_tokens.shift end |
#scan_evaluate(str) ⇒ Object
51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 |
# File 'lib/sgf_tools/tokenizer.rb', line 51 def scan_evaluate( str ) scan_setup @rex_tokens = [] @lineno = 1 ss = StringScanner.new(str) state = nil until ss.eos? text = ss.peek(1) @lineno += 1 if text == "\n" case state when nil case when (text = ss.scan(/[A-Z]+/)) @rex_tokens.push action { [:prop_ident, text] } when (text = ss.scan(/\[[^\]]*\]/)) @rex_tokens.push action { [:prop_value, text[1..-2]] } when (text = ss.scan(/\s+/)) ; when (text = ss.scan(/./)) @rex_tokens.push action { [text, text] } else text = ss.string[ss.pos .. -1] raise ScanError, "can not match: '" + text + "'" end # if else raise ScanError, "undefined state: '" + state.to_s + "'" end # case state end # until ss end |
#scan_file(filename) ⇒ Object
42 43 44 45 |
# File 'lib/sgf_tools/tokenizer.rb', line 42 def scan_file( filename ) load_file filename do_parse end |
#scan_setup ⇒ Object
24 |
# File 'lib/sgf_tools/tokenizer.rb', line 24 def scan_setup ; end |
#scan_str(str) ⇒ Object
30 31 32 33 |
# File 'lib/sgf_tools/tokenizer.rb', line 30 def scan_str( str ) scan_evaluate str do_parse end |