Class: Hocon::Impl::Parser::ParseContext
- Inherits:
-
Object
- Object
- Hocon::Impl::Parser::ParseContext
- Defined in:
- lib/hocon/impl/parser.rb
Defined Under Namespace
Classes: Element
Class Method Summary collapse
- .attracts_leading_comments?(token) ⇒ Boolean
- .attracts_trailing_comments?(token) ⇒ Boolean
- .include_keyword?(token) ⇒ Boolean
Instance Method Summary collapse
- #add_any_comments_after_any_comma(v) ⇒ Object
- #add_key_name(message) ⇒ Object
- #add_quote_suggestion(bad_token, message, last_path = nil, inside_equals = (@equals_count > 0)) ⇒ Object
-
#check_element_separator ⇒ Object
In arrays and objects, comma can be omitted as long as there’s at least one newline instead.
- #consolidate_comment_block(comment_token) ⇒ Object
-
#consolidate_value_tokens ⇒ Object
merge a bunch of adjacent values into one value; change unquoted text into a string value.
- #create_value_under_path(path, value) ⇒ Object
-
#initialize(flavor, origin, tokens, includer, include_context) ⇒ ParseContext
constructor
A new instance of ParseContext.
- #key_value_separator_token?(t) ⇒ Boolean
- #line_origin ⇒ Object
- #next_token ⇒ Object
- #next_token_ignoring_newline ⇒ Object
- #parse ⇒ Object
- #parse_array ⇒ Object
- #parse_error(message, cause = nil) ⇒ Object
- #parse_key(token) ⇒ Object
- #parse_object(had_open_curly) ⇒ Object
- #parse_value(t) ⇒ Object
- #pop_token ⇒ Object
- #pop_token_without_trailing_comment ⇒ Object
- #previous_field_name(last_path = nil) ⇒ Object
- #put_back(token) ⇒ Object
Constructor Details
#initialize(flavor, origin, tokens, includer, include_context) ⇒ ParseContext
Returns a new instance of ParseContext.
136 137 138 139 140 141 142 143 144 145 146 147 148 149 |
# File 'lib/hocon/impl/parser.rb', line 136 def initialize(flavor, origin, tokens, includer, include_context) @line_number = 1 @flavor = flavor @base_origin = origin @buffer = [] @tokens = tokens @includer = includer @include_context = include_context @path_stack = [] # this is the number of "equals" we are inside, # used to modify the error message to reflect that # someone may think this is .properties format. @equals_count = 0 end |
Class Method Details
.attracts_leading_comments?(token) ⇒ Boolean
121 122 123 124 125 126 127 128 129 |
# File 'lib/hocon/impl/parser.rb', line 121 def self.attracts_leading_comments?(token) # a comment just before a close } generally doesn't go with the # value before it, unless it's on the same line as that value !(Tokens.newline?(token) || token == Tokens::START || token == Tokens::CLOSE_CURLY || token == Tokens::CLOSE_SQUARE || token == Tokens::EOF) end |
.attracts_trailing_comments?(token) ⇒ Boolean
110 111 112 113 114 115 116 117 118 119 |
# File 'lib/hocon/impl/parser.rb', line 110 def self.attracts_trailing_comments?(token) # EOF can't have a trailing comment; START, OPEN_CURLY, and # OPEN_SQUARE followed by a comment should behave as if the comment # went with the following field or element. Associating a comment # with a newline would mess up all the logic for comment tracking, # so don't do that either. !(Tokens.newline?(token) || token == Tokens::START || token == Tokens::OPEN_CURLY) end |
Instance Method Details
#add_any_comments_after_any_comma(v) ⇒ Object
652 653 654 655 656 657 658 659 660 661 662 663 |
# File 'lib/hocon/impl/parser.rb', line 652 def add_any_comments_after_any_comma(v) t = next_token # do NOT skip newlines, we only # want same-line comments if t.token == Tokens::COMMA # steal the comments from after the comma put_back(t.remove_all) v.with_origin(t.append_comments(v.origin)) else put_back(t) v end end |
#add_key_name(message) ⇒ Object
716 717 718 719 720 721 722 723 |
# File 'lib/hocon/impl/parser.rb', line 716 def add_key_name() prev_field_name = previous_field_name if !prev_field_name.nil? "in value for key '#{prev_field_name}': #{}" else end end |
#add_quote_suggestion(bad_token, message, last_path = nil, inside_equals = (@equals_count > 0)) ⇒ Object
725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 |
# File 'lib/hocon/impl/parser.rb', line 725 def add_quote_suggestion(bad_token, , last_path = nil, inside_equals = (@equals_count > 0)) prev_field_name = previous_field_name(last_path) part = if bad_token == Tokens::EOF.to_s # EOF requires special handling for the error to make sense. if !prev_field_name.nil? "#{} (if you intended '#{prev_field_name}' " + "to be part of a value, instead of a key, " + "try adding double quotes around the whole value" else end else if !prev_field_name.nil? "#{} (if you intended #{bad_token} " + "to be part of the value for '#{prev_field_name}', " + "try enclosing the value in double quotes" else "#{} (if you intended #{bad_token} " + "to be part of a key or string value, " + "try enclosing the key or value in double quotes" end end if inside_equals "#{part}, or you may be able to rename the file .properties rather than .conf)" else "#{part})" end end |
#check_element_separator ⇒ Object
In arrays and objects, comma can be omitted as long as there’s at least one newline instead. this skips any newlines in front of a comma, skips the comma, and returns true if it found either a newline or a comma. The iterator is left just after the comma or the newline.
671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 |
# File 'lib/hocon/impl/parser.rb', line 671 def check_element_separator if @flavor == ConfigSyntax::JSON t = next_token_ignoring_newline if (t.token == Tokens::COMMA) true else put_back(t) false end else saw_separator_or_newline = false t = next_token while true if Tokens.newline?(t.token) # newline number is the line just ended, so add one @line_number = t.token.line_number + 1 saw_separator_or_newline = true # we want to continue to also eat a comma if there is one elsif t.token == Tokens::COMMA return true else # non-newline-or-comma put_back(t) return saw_separator_or_newline end t = next_token end end end |
#consolidate_comment_block(comment_token) ⇒ Object
159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 |
# File 'lib/hocon/impl/parser.rb', line 159 def consolidate_comment_block(comment_token) # a comment block "goes with" the following token # unless it's separated from it by a blank line. # we want to build a list of newline tokens followed # by a non-newline non-comment token; with all comments # associated with that final non-newline non-comment token. # a comment AFTER a token, without an intervening newline, # also goes with that token, but isn't handled in this method, # instead we handle it later by peeking ahead. new_lines = [] comments = [] previous_token = nil next_token = comment_token while true if Tokens.newline?(next_token) if (previous_token != nil) && Tokens.newline?(previous_token) # blank line; drop all comments to this point and # start a new comment block comments.clear end new_lines.push(next_token) elsif Tokens.comment?(next_token) comments.push(next_token) else # a non-newline non-comment token # comments before a close brace or bracket just get dumped unless self.class.attracts_leading_comments?(next_token) comments.clear end break end previous_token = next_token next_token = @tokens.next end # put our concluding token in the queue with all the comments # attached @buffer.push(TokenWithComments.new(next_token, comments)) # now put all the newlines back in front of it new_lines.reverse.each do |nl| @buffer.push(TokenWithComments.new(nl)) end end |
#consolidate_value_tokens ⇒ Object
merge a bunch of adjacent values into one value; change unquoted text into a string value.
210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 |
# File 'lib/hocon/impl/parser.rb', line 210 def consolidate_value_tokens # this trick is not done in JSON return if @flavor == ConfigSyntax::JSON # create only if we have value tokens values = nil # ignore a newline up front t = next_token_ignoring_newline while true v = nil if (Tokens.value?(t.token)) || (Tokens.unquoted_text?(t.token)) || (Tokens.substitution?(t.token)) || (t.token == Tokens::OPEN_CURLY) || (t.token == Tokens::OPEN_SQUARE) # there may be newlines _within_ the objects and arrays v = parse_value(t) else break end if v.nil? raise ConfigBugError("no value") end if values.nil? values = [] end values.push(v) t = next_token # but don't consolidate across a newline end # the last one wasn't a value token put_back(t) return if values.nil? consolidated = ConfigConcatenation.concatenate(values) put_back(TokenWithComments.new(Tokens.new_value(consolidated))) end |
#create_value_under_path(path, value) ⇒ Object
280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 |
# File 'lib/hocon/impl/parser.rb', line 280 def create_value_under_path(path, value) # for path foo.bar, we are creating # { "foo" : { "bar" : value } } keys = [] key = path.first remaining = path.remainder while !key.nil? # for ruby: convert string keys to symbols if key.is_a?(String) key = key end keys.push(key) if remaining.nil? break else key = remaining.first remaining = remaining.remainder end end # the setComments(null) is to ensure comments are only # on the exact leaf node they apply to. # a comment before "foo.bar" applies to the full setting # "foo.bar" not also to "foo" keys = keys.reverse # this is just a ruby means for doing first/rest deepest, *rest = *keys o = SimpleConfigObject.new(value.origin.set_comments(nil), {deepest => value}) while !rest.empty? deepest, *rest = *rest o = SimpleConfigObject.new(value.origin.set_comments(nil), {deepest => o}) end o end |
#key_value_separator_token?(t) ⇒ Boolean
151 152 153 154 155 156 157 |
# File 'lib/hocon/impl/parser.rb', line 151 def key_value_separator_token?(t) if @flavor == ConfigSyntax::JSON t == Tokens::COLON else [Tokens::COLON, Tokens::EQUALS, Tokens::PLUS_EQUALS].any? { |sep| sep == t } end end |
#line_origin ⇒ Object
251 252 253 |
# File 'lib/hocon/impl/parser.rb', line 251 def line_origin @base_origin.set_line_number(@line_number) end |
#next_token ⇒ Object
624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 |
# File 'lib/hocon/impl/parser.rb', line 624 def next_token with_comments = pop_token t = with_comments.token if Tokens.problem?(t) origin = t.origin = Tokens.(t) cause = Tokens.get_problem_cause(t) suggest_quotes = Tokens.get_problem_suggest_quotes(t) if suggest_quotes = add_quote_suggestion(t.to_s, ) else = add_key_name() end raise ConfigParseError.new(origin, , cause) else if @syntax == ConfigSyntax::JSON if Tokens.unquoted_text?(t) raise parse_error(add_key_name("Token not allowed in valid JSON: '#{Tokens.get_unquoted_text(t)}'")) elsif Tokens.substitution?(t) raise parse_error(add_key_name("Substitutions (${} syntax) not allowed in JSON")) end end with_comments end end |
#next_token_ignoring_newline ⇒ Object
603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 |
# File 'lib/hocon/impl/parser.rb', line 603 def next_token_ignoring_newline t = next_token while Tokens.newline?(t.token) # line number tokens have the line that was _ended_ by the # newline, so we have to add one. We have to update lineNumber # here and also below, because not all tokens store a line # number, but newline tokens always do. @line_number = t.token.line_number + 1 t = next_token end # update line number again, iff we have one new_number = t.token.line_number if new_number >= 0 @line_number = new_number end t end |
#parse ⇒ Object
558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 |
# File 'lib/hocon/impl/parser.rb', line 558 def parse t = next_token_ignoring_newline if t.token != Tokens::START raise ConfigBugError, "token stream did not begin with START, had #{t}" end t = next_token_ignoring_newline result = nil if (t.token == Tokens::OPEN_CURLY) or (t.token == Tokens::OPEN_SQUARE) result = parse_value(t) else if @syntax == ConfigSyntax::JSON if t.token == Tokens::END raise parse_error("Empty document") else raise parse_error("Document must have an object or array at root, unexpected token: #{t}") end else ## the root object can omit the surrounding braces. ## this token should be the first field's key, or part ## of it, so put it back. put_back(t) result = parse_object(false) ## in this case we don't try to use commentsStack comments ## since they would all presumably apply to fields not the ## root object end end t = next_token_ignoring_newline if t.token == Tokens::EOF result else raise parse_error("Document has trailing tokens after first object or array: #{t}") end end |
#parse_array ⇒ Object
498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 |
# File 'lib/hocon/impl/parser.rb', line 498 def parse_array # invoked just after the OPEN_SQUARE array_origin = line_origin values = [] consolidate_value_tokens t = next_token_ignoring_newline # special-case the first element if t.token == Tokens::CLOSE_SQUARE return SimpleConfigList.new(t.append_comments(array_origin), []) elsif (Tokens.value?(t.token)) || (t.token == Tokens::OPEN_CURLY) || (t.token == Tokens::OPEN_SQUARE) v = parse_value(t) v = add_any_comments_after_any_comma(v) values.push(v) else raise parse_error(add_key_name("List should have ] or a first element after the open [, instead had token: " + "#{t} (if you want #{t} to be part of a string value, then double-quote it)")) end # now remaining elements while true # just after a value if check_element_separator # comma (or newline equivalent) consumed else t = next_token_ignoring_newline if t.token == Tokens::CLOSE_SQUARE return SimpleConfigList.new(t.append_comments(array_origin), values) else raise parse_error(add_key_name("List should have ended with ] or had a comma, instead had token: " + "#{t} (if you want #{t} to be part of a string value, then double-quote it)")) end end # now just after a comma consolidate_value_tokens t = next_token_ignoring_newline if (Tokens.value?(t.token)) || (t.token == Tokens::OPEN_CURLY) || (t.token == Tokens::OPEN_SQUARE) v = parse_value(t) v = add_any_comments_after_any_comma(v) values.push(v) elsif (@flavor != ConfigSyntax::JSON) && (t.token == Tokens::CLOSE_SQUARE) # we allow one trailing comma put_back(t) else raise parse_error(add_key_name("List should have had new element after a comma, instead had token: " + "#{t} (if you want the comma or #{t} to be part of a string value, then double-quote it)")) end end end |
#parse_error(message, cause = nil) ⇒ Object
702 703 704 |
# File 'lib/hocon/impl/parser.rb', line 702 def parse_error(, cause = nil) ConfigParseError.new(line_origin, , cause) end |
#parse_key(token) ⇒ Object
319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 |
# File 'lib/hocon/impl/parser.rb', line 319 def parse_key(token) if @flavor == ConfigSyntax::JSON if Tokens.value_with_type?(token.token, ConfigValueType::STRING) key = Tokens.value(token.token).unwrapped Path.new_key(key) else raise parse_error(add_key_name("Expecting close brace } or a field name here, got #{token}")) end else expression = [] t = token while Tokens.value?(t.token) || Tokens.unquoted_text?(t.token) expression.push(t.token) t = next_token # note: don't cross a newline end if expression.empty? raise parse_error(add_key_name("expecting a close brace or a field name here, got #{t}")) end put_back(t) Hocon::Impl::Parser.parse_path_expression(expression, line_origin) end end |
#parse_object(had_open_curly) ⇒ Object
344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 |
# File 'lib/hocon/impl/parser.rb', line 344 def parse_object(had_open_curly) # invoked just after the OPEN_CURLY (or START, if !hadOpenCurly) values = {} object_origin = line_origin after_comma = false last_path = nil last_inside_equals = false while true t = next_token_ignoring_newline if t.token == Tokens::CLOSE_CURLY if (@flavor == ConfigSyntax::JSON) && after_comma raise parse_error( add_quote_suggestion(t, "unbalanced close brace '}' with no open brace")) end object_origin = t.append_comments(object_origin) break elsif (t.token == Tokens::EOF) && !had_open_curly put_back(t) break elsif (@flavor != ConfigSyntax::JSON) && self.class.include_keyword?(t.token) parse_include(values) after_comma = false else key_token = t path = parse_key(key_token) after_key = next_token_ignoring_newline inside_equals = false # path must be on-stack while we parse the value @path_stack.push(path) value_token = nil new_value = nil if (@flavor == ConfigSyntax::CONF) && (after_key.token == Tokens::OPEN_CURLY) # can omit the ':' or '=' before an object value value_token = after_key else if !key_value_separator_token?(after_key.token) raise parse_error( add_quote_suggestion(after_key, "Key '#{path.render}' may not be followed by token: #{after_key}")) end if after_key.token == Tokens::EQUALS inside_equals = true @equals_count += 1 end consolidate_value_tokens value_token = next_token_ignoring_newline # put comments from separator token on the value token value_token = value_token.prepend(after_key.comments) end # comments from the key token go to the value token new_value = parse_value(value_token.prepend(key_token.comments)) if after_key.token == Tokens::PLUS_EQUALS previous_ref = ConfigReference.new( new_value.origin, SubstitutionExpression.new(full_current_path, true)) list = SimpleConfigList.new(new_value.origin, [new_value]) new_value = ConfigConcatenation.concatenate([previous_ref, list]) end new_value = add_any_comments_after_any_comma(new_value) last_path = @path_stack.pop if inside_equals @equals_count -= 1 end last_inside_equals = inside_equals key = path.first # for ruby: convert string keys to symbols if key.is_a?(String) key = key end remaining = path.remainder if !remaining existing = values[key] if existing # In strict JSON, dups should be an error; while in # our custom config language, they should be merged # if the value is an object (or substitution that # could become an object). if @flavor == ConfigSyntax::JSON raise parse_error("JSON does not allow duplicate fields: '#{key}'" + " was already seen at #{existing.origin().description()}") else new_value = new_value.with_fallback(existing) end end values[key] = new_value else if @flavor == ConfigSyntax::JSON raise ConfigBugError, "somehow got multi-element path in JSON mode" end obj = create_value_under_path(remaining, new_value) existing = values[key] if !existing.nil? obj = obj.with_fallback(existing) end values[key] = obj end after_comma = false end if check_element_separator # continue looping after_comma = true else t = next_token_ignoring_newline if t.token == Tokens::CLOSE_CURLY if !had_open_curly raise parse_error( add_quote_suggestion(last_path, last_inside_equals, t, "unbalanced close brace '}' with no open brace")) end object_origin = t.append_comments(object_origin) break elsif had_open_curly raise parse_error( add_quote_suggestion(t, "Expecting close brace } or a comma, got #{t}", last_path, last_inside_equals)) else if t.token == Tokens::END put_back(t) break else raise parse_error( add_quote_suggestion(t, "Expecting end of input or a comma, got #{t}", last_path, last_inside_equals)) end end end end SimpleConfigObject.new(object_origin, values) end |
#parse_value(t) ⇒ Object
255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 |
# File 'lib/hocon/impl/parser.rb', line 255 def parse_value(t) v = nil if Tokens.value?(t.token) # if we consolidateValueTokens() multiple times then # this value could be a concatenation, object, array, # or substitution already. v = Tokens.value(t.token) elsif Tokens.unquoted_text?(t.token) v = Hocon::Impl::ConfigString.new(t.token.origin, Tokens.unquoted_text(t.token)) elsif Tokens.substitution?(t.token) v = ConfigReference.new(t.token.origin, token_to_substitution_expression(t.token)) elsif t.token == Tokens::OPEN_CURLY v = parse_object(true) elsif t.token == Tokens::OPEN_SQUARE v = parse_array else raise parse_error( add_quote_suggestion(t.token.to_s, "Expecting a value but got wrong token: #{t.token}")) end v.with_origin(t.prepend_comments(v.origin)) end |
#pop_token ⇒ Object
757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 |
# File 'lib/hocon/impl/parser.rb', line 757 def pop_token with_preceding_comments = pop_token_without_trailing_comment # handle a comment AFTER the other token, # but before a newline. If the next token is not # a comment, then any comment later on the line is irrelevant # since it would end up going with that later token, not # this token. Comments are supposed to be processed prior # to adding stuff to the buffer, so they can only be found # in "tokens" not in "buffer" in theory. if !self.class.attracts_trailing_comments?(with_preceding_comments.token) with_preceding_comments elsif @buffer.empty? after = @tokens.next if Tokens.comment?(after) with_preceding_comments.add(after) else @buffer << TokenWithComments.new(after) with_preceding_comments end else # comments are supposed to get attached to a token, # not put back in the buffer. Assert this as an invariant. if Tokens.comment?(@buffer.last.token) raise ConfigBugError, "comment token should not have been in buffer: #{@buffer}" end with_preceding_comments end end |
#pop_token_without_trailing_comment ⇒ Object
786 787 788 789 790 791 792 793 794 795 796 797 798 |
# File 'lib/hocon/impl/parser.rb', line 786 def pop_token_without_trailing_comment if @buffer.empty? t = @tokens.next if Tokens.comment?(t) consolidate_comment_block(t) @buffer.pop else TokenWithComments.new(t) end else @buffer.pop end end |
#previous_field_name(last_path = nil) ⇒ Object
706 707 708 709 710 711 712 713 714 |
# File 'lib/hocon/impl/parser.rb', line 706 def previous_field_name(last_path = nil) if !last_path.nil? last_path.render elsif @path_stack.empty? nil else @path_stack[0].render end end |
#put_back(token) ⇒ Object
596 597 598 599 600 601 |
# File 'lib/hocon/impl/parser.rb', line 596 def put_back(token) if Tokens.comment?(token.token) raise ConfigBugError, "comment token should have been stripped before it was available to put back" end @buffer.push(token) end |