Module: OkJson
Overview
Some parts adapted from golang.org/src/pkg/json/decode.go and golang.org/src/pkg/utf8/utf8.go
Defined Under Namespace
Constant Summary collapse
- Upstream =
'LTD7LBKLZWFF7OZK'
- Utagx =
1000 0000
0x80
- Utag2 =
1100 0000
0xc0
- Utag3 =
1110 0000
0xe0
- Utag4 =
1111 0000
0xf0
- Utag5 =
1111 1000
0xF8
- Umaskx =
0011 1111
0x3f
- Umask2 =
0001 1111
0x1f
- Umask3 =
0000 1111
0x0f
- Umask4 =
0000 0111
0x07
- Uchar1max =
(1<<7) - 1
- Uchar2max =
(1<<11) - 1
- Uchar3max =
(1<<16) - 1
- Ucharerr =
unicode “replacement char”
0xFFFD
- Ustrerr =
unicode “replacement char”
"\xef\xbf\xbd"
- Usurrself =
0x10000
- Usurr1 =
0xd800
- Usurr2 =
0xdc00
- Usurr3 =
0xe000
- Spc =
- Unesc =
{?b=>?\b, ?f=>?\f, ?n=>?\n, ?r=>?\r, ?t=>?\t}
Instance Method Summary collapse
- #abbrev(s) ⇒ Object
- #arrenc(a) ⇒ Object
-
#arrparse(ts) ⇒ Object
Parses an “array” in the sense of RFC 4627.
-
#decode(s) ⇒ Object
Decodes a json document in string s and returns the corresponding ruby value.
- #eat(typ, ts) ⇒ Object
-
#encode(x) ⇒ Object
Encodes x into a json text.
- #falsetok(s) ⇒ Object
- #hexdec4(s) ⇒ Object
- #keyenc(k) ⇒ Object
-
#lex(s) ⇒ Object
Scans s and returns a list of json tokens, excluding white space (as defined in RFC 4627).
- #nibble(c) ⇒ Object
- #nulltok(s) ⇒ Object
- #numenc(x) ⇒ Object
- #numtok(s) ⇒ Object
- #objenc(x) ⇒ Object
-
#objparse(ts) ⇒ Object
Parses an “object” in the sense of RFC 4627.
-
#pairparse(ts) ⇒ Object
Parses a “member” in the sense of RFC 4627.
- #strenc(s) ⇒ Object
- #strtok(s) ⇒ Object
- #subst(u1, u2) ⇒ Object
- #surrogate?(u) ⇒ Boolean
-
#textparse(ts) ⇒ Object
Parses a “json text” in the sense of RFC 4627.
-
#tok(s) ⇒ Object
Scans the first token in s and returns a 3-element list, or nil if s does not begin with a valid token.
- #truetok(s) ⇒ Object
-
#ucharcopy(t, s, i) ⇒ Object
Copies the valid UTF-8 bytes of a single character from string s at position i to I/O object t, and returns the number of bytes copied.
-
#ucharenc(a, i, u) ⇒ Object
Encodes unicode character u as UTF-8 bytes in string a at position i.
-
#unquote(q) ⇒ Object
Converts a quoted json string literal q into a UTF-8-encoded string.
- #valenc(x) ⇒ Object
-
#valparse(ts) ⇒ Object
Parses a “value” in the sense of RFC 4627.
Instance Method Details
#abbrev(s) ⇒ Object
253 254 255 256 257 258 259 |
# File 'lib/okjson.rb', line 253 def abbrev(s) t = s[0,10] p = t['`'] t = t[0,p] if p t = t + '...' if t.length < s.length '`' + t + '`' end |
#arrenc(a) ⇒ Object
435 436 437 |
# File 'lib/okjson.rb', line 435 def arrenc(a) '[' + a.map{|x| valenc(x)}.join(',') + ']' end |
#arrparse(ts) ⇒ Object
Parses an “array” in the sense of RFC 4627. Returns the parsed value and any trailing tokens.
135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 |
# File 'lib/okjson.rb', line 135 def arrparse(ts) ts = eat('[', ts) arr = [] if ts[0][0] == ']' return arr, ts[1..-1] end v, ts = valparse(ts) arr << v if ts[0][0] == ']' return arr, ts[1..-1] end loop do ts = eat(',', ts) v, ts = valparse(ts) arr << v if ts[0][0] == ']' return arr, ts[1..-1] end end end |
#decode(s) ⇒ Object
Decodes a json document in string s and returns the corresponding ruby value. String s must be valid UTF-8. If you have a string in some other encoding, convert it first.
String values in the resulting structure will be UTF-8.
43 44 45 46 47 48 49 50 |
# File 'lib/okjson.rb', line 43 def decode(s) ts = lex(s) v, ts = textparse(ts) if ts.length > 0 raise Error, 'trailing garbage' end v end |
#eat(typ, ts) ⇒ Object
163 164 165 166 167 168 |
# File 'lib/okjson.rb', line 163 def eat(typ, ts) if ts[0][0] != typ raise Error, "expected #{typ} (got #{ts[0].inspect})" end ts[1..-1] end |
#encode(x) ⇒ Object
Encodes x into a json text. It may contain only Array, Hash, String, Numeric, true, false, nil. (Note, this list excludes Symbol.) X itself must be an Array or a Hash. No other value can be encoded, and an error will be raised if x contains any other value, such as Nan, Infinity, Symbol, and Proc, or if a Hash key is not a String. Strings contained in x must be valid UTF-8.
405 406 407 408 409 410 411 412 |
# File 'lib/okjson.rb', line 405 def encode(x) case x when Hash then objenc(x) when Array then arrenc(x) else raise Error, 'root value must be an Array or a Hash' end end |
#falsetok(s) ⇒ Object
225 |
# File 'lib/okjson.rb', line 225 def falsetok(s); s[0,5] == 'false' ? [:val, 'false', false] : [] end |
#hexdec4(s) ⇒ Object
364 365 366 367 368 369 |
# File 'lib/okjson.rb', line 364 def hexdec4(s) if s.length != 4 raise Error, 'short' end (nibble(s[0])<<12) | (nibble(s[1])<<8) | (nibble(s[2])<<4) | nibble(s[3]) end |
#keyenc(k) ⇒ Object
440 441 442 443 444 445 446 |
# File 'lib/okjson.rb', line 440 def keyenc(k) case k when String then strenc(k) else raise Error, "Hash key is not a string: #{k.inspect}" end end |
#lex(s) ⇒ Object
Scans s and returns a list of json tokens, excluding white space (as defined in RFC 4627).
173 174 175 176 177 178 179 180 181 182 183 184 185 186 |
# File 'lib/okjson.rb', line 173 def lex(s) ts = [] while s.length > 0 typ, lexeme, val = tok(s) if typ == nil raise Error, "invalid character at #{s[0,10].inspect}" end if typ != :space ts << [typ, lexeme, val] end s = s[lexeme.length..-1] end ts end |
#nibble(c) ⇒ Object
385 386 387 388 389 390 391 392 393 |
# File 'lib/okjson.rb', line 385 def nibble(c) case true when ?0 <= c && c <= ?9 then c.ord - ?0.ord when ?a <= c && c <= ?z then c.ord - ?a.ord + 10 when ?A <= c && c <= ?Z then c.ord - ?A.ord + 10 else raise Error, "invalid hex code #{c}" end end |
#nulltok(s) ⇒ Object
223 |
# File 'lib/okjson.rb', line 223 def nulltok(s); s[0,4] == 'null' ? [:val, 'null', nil] : [] end |
#numenc(x) ⇒ Object
490 491 492 493 494 495 |
# File 'lib/okjson.rb', line 490 def numenc(x) if ((x.nan? || x.infinite?) rescue false) raise Error, "Numeric cannot be represented: #{x}" end "#{x}" end |
#numtok(s) ⇒ Object
228 229 230 231 232 233 234 235 236 237 238 239 240 241 |
# File 'lib/okjson.rb', line 228 def numtok(s) m = /-?([1-9][0-9]+|[0-9])([.][0-9]+)?([eE][+-]?[0-9]+)?/.match(s) if m && m.begin(0) == 0 if m[3] && !m[2] [:val, m[0], Integer(m[1])*(10**Integer(m[3][1..-1]))] elsif m[2] [:val, m[0], Float(m[0])] else [:val, m[0], Integer(m[0])] end else [] end end |
#objenc(x) ⇒ Object
430 431 432 |
# File 'lib/okjson.rb', line 430 def objenc(x) '{' + x.map{|k,v| keyenc(k) + ':' + valenc(v)}.join(',') + '}' end |
#objparse(ts) ⇒ Object
Parses an “object” in the sense of RFC 4627. Returns the parsed value and any trailing tokens.
92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 |
# File 'lib/okjson.rb', line 92 def objparse(ts) ts = eat('{', ts) obj = {} if ts[0][0] == '}' return obj, ts[1..-1] end k, v, ts = pairparse(ts) obj[k] = v if ts[0][0] == '}' return obj, ts[1..-1] end loop do ts = eat(',', ts) k, v, ts = pairparse(ts) obj[k] = v if ts[0][0] == '}' return obj, ts[1..-1] end end end |
#pairparse(ts) ⇒ Object
Parses a “member” in the sense of RFC 4627. Returns the parsed values and any trailing tokens.
122 123 124 125 126 127 128 129 130 |
# File 'lib/okjson.rb', line 122 def pairparse(ts) (typ, _, k), ts = ts[0], ts[1..-1] if typ != :str raise Error, "unexpected #{k.inspect}" end ts = eat(':', ts) v, ts = valparse(ts) [k, v, ts] end |
#strenc(s) ⇒ Object
449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 |
# File 'lib/okjson.rb', line 449 def strenc(s) t = StringIO.new t.putc(?") r = 0 # In ruby >= 1.9, s[r] is a codepoint, not a byte. rubydoesenc = s.class.method_defined?(:encoding) while r < s.length case s[r] when ?" then t.print('\\"') when ?\\ then t.print('\\\\') when ?\b then t.print('\\b') when ?\f then t.print('\\f') when ?\n then t.print('\\n') when ?\r then t.print('\\r') when ?\t then t.print('\\t') else c = s[r] case true when rubydoesenc begin c.ord # will raise an error if c is invalid UTF-8 t.write(c) rescue t.write(Ustrerr) end when Spc <= c && c <= ?~ t.putc(c) else n = ucharcopy(t, s, r) # ensure valid UTF-8 output r += n - 1 # r is incremented below end end r += 1 end t.putc(?") t.string end |
#strtok(s) ⇒ Object
244 245 246 247 248 249 250 |
# File 'lib/okjson.rb', line 244 def strtok(s) m = /"([^"\\]|\\["\/\\bfnrt]|\\u[0-9a-fA-F]{4})*"/.match(s) if ! m raise Error, "invalid string literal at #{abbrev(s)}" end [:str, m[0], unquote(m[0])] end |
#subst(u1, u2) ⇒ Object
372 373 374 375 376 377 |
# File 'lib/okjson.rb', line 372 def subst(u1, u2) if Usurr1 <= u1 && u1 < Usurr2 && Usurr2 <= u2 && u2 < Usurr3 return ((u1-Usurr1)<<10) | (u2-Usurr2) + Usurrself end return Ucharerr end |
#surrogate?(u) ⇒ Boolean
380 381 382 |
# File 'lib/okjson.rb', line 380 def surrogate?(u) Usurr1 <= u && u < Usurr3 end |
#textparse(ts) ⇒ Object
Parses a “json text” in the sense of RFC 4627. Returns the parsed value and any trailing tokens. Note: this is almost the same as valparse, except that it does not accept atomic values.
57 58 59 60 61 62 63 64 65 66 67 68 69 |
# File 'lib/okjson.rb', line 57 def textparse(ts) if ts.length < 0 raise Error, 'empty' end typ, _, val = ts[0] case typ when '{' then objparse(ts) when '[' then arrparse(ts) else raise Error, "unexpected #{val.inspect}" end end |
#tok(s) ⇒ Object
Scans the first token in s and returns a 3-element list, or nil if s does not begin with a valid token.
The first list element is one of ‘‘’, ‘:’, ‘,’, ‘[’, ‘]’, :val, :str, and :space.
The second element is the lexeme.
The third element is the value of the token for :val and :str, otherwise it is the lexeme.
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 |
# File 'lib/okjson.rb', line 202 def tok(s) case s[0] when ?{ then ['{', s[0,1], s[0,1]] when ?} then ['}', s[0,1], s[0,1]] when ?: then [':', s[0,1], s[0,1]] when ?, then [',', s[0,1], s[0,1]] when ?[ then ['[', s[0,1], s[0,1]] when ?] then [']', s[0,1], s[0,1]] when ?n then nulltok(s) when ?t then truetok(s) when ?f then falsetok(s) when ?" then strtok(s) when Spc then [:space, s[0,1], s[0,1]] when ?\t then [:space, s[0,1], s[0,1]] when ?\n then [:space, s[0,1], s[0,1]] when ?\r then [:space, s[0,1], s[0,1]] else numtok(s) end end |
#truetok(s) ⇒ Object
224 |
# File 'lib/okjson.rb', line 224 def truetok(s); s[0,4] == 'true' ? [:val, 'true', true] : [] end |
#ucharcopy(t, s, i) ⇒ Object
Copies the valid UTF-8 bytes of a single character from string s at position i to I/O object t, and returns the number of bytes copied. If no valid UTF-8 char exists at position i, ucharcopy writes Ustrerr and returns 1.
503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 |
# File 'lib/okjson.rb', line 503 def ucharcopy(t, s, i) n = s.length - i raise Utf8Error if n < 1 c0 = s[i].ord # 1-byte, 7-bit sequence? if c0 < Utagx t.putc(c0) return 1 end raise Utf8Error if c0 < Utag2 # unexpected continuation byte? raise Utf8Error if n < 2 # need continuation byte c1 = s[i+1].ord raise Utf8Error if c1 < Utagx || Utag2 <= c1 # 2-byte, 11-bit sequence? if c0 < Utag3 raise Utf8Error if ((c0&Umask2)<<6 | (c1&Umaskx)) <= Uchar1max t.putc(c0) t.putc(c1) return 2 end # need second continuation byte raise Utf8Error if n < 3 c2 = s[i+2].ord raise Utf8Error if c2 < Utagx || Utag2 <= c2 # 3-byte, 16-bit sequence? if c0 < Utag4 u = (c0&Umask3)<<12 | (c1&Umaskx)<<6 | (c2&Umaskx) raise Utf8Error if u <= Uchar2max t.putc(c0) t.putc(c1) t.putc(c2) return 3 end # need third continuation byte raise Utf8Error if n < 4 c3 = s[i+3].ord raise Utf8Error if c3 < Utagx || Utag2 <= c3 # 4-byte, 21-bit sequence? if c0 < Utag5 u = (c0&Umask4)<<18 | (c1&Umaskx)<<12 | (c2&Umaskx)<<6 | (c3&Umaskx) raise Utf8Error if u <= Uchar3max t.putc(c0) t.putc(c1) t.putc(c2) t.putc(c3) return 4 end raise Utf8Error rescue Utf8Error t.write(Ustrerr) return 1 end |
#ucharenc(a, i, u) ⇒ Object
Encodes unicode character u as UTF-8 bytes in string a at position i. Returns the number of bytes written.
340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 |
# File 'lib/okjson.rb', line 340 def ucharenc(a, i, u) case true when u <= Uchar1max a[i] = (u & 0xff).chr 1 when u <= Uchar2max a[i+0] = (Utag2 | ((u>>6)&0xff)).chr a[i+1] = (Utagx | (u&Umaskx)).chr 2 when u <= Uchar3max a[i+0] = (Utag3 | ((u>>12)&0xff)).chr a[i+1] = (Utagx | ((u>>6)&Umaskx)).chr a[i+2] = (Utagx | (u&Umaskx)).chr 3 else a[i+0] = (Utag4 | ((u>>18)&0xff)).chr a[i+1] = (Utagx | ((u>>12)&Umaskx)).chr a[i+2] = (Utagx | ((u>>6)&Umaskx)).chr a[i+3] = (Utagx | (u&Umaskx)).chr 4 end end |
#unquote(q) ⇒ Object
Converts a quoted json string literal q into a UTF-8-encoded string. The rules are different than for Ruby, so we cannot use eval. Unquote will raise an error if q contains control characters.
265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 |
# File 'lib/okjson.rb', line 265 def unquote(q) q = q[1...-1] a = q.dup # allocate a big enough string rubydoesenc = false # In ruby >= 1.9, a[w] is a codepoint, not a byte. if a.class.method_defined?(:force_encoding) a.force_encoding('UTF-8') rubydoesenc = true end r, w = 0, 0 while r < q.length c = q[r] case true when c == ?\\ r += 1 if r >= q.length raise Error, "string literal ends with a \"\\\": \"#{q}\"" end case q[r] when ?",?\\,?/,?' a[w] = q[r] r += 1 w += 1 when ?b,?f,?n,?r,?t a[w] = Unesc[q[r]] r += 1 w += 1 when ?u r += 1 uchar = begin hexdec4(q[r,4]) rescue RuntimeError => e raise Error, "invalid escape sequence \\u#{q[r,4]}: #{e}" end r += 4 if surrogate? uchar if q.length >= r+6 uchar1 = hexdec4(q[r+2,4]) uchar = subst(uchar, uchar1) if uchar != Ucharerr # A valid pair; consume. r += 6 end end end if rubydoesenc a[w] = '' << uchar w += 1 else w += ucharenc(a, w, uchar) end else raise Error, "invalid escape char #{q[r]} in \"#{q}\"" end when c == ?", c < Spc raise Error, "invalid character in string literal \"#{q}\"" else # Copy anything else byte-for-byte. # Valid UTF-8 will remain valid UTF-8. # Invalid UTF-8 will remain invalid UTF-8. # In ruby >= 1.9, c is a codepoint, not a byte, # in which case this is still what we want. a[w] = c r += 1 w += 1 end end a[0,w] end |
#valenc(x) ⇒ Object
415 416 417 418 419 420 421 422 423 424 425 426 427 |
# File 'lib/okjson.rb', line 415 def valenc(x) case x when Hash then objenc(x) when Array then arrenc(x) when String then strenc(x) when Numeric then numenc(x) when true then "true" when false then "false" when nil then "null" else raise Error, "cannot encode #{x.class}: #{x.inspect}" end end |
#valparse(ts) ⇒ Object
Parses a “value” in the sense of RFC 4627. Returns the parsed value and any trailing tokens.
74 75 76 77 78 79 80 81 82 83 84 85 86 87 |
# File 'lib/okjson.rb', line 74 def valparse(ts) if ts.length < 0 raise Error, 'empty' end typ, _, val = ts[0] case typ when '{' then objparse(ts) when '[' then arrparse(ts) when :val,:str then [val, ts[1..-1]] else raise Error, "unexpected #{val.inspect}" end end |