· 10 years ago · Sep 04, 2016, 06:12 PM
1-- -*- coding: utf-8 -*-
2--
3-- Simple JSON encoding and decoding in pure Lua.
4--
5-- Copyright 2010-2016 Jeffrey Friedl
6-- http://regex.info/blog/
7-- Latest version: http://regex.info/blog/lua/json
8--
9-- This code is released under a Creative Commons CC-BY "Attribution" License:
10-- http://creativecommons.org/licenses/by/3.0/deed.en_US
11--
12-- It can be used for any purpose so long as the copyright notice above,
13-- the web-page links above, and the 'AUTHOR_NOTE' string below are
14-- maintained. Enjoy.
15--
16local VERSION = 20160728.17 -- version history at end of file
17local AUTHOR_NOTE = "-[ JSON.lua package by Jeffrey Friedl (http://regex.info/blog/lua/json) version 20160728.17 ]-"
18
19--
20-- The 'AUTHOR_NOTE' variable exists so that information about the source
21-- of the package is maintained even in compiled versions. It's also
22-- included in OBJDEF below mostly to quiet warnings about unused variables.
23--
24local OBJDEF = {
25 VERSION = VERSION,
26 AUTHOR_NOTE = AUTHOR_NOTE,
27}
28
29
30--
31-- Simple JSON encoding and decoding in pure Lua.
32-- JSON definition: http://www.json.org/
33--
34--
35-- JSON = assert(loadfile "JSON.lua")() -- one-time load of the routines
36--
37-- local lua_value = JSON:decode(raw_json_text)
38--
39-- local raw_json_text = JSON:encode(lua_table_or_value)
40-- local pretty_json_text = JSON:encode_pretty(lua_table_or_value) -- "pretty printed" version for human readability
41--
42--
43--
44-- DECODING (from a JSON string to a Lua table)
45--
46--
47-- JSON = assert(loadfile "JSON.lua")() -- one-time load of the routines
48--
49-- local lua_value = JSON:decode(raw_json_text)
50--
51-- If the JSON text is for an object or an array, e.g.
52-- { "what": "books", "count": 3 }
53-- or
54-- [ "Larry", "Curly", "Moe" ]
55--
56-- the result is a Lua table, e.g.
57-- { what = "books", count = 3 }
58-- or
59-- { "Larry", "Curly", "Moe" }
60--
61--
62-- The encode and decode routines accept an optional second argument,
63-- "etc", which is not used during encoding or decoding, but upon error
64-- is passed along to error handlers. It can be of any type (including nil).
65--
66--
67--
68-- ERROR HANDLING
69--
70-- With most errors during decoding, this code calls
71--
72-- JSON:onDecodeError(message, text, location, etc)
73--
74-- with a message about the error, and if known, the JSON text being
75-- parsed and the byte count where the problem was discovered. You can
76-- replace the default JSON:onDecodeError() with your own function.
77--
78-- The default onDecodeError() merely augments the message with data
79-- about the text and the location if known (and if a second 'etc'
80-- argument had been provided to decode(), its value is tacked onto the
81-- message as well), and then calls JSON.assert(), which itself defaults
82-- to Lua's built-in assert(), and can also be overridden.
83--
84-- For example, in an Adobe Lightroom plugin, you might use something like
85--
86-- function JSON:onDecodeError(message, text, location, etc)
87-- LrErrors.throwUserError("Internal Error: invalid JSON data")
88-- end
89--
90-- or even just
91--
92-- function JSON.assert(message)
93-- LrErrors.throwUserError("Internal Error: " .. message)
94-- end
95--
96-- If JSON:decode() is passed a nil, this is called instead:
97--
98-- JSON:onDecodeOfNilError(message, nil, nil, etc)
99--
100-- and if JSON:decode() is passed HTML instead of JSON, this is called:
101--
102-- JSON:onDecodeOfHTMLError(message, text, nil, etc)
103--
104-- The use of the fourth 'etc' argument allows stronger coordination
105-- between decoding and error reporting, especially when you provide your
106-- own error-handling routines. Continuing with the the Adobe Lightroom
107-- plugin example:
108--
109-- function JSON:onDecodeError(message, text, location, etc)
110-- local note = "Internal Error: invalid JSON data"
111-- if type(etc) = 'table' and etc.photo then
112-- note = note .. " while processing for " .. etc.photo:getFormattedMetadata('fileName')
113-- end
114-- LrErrors.throwUserError(note)
115-- end
116--
117-- :
118-- :
119--
120-- for i, photo in ipairs(photosToProcess) do
121-- :
122-- :
123-- local data = JSON:decode(someJsonText, { photo = photo })
124-- :
125-- :
126-- end
127--
128--
129--
130--
131--
132-- DECODING AND STRICT TYPES
133--
134-- Because both JSON objects and JSON arrays are converted to Lua tables,
135-- it's not normally possible to tell which original JSON type a
136-- particular Lua table was derived from, or guarantee decode-encode
137-- round-trip equivalency.
138--
139-- However, if you enable strictTypes, e.g.
140--
141-- JSON = assert(loadfile "JSON.lua")() --load the routines
142-- JSON.strictTypes = true
143--
144-- then the Lua table resulting from the decoding of a JSON object or
145-- JSON array is marked via Lua metatable, so that when re-encoded with
146-- JSON:encode() it ends up as the appropriate JSON type.
147--
148-- (This is not the default because other routines may not work well with
149-- tables that have a metatable set, for example, Lightroom API calls.)
150--
151--
152-- ENCODING (from a lua table to a JSON string)
153--
154-- JSON = assert(loadfile "JSON.lua")() -- one-time load of the routines
155--
156-- local raw_json_text = JSON:encode(lua_table_or_value)
157-- local pretty_json_text = JSON:encode_pretty(lua_table_or_value) -- "pretty printed" version for human readability
158-- local custom_pretty = JSON:encode(lua_table_or_value, etc, { pretty = true, indent = "| ", align_keys = false })
159--
160-- On error during encoding, this code calls:
161--
162-- JSON:onEncodeError(message, etc)
163--
164-- which you can override in your local JSON object.
165--
166-- The 'etc' in the error call is the second argument to encode()
167-- and encode_pretty(), or nil if it wasn't provided.
168--
169--
170-- ENCODING OPTIONS
171--
172-- An optional third argument, a table of options, can be provided to encode().
173--
174-- encode_options = {
175-- -- options for making "pretty" human-readable JSON (see "PRETTY-PRINTING" below)
176-- pretty = true,
177-- indent = " ",
178-- align_keys = false,
179--
180-- -- other output-related options
181-- null = "\0", -- see "ENCODING JSON NULL VALUES" below
182-- stringsAreUtf8 = false, -- see "HANDLING UNICODE LINE AND PARAGRAPH SEPARATORS FOR JAVA" below
183-- }
184--
185-- json_string = JSON:encode(mytable, etc, encode_options)
186--
187--
188--
189-- For reference, the defaults are:
190--
191-- pretty = false
192-- null = nil,
193-- stringsAreUtf8 = false,
194--
195--
196--
197-- PRETTY-PRINTING
198--
199-- Enabling the 'pretty' encode option helps generate human-readable JSON.
200--
201-- pretty = JSON:encode(val, etc, {
202-- pretty = true,
203-- indent = " ",
204-- align_keys = false,
205-- })
206--
207-- encode_pretty() is also provided: it's identical to encode() except
208-- that encode_pretty() provides a default options table if none given in the call:
209--
210-- { pretty = true, align_keys = false, indent = " " }
211--
212-- For example, if
213--
214-- JSON:encode(data)
215--
216-- produces:
217--
218-- {"city":"Kyoto","climate":{"avg_temp":16,"humidity":"high","snowfall":"minimal"},"country":"Japan","wards":11}
219--
220-- then
221--
222-- JSON:encode_pretty(data)
223--
224-- produces:
225--
226-- {
227-- "city": "Kyoto",
228-- "climate": {
229-- "avg_temp": 16,
230-- "humidity": "high",
231-- "snowfall": "minimal"
232-- },
233-- "country": "Japan",
234-- "wards": 11
235-- }
236--
237-- The following three lines return identical results:
238-- JSON:encode_pretty(data)
239-- JSON:encode_pretty(data, nil, { pretty = true, align_keys = false, indent = " " })
240-- JSON:encode (data, nil, { pretty = true, align_keys = false, indent = " " })
241--
242-- An example of setting your own indent string:
243--
244-- JSON:encode_pretty(data, nil, { pretty = true, indent = "| " })
245--
246-- produces:
247--
248-- {
249-- | "city": "Kyoto",
250-- | "climate": {
251-- | | "avg_temp": 16,
252-- | | "humidity": "high",
253-- | | "snowfall": "minimal"
254-- | },
255-- | "country": "Japan",
256-- | "wards": 11
257-- }
258--
259-- An example of setting align_keys to true:
260--
261-- JSON:encode_pretty(data, nil, { pretty = true, indent = " ", align_keys = true })
262--
263-- produces:
264--
265-- {
266-- "city": "Kyoto",
267-- "climate": {
268-- "avg_temp": 16,
269-- "humidity": "high",
270-- "snowfall": "minimal"
271-- },
272-- "country": "Japan",
273-- "wards": 11
274-- }
275--
276-- which I must admit is kinda ugly, sorry. This was the default for
277-- encode_pretty() prior to version 20141223.14.
278--
279--
280-- HANDLING UNICODE LINE AND PARAGRAPH SEPARATORS FOR JAVA
281--
282-- If the 'stringsAreUtf8' encode option is set to true, consider Lua strings not as a sequence of bytes,
283-- but as a sequence of UTF-8 characters.
284--
285-- Currently, the only practical effect of setting this option is that Unicode LINE and PARAGRAPH
286-- separators, if found in a string, are encoded with a JSON escape instead of being dumped as is.
287-- The JSON is valid either way, but encoding this way, apparently, allows the resulting JSON
288-- to also be valid Java.
289--
290-- AMBIGUOUS SITUATIONS DURING THE ENCODING
291--
292-- During the encode, if a Lua table being encoded contains both string
293-- and numeric keys, it fits neither JSON's idea of an object, nor its
294-- idea of an array. To get around this, when any string key exists (or
295-- when non-positive numeric keys exist), numeric keys are converted to
296-- strings.
297--
298-- For example,
299-- JSON:encode({ "one", "two", "three", SOMESTRING = "some string" }))
300-- produces the JSON object
301-- {"1":"one","2":"two","3":"three","SOMESTRING":"some string"}
302--
303-- To prohibit this conversion and instead make it an error condition, set
304-- JSON.noKeyConversion = true
305--
306--
307-- ENCODING JSON NULL VALUES
308--
309-- Lua tables completely omit keys whose value is nil, so without special handling there's
310-- no way to get a field in a JSON object with a null value. For example
311-- JSON:encode({ username = "admin", password = nil })
312-- produces
313-- {"username":"admin"}
314--
315-- In order to actually produce
316-- {"username":"admin", "password":null}
317-- one can include a string value for a "null" field in the options table passed to encode()....
318-- any Lua table entry with that value becomes null in the JSON output:
319-- JSON:encode({ username = "admin", password = "xyzzy" }, nil, { null = "xyzzy" })
320-- produces
321-- {"username":"admin", "password":null}
322--
323-- Just be sure to use a string that is otherwise unlikely to appear in your data.
324-- The string "\0" (a string with one null byte) may well be appropriate for many applications.
325--
326-- The "null" options also applies to Lua tables that become JSON arrays.
327-- JSON:encode({ "one", "two", nil, nil })
328-- produces
329-- ["one","two"]
330-- while
331-- NULL = "\0"
332-- JSON:encode({ "one", "two", NULL, NULL}, nil, { null = NULL })
333-- produces
334-- ["one","two",null,null]
335--
336--
337-- HANDLING LARGE AND/OR PRECISE NUMBERS
338--
339-- Without special handling, numbers in JSON can lose precision in Lua.
340-- For example:
341--
342-- T = JSON:decode('{ "small":12345, "big":12345678901234567890123456789, "precise":9876.67890123456789012345 }')
343--
344-- print("small: ", type(T.small), T.small)
345-- print("big: ", type(T.big), T.big)
346-- print("precise: ", type(T.precise), T.precise)
347--
348-- produces
349--
350-- small: number 12345
351-- big: number 1.2345678901235e+28
352-- precise: number 9876.6789012346
353--
354-- Precision is lost with both 'big' and 'precise'.
355--
356-- This package offers ways to try to handle this better (for some definitions of "better")...
357--
358-- The most precise method is by setting the global:
359--
360-- JSON.decodeNumbersAsObjects = true
361--
362-- When this is set, numeric JSON data is encoded into Lua in a form that preserves the exact
363-- JSON numeric presentation when re-encoded back out to JSON, or accessed in Lua as a string.
364--
365-- (This is done by encoding the numeric data with a Lua table/metatable that returns
366-- the possibly-imprecise numeric form when accessed numerically, but the original precise
367-- representation when accessed as a string.)
368--
369-- Consider the example above, with this option turned on:
370--
371-- JSON.decodeNumbersAsObjects = true
372--
373-- T = JSON:decode('{ "small":12345, "big":12345678901234567890123456789, "precise":9876.67890123456789012345 }')
374--
375-- print("small: ", type(T.small), T.small)
376-- print("big: ", type(T.big), T.big)
377-- print("precise: ", type(T.precise), T.precise)
378--
379-- This now produces:
380--
381-- small: table 12345
382-- big: table 12345678901234567890123456789
383-- precise: table 9876.67890123456789012345
384--
385-- However, within Lua you can still use the values (e.g. T.precise in the example above) in numeric
386-- contexts. In such cases you'll get the possibly-imprecise numeric version, but in string contexts
387-- and when the data finds its way to this package's encode() function, the original full-precision
388-- representation is used.
389--
390-- Even without using the JSON.decodeNumbersAsObjects option, you can encode numbers
391-- in your Lua table that retain high precision upon encoding to JSON, by using the JSON:asNumber()
392-- function:
393--
394-- T = {
395-- imprecise = 123456789123456789.123456789123456789,
396-- precise = JSON:asNumber("123456789123456789.123456789123456789")
397-- }
398--
399-- print(JSON:encode_pretty(T))
400--
401-- This produces:
402--
403-- {
404-- "precise": 123456789123456789.123456789123456789,
405-- "imprecise": 1.2345678912346e+17
406-- }
407--
408--
409--
410-- A different way to handle big/precise JSON numbers is to have decode() merely return
411-- the exact string representation of the number instead of the number itself.
412-- This approach might be useful when the numbers are merely some kind of opaque
413-- object identifier and you want to work with them in Lua as strings anyway.
414--
415-- This approach is enabled by setting
416--
417-- JSON.decodeIntegerStringificationLength = 10
418--
419-- The value is the number of digits (of the integer part of the number) at which to stringify numbers.
420--
421-- Consider our previous example with this option set to 10:
422--
423-- JSON.decodeIntegerStringificationLength = 10
424--
425-- T = JSON:decode('{ "small":12345, "big":12345678901234567890123456789, "precise":9876.67890123456789012345 }')
426--
427-- print("small: ", type(T.small), T.small)
428-- print("big: ", type(T.big), T.big)
429-- print("precise: ", type(T.precise), T.precise)
430--
431-- This produces:
432--
433-- small: number 12345
434-- big: string 12345678901234567890123456789
435-- precise: number 9876.6789012346
436--
437-- The long integer of the 'big' field is at least JSON.decodeIntegerStringificationLength digits
438-- in length, so it's converted not to a Lua integer but to a Lua string. Using a value of 0 or 1 ensures
439-- that all JSON numeric data becomes strings in Lua.
440--
441-- Note that unlike
442-- JSON.decodeNumbersAsObjects = true
443-- this stringification is simple and unintelligent: the JSON number simply becomes a Lua string, and that's the end of it.
444-- If the string is then converted back to JSON, it's still a string. After running the code above, adding
445-- print(JSON:encode(T))
446-- produces
447-- {"big":"12345678901234567890123456789","precise":9876.6789012346,"small":12345}
448-- which is unlikely to be desired.
449--
450-- There's a comparable option for the length of the decimal part of a number:
451--
452-- JSON.decodeDecimalStringificationLength
453--
454-- This can be used alone or in conjunction with
455--
456-- JSON.decodeIntegerStringificationLength
457--
458-- to trip stringification on precise numbers with at least JSON.decodeIntegerStringificationLength digits after
459-- the decimal point.
460--
461-- This example:
462--
463-- JSON.decodeIntegerStringificationLength = 10
464-- JSON.decodeDecimalStringificationLength = 5
465--
466-- T = JSON:decode('{ "small":12345, "big":12345678901234567890123456789, "precise":9876.67890123456789012345 }')
467--
468-- print("small: ", type(T.small), T.small)
469-- print("big: ", type(T.big), T.big)
470-- print("precise: ", type(T.precise), T.precise)
471--
472-- produces:
473--
474-- small: number 12345
475-- big: string 12345678901234567890123456789
476-- precise: string 9876.67890123456789012345
477--
478--
479--
480--
481--
482-- SUMMARY OF METHODS YOU CAN OVERRIDE IN YOUR LOCAL LUA JSON OBJECT
483--
484-- assert
485-- onDecodeError
486-- onDecodeOfNilError
487-- onDecodeOfHTMLError
488-- onEncodeError
489--
490-- If you want to create a separate Lua JSON object with its own error handlers,
491-- you can reload JSON.lua or use the :new() method.
492--
493---------------------------------------------------------------------------
494
495local default_pretty_indent = " "
496local default_pretty_options = { pretty = true, align_keys = false, indent = default_pretty_indent }
497
498local isArray = { __tostring = function() return "JSON array" end } isArray.__index = isArray
499local isObject = { __tostring = function() return "JSON object" end } isObject.__index = isObject
500
501function OBJDEF:newArray(tbl)
502 return setmetatable(tbl or {}, isArray)
503end
504
505function OBJDEF:newObject(tbl)
506 return setmetatable(tbl or {}, isObject)
507end
508
509
510
511
512local function getnum(op)
513 return type(op) == 'number' and op or op.N
514end
515
516local isNumber = {
517 __index = isNumber,
518
519 __tostring = function(T) return T.S end,
520 __unm = function(op) return getnum(op) end,
521
522 __concat = function(op1, op2) return tostring(op1) .. tostring(op2) end,
523 __add = function(op1, op2) return getnum(op1) + getnum(op2) end,
524 __sub = function(op1, op2) return getnum(op1) - getnum(op2) end,
525 __mul = function(op1, op2) return getnum(op1) * getnum(op2) end,
526 __div = function(op1, op2) return getnum(op1) / getnum(op2) end,
527 __mod = function(op1, op2) return getnum(op1) % getnum(op2) end,
528 __pow = function(op1, op2) return getnum(op1) ^ getnum(op2) end,
529 __lt = function(op1, op2) return getnum(op1) < getnum(op2) end,
530 __eq = function(op1, op2) return getnum(op1) == getnum(op2) end,
531 __le = function(op1, op2) return getnum(op1) <= getnum(op2) end,
532}
533
534function OBJDEF:asNumber(item)
535
536 if getmetatable(item) == isNumber then
537 -- it's already a JSON number object.
538 return item
539 elseif type(item) == 'table' and type(item.S) == 'string' and type(item.N) == 'number' then
540 -- it's a number-object table that lost its metatable, so give it one
541 return setmetatable(item, isNumber)
542 else
543 -- the normal situation... given a number or a string representation of a number....
544 local holder = {
545 S = tostring(item), -- S is the representation of the number as a string, which remains precise
546 N = tonumber(item), -- N is the number as a Lua number.
547 }
548 return setmetatable(holder, isNumber)
549 end
550end
551
552local function unicode_codepoint_as_utf8(codepoint)
553 --
554 -- codepoint is a number
555 --
556 if codepoint <= 127 then
557 return string.char(codepoint)
558
559 elseif codepoint <= 2047 then
560 --
561 -- 110yyyxx 10xxxxxx <-- useful notation from http://en.wikipedia.org/wiki/Utf8
562 --
563 local highpart = math.floor(codepoint / 0x40)
564 local lowpart = codepoint - (0x40 * highpart)
565 return string.char(0xC0 + highpart,
566 0x80 + lowpart)
567
568 elseif codepoint <= 65535 then
569 --
570 -- 1110yyyy 10yyyyxx 10xxxxxx
571 --
572 local highpart = math.floor(codepoint / 0x1000)
573 local remainder = codepoint - 0x1000 * highpart
574 local midpart = math.floor(remainder / 0x40)
575 local lowpart = remainder - 0x40 * midpart
576
577 highpart = 0xE0 + highpart
578 midpart = 0x80 + midpart
579 lowpart = 0x80 + lowpart
580
581 --
582 -- Check for an invalid character (thanks Andy R. at Adobe).
583 -- See table 3.7, page 93, in http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#G28070
584 --
585 if ( highpart == 0xE0 and midpart < 0xA0 ) or
586 ( highpart == 0xED and midpart > 0x9F ) or
587 ( highpart == 0xF0 and midpart < 0x90 ) or
588 ( highpart == 0xF4 and midpart > 0x8F )
589 then
590 return "?"
591 else
592 return string.char(highpart,
593 midpart,
594 lowpart)
595 end
596
597 else
598 --
599 -- 11110zzz 10zzyyyy 10yyyyxx 10xxxxxx
600 --
601 local highpart = math.floor(codepoint / 0x40000)
602 local remainder = codepoint - 0x40000 * highpart
603 local midA = math.floor(remainder / 0x1000)
604 remainder = remainder - 0x1000 * midA
605 local midB = math.floor(remainder / 0x40)
606 local lowpart = remainder - 0x40 * midB
607
608 return string.char(0xF0 + highpart,
609 0x80 + midA,
610 0x80 + midB,
611 0x80 + lowpart)
612 end
613end
614
615function OBJDEF:onDecodeError(message, text, location, etc)
616 if text then
617 if location then
618 message = string.format("%s at char %d of: %s", message, location, text)
619 else
620 message = string.format("%s: %s", message, text)
621 end
622 end
623
624 if etc ~= nil then
625 message = message .. " (" .. OBJDEF:encode(etc) .. ")"
626 end
627
628 if self.assert then
629 self.assert(false, message)
630 else
631 assert(false, message)
632 end
633end
634
635OBJDEF.onDecodeOfNilError = OBJDEF.onDecodeError
636OBJDEF.onDecodeOfHTMLError = OBJDEF.onDecodeError
637
638function OBJDEF:onEncodeError(message, etc)
639 if etc ~= nil then
640 message = message .. " (" .. OBJDEF:encode(etc) .. ")"
641 end
642
643 if self.assert then
644 self.assert(false, message)
645 else
646 assert(false, message)
647 end
648end
649
650local function grok_number(self, text, start, options)
651 --
652 -- Grab the integer part
653 --
654 local integer_part = text:match('^-?[1-9]%d*', start)
655 or text:match("^-?0", start)
656
657 if not integer_part then
658 self:onDecodeError("expected number", text, start, options.etc)
659 end
660
661 local i = start + integer_part:len()
662
663 --
664 -- Grab an optional decimal part
665 --
666 local decimal_part = text:match('^%.%d+', i) or ""
667
668 i = i + decimal_part:len()
669
670 --
671 -- Grab an optional exponential part
672 --
673 local exponent_part = text:match('^[eE][-+]?%d+', i) or ""
674
675 i = i + exponent_part:len()
676
677 local full_number_text = integer_part .. decimal_part .. exponent_part
678
679 if options.decodeNumbersAsObjects then
680 return OBJDEF:asNumber(full_number_text), i
681 end
682
683 --
684 -- If we're told to stringify under certain conditions, so do.
685 -- We punt a bit when there's an exponent by just stringifying no matter what.
686 -- I suppose we should really look to see whether the exponent is actually big enough one
687 -- way or the other to trip stringification, but I'll be lazy about it until someone asks.
688 --
689 if (options.decodeIntegerStringificationLength
690 and
691 (integer_part:len() >= options.decodeIntegerStringificationLength or exponent_part:len() > 0))
692
693 or
694
695 (options.decodeDecimalStringificationLength
696 and
697 (decimal_part:len() >= options.decodeDecimalStringificationLength or exponent_part:len() > 0))
698 then
699 return full_number_text, i -- this returns the exact string representation seen in the original JSON
700 end
701
702
703
704 local as_number = tonumber(full_number_text)
705
706 if not as_number then
707 self:onDecodeError("bad number", text, start, options.etc)
708 end
709
710 return as_number, i
711end
712
713
714local function grok_string(self, text, start, options)
715
716 if text:sub(start,start) ~= '"' then
717 self:onDecodeError("expected string's opening quote", text, start, options.etc)
718 end
719
720 local i = start + 1 -- +1 to bypass the initial quote
721 local text_len = text:len()
722 local VALUE = ""
723 while i <= text_len do
724 local c = text:sub(i,i)
725 if c == '"' then
726 return VALUE, i + 1
727 end
728 if c ~= '\\' then
729 VALUE = VALUE .. c
730 i = i + 1
731 elseif text:match('^\\b', i) then
732 VALUE = VALUE .. "\b"
733 i = i + 2
734 elseif text:match('^\\f', i) then
735 VALUE = VALUE .. "\f"
736 i = i + 2
737 elseif text:match('^\\n', i) then
738 VALUE = VALUE .. "\n"
739 i = i + 2
740 elseif text:match('^\\r', i) then
741 VALUE = VALUE .. "\r"
742 i = i + 2
743 elseif text:match('^\\t', i) then
744 VALUE = VALUE .. "\t"
745 i = i + 2
746 else
747 local hex = text:match('^\\u([0123456789aAbBcCdDeEfF][0123456789aAbBcCdDeEfF][0123456789aAbBcCdDeEfF][0123456789aAbBcCdDeEfF])', i)
748 if hex then
749 i = i + 6 -- bypass what we just read
750
751 -- We have a Unicode codepoint. It could be standalone, or if in the proper range and
752 -- followed by another in a specific range, it'll be a two-code surrogate pair.
753 local codepoint = tonumber(hex, 16)
754 if codepoint >= 0xD800 and codepoint <= 0xDBFF then
755 -- it's a hi surrogate... see whether we have a following low
756 local lo_surrogate = text:match('^\\u([dD][cdefCDEF][0123456789aAbBcCdDeEfF][0123456789aAbBcCdDeEfF])', i)
757 if lo_surrogate then
758 i = i + 6 -- bypass the low surrogate we just read
759 codepoint = 0x2400 + (codepoint - 0xD800) * 0x400 + tonumber(lo_surrogate, 16)
760 else
761 -- not a proper low, so we'll just leave the first codepoint as is and spit it out.
762 end
763 end
764 VALUE = VALUE .. unicode_codepoint_as_utf8(codepoint)
765
766 else
767
768 -- just pass through what's escaped
769 VALUE = VALUE .. text:match('^\\(.)', i)
770 i = i + 2
771 end
772 end
773 end
774
775 self:onDecodeError("unclosed string", text, start, options.etc)
776end
777
778local function skip_whitespace(text, start)
779
780 local _, match_end = text:find("^[ \n\r\t]+", start) -- [http://www.ietf.org/rfc/rfc4627.txt] Section 2
781 if match_end then
782 return match_end + 1
783 else
784 return start
785 end
786end
787
788local grok_one -- assigned later
789
790local function grok_object(self, text, start, options)
791
792 if text:sub(start,start) ~= '{' then
793 self:onDecodeError("expected '{'", text, start, options.etc)
794 end
795
796 local i = skip_whitespace(text, start + 1) -- +1 to skip the '{'
797
798 local VALUE = self.strictTypes and self:newObject { } or { }
799
800 if text:sub(i,i) == '}' then
801 return VALUE, i + 1
802 end
803 local text_len = text:len()
804 while i <= text_len do
805 local key, new_i = grok_string(self, text, i, options)
806
807 i = skip_whitespace(text, new_i)
808
809 if text:sub(i, i) ~= ':' then
810 self:onDecodeError("expected colon", text, i, options.etc)
811 end
812
813 i = skip_whitespace(text, i + 1)
814
815 local new_val, new_i = grok_one(self, text, i, options)
816
817 VALUE[key] = new_val
818
819 --
820 -- Expect now either '}' to end things, or a ',' to allow us to continue.
821 --
822 i = skip_whitespace(text, new_i)
823
824 local c = text:sub(i,i)
825
826 if c == '}' then
827 return VALUE, i + 1
828 end
829
830 if text:sub(i, i) ~= ',' then
831 self:onDecodeError("expected comma or '}'", text, i, options.etc)
832 end
833
834 i = skip_whitespace(text, i + 1)
835 end
836
837 self:onDecodeError("unclosed '{'", text, start, options.etc)
838end
839
840local function grok_array(self, text, start, options)
841 if text:sub(start,start) ~= '[' then
842 self:onDecodeError("expected '['", text, start, options.etc)
843 end
844
845 local i = skip_whitespace(text, start + 1) -- +1 to skip the '['
846 local VALUE = self.strictTypes and self:newArray { } or { }
847 if text:sub(i,i) == ']' then
848 return VALUE, i + 1
849 end
850
851 local VALUE_INDEX = 1
852
853 local text_len = text:len()
854 while i <= text_len do
855 local val, new_i = grok_one(self, text, i, options)
856
857 -- can't table.insert(VALUE, val) here because it's a no-op if val is nil
858 VALUE[VALUE_INDEX] = val
859 VALUE_INDEX = VALUE_INDEX + 1
860
861 i = skip_whitespace(text, new_i)
862
863 --
864 -- Expect now either ']' to end things, or a ',' to allow us to continue.
865 --
866 local c = text:sub(i,i)
867 if c == ']' then
868 return VALUE, i + 1
869 end
870 if text:sub(i, i) ~= ',' then
871 self:onDecodeError("expected comma or '['", text, i, options.etc)
872 end
873 i = skip_whitespace(text, i + 1)
874 end
875 self:onDecodeError("unclosed '['", text, start, options.etc)
876end
877
878
879grok_one = function(self, text, start, options)
880 -- Skip any whitespace
881 start = skip_whitespace(text, start)
882
883 if start > text:len() then
884 self:onDecodeError("unexpected end of string", text, nil, options.etc)
885 end
886
887 if text:find('^"', start) then
888 return grok_string(self, text, start, options.etc)
889
890 elseif text:find('^[-0123456789 ]', start) then
891 return grok_number(self, text, start, options)
892
893 elseif text:find('^%{', start) then
894 return grok_object(self, text, start, options)
895
896 elseif text:find('^%[', start) then
897 return grok_array(self, text, start, options)
898
899 elseif text:find('^true', start) then
900 return true, start + 4
901
902 elseif text:find('^false', start) then
903 return false, start + 5
904
905 elseif text:find('^null', start) then
906 return nil, start + 4
907
908 else
909 self:onDecodeError("can't parse JSON", text, start, options.etc)
910 end
911end
912
913function OBJDEF:decode(text, etc, options)
914 --
915 -- If the user didn't pass in a table of decode options, make an empty one.
916 --
917 if type(options) ~= 'table' then
918 options = {}
919 end
920
921 --
922 -- If they passed in an 'etc' argument, stuff it into the options.
923 -- (If not, any 'etc' field in the options they passed in remains to be used)
924 --
925 if etc ~= nil then
926 options.etc = etc
927 end
928
929
930 if type(self) ~= 'table' or self.__index ~= OBJDEF then
931 OBJDEF:onDecodeError("JSON:decode must be called in method format", nil, nil, options.etc)
932 end
933
934 if text == nil then
935 self:onDecodeOfNilError(string.format("nil passed to JSON:decode()"), nil, nil, options.etc)
936 elseif type(text) ~= 'string' then
937 self:onDecodeError(string.format("expected string argument to JSON:decode(), got %s", type(text)), nil, nil, options.etc)
938 end
939
940 if text:match('^%s*$') then
941 return nil
942 end
943
944 if text:match('^%s*<') then
945 -- Can't be JSON... we'll assume it's HTML
946 self:onDecodeOfHTMLError(string.format("html passed to JSON:decode()"), text, nil, options.etc)
947 end
948
949 --
950 -- Ensure that it's not UTF-32 or UTF-16.
951 -- Those are perfectly valid encodings for JSON (as per RFC 4627 section 3),
952 -- but this package can't handle them.
953 --
954 if text:sub(1,1):byte() == 0 or (text:len() >= 2 and text:sub(2,2):byte() == 0) then
955 self:onDecodeError("JSON package groks only UTF-8, sorry", text, nil, options.etc)
956 end
957
958 --
959 -- apply global options
960 --
961 if options.decodeNumbersAsObjects == nil then
962 options.decodeNumbersAsObjects = self.decodeNumbersAsObjects
963 end
964 if options.decodeIntegerStringificationLength == nil then
965 options.decodeIntegerStringificationLength = self.decodeIntegerStringificationLength
966 end
967 if options.decodeDecimalStringificationLength == nil then
968 options.decodeDecimalStringificationLength = self.decodeDecimalStringificationLength
969 end
970
971 local success, value = pcall(grok_one, self, text, 1, options)
972
973 if success then
974 return value
975 else
976 -- if JSON:onDecodeError() didn't abort out of the pcall, we'll have received the error message here as "value", so pass it along as an assert.
977 if self.assert then
978 self.assert(false, value)
979 else
980 assert(false, value)
981 end
982 -- and if we're still here, return a nil and throw the error message on as a second arg
983 return nil, value
984 end
985end
986
987local function backslash_replacement_function(c)
988 if c == "\n" then
989 return "\\n"
990 elseif c == "\r" then
991 return "\\r"
992 elseif c == "\t" then
993 return "\\t"
994 elseif c == "\b" then
995 return "\\b"
996 elseif c == "\f" then
997 return "\\f"
998 elseif c == '"' then
999 return '\\"'
1000 elseif c == '\\' then
1001 return '\\\\'
1002 else
1003 return string.format("\\u%04x", c:byte())
1004 end
1005end
1006
1007local chars_to_be_escaped_in_JSON_string
1008 = '['
1009 .. '"' -- class sub-pattern to match a double quote
1010 .. '%\\' -- class sub-pattern to match a backslash
1011 .. '%z' -- class sub-pattern to match a null
1012 .. '\001' .. '-' .. '\031' -- class sub-pattern to match control characters
1013 .. ']'
1014
1015
1016local LINE_SEPARATOR_as_utf8 = unicode_codepoint_as_utf8(0x2028)
1017local PARAGRAPH_SEPARATOR_as_utf8 = unicode_codepoint_as_utf8(0x2029)
1018local function json_string_literal(value, options)
1019 local newval = value:gsub(chars_to_be_escaped_in_JSON_string, backslash_replacement_function)
1020 if options.stringsAreUtf8 then
1021 --
1022 -- This feels really ugly to just look into a string for the sequence of bytes that we know to be a particular utf8 character,
1023 -- but utf8 was designed purposefully to make this kind of thing possible. Still, feels dirty.
1024 -- I'd rather decode the byte stream into a character stream, but it's not technically needed so
1025 -- not technically worth it.
1026 --
1027 newval = newval:gsub(LINE_SEPARATOR_as_utf8, '\\u2028'):gsub(PARAGRAPH_SEPARATOR_as_utf8,'\\u2029')
1028 end
1029 return '"' .. newval .. '"'
1030end
1031
1032local function object_or_array(self, T, etc)
1033 --
1034 -- We need to inspect all the keys... if there are any strings, we'll convert to a JSON
1035 -- object. If there are only numbers, it's a JSON array.
1036 --
1037 -- If we'll be converting to a JSON object, we'll want to sort the keys so that the
1038 -- end result is deterministic.
1039 --
1040 local string_keys = { }
1041 local number_keys = { }
1042 local number_keys_must_be_strings = false
1043 local maximum_number_key
1044
1045 for key in pairs(T) do
1046 if type(key) == 'string' then
1047 table.insert(string_keys, key)
1048 elseif type(key) == 'number' then
1049 table.insert(number_keys, key)
1050 if key <= 0 or key >= math.huge then
1051 number_keys_must_be_strings = true
1052 elseif not maximum_number_key or key > maximum_number_key then
1053 maximum_number_key = key
1054 end
1055 else
1056 self:onEncodeError("can't encode table with a key of type " .. type(key), etc)
1057 end
1058 end
1059
1060 if #string_keys == 0 and not number_keys_must_be_strings then
1061 --
1062 -- An empty table, or a numeric-only array
1063 --
1064 if #number_keys > 0 then
1065 return nil, maximum_number_key -- an array
1066 elseif tostring(T) == "JSON array" then
1067 return nil
1068 elseif tostring(T) == "JSON object" then
1069 return { }
1070 else
1071 -- have to guess, so we'll pick array, since empty arrays are likely more common than empty objects
1072 return nil
1073 end
1074 end
1075
1076 table.sort(string_keys)
1077
1078 local map
1079 if #number_keys > 0 then
1080 --
1081 -- If we're here then we have either mixed string/number keys, or numbers inappropriate for a JSON array
1082 -- It's not ideal, but we'll turn the numbers into strings so that we can at least create a JSON object.
1083 --
1084
1085 if self.noKeyConversion then
1086 self:onEncodeError("a table with both numeric and string keys could be an object or array; aborting", etc)
1087 end
1088
1089 --
1090 -- Have to make a shallow copy of the source table so we can remap the numeric keys to be strings
1091 --
1092 map = { }
1093 for key, val in pairs(T) do
1094 map[key] = val
1095 end
1096
1097 table.sort(number_keys)
1098
1099 --
1100 -- Throw numeric keys in there as strings
1101 --
1102 for _, number_key in ipairs(number_keys) do
1103 local string_key = tostring(number_key)
1104 if map[string_key] == nil then
1105 table.insert(string_keys , string_key)
1106 map[string_key] = T[number_key]
1107 else
1108 self:onEncodeError("conflict converting table with mixed-type keys into a JSON object: key " .. number_key .. " exists both as a string and a number.", etc)
1109 end
1110 end
1111 end
1112
1113 return string_keys, nil, map
1114end
1115
1116--
1117-- Encode
1118--
1119-- 'options' is nil, or a table with possible keys:
1120--
1121-- pretty -- If true, return a pretty-printed version.
1122--
1123-- indent -- A string (usually of spaces) used to indent each nested level.
1124--
1125-- align_keys -- If true, align all the keys when formatting a table.
1126--
1127-- null -- If this exists with a string value, table elements with this value are output as JSON null.
1128--
1129-- stringsAreUtf8 -- If true, consider Lua strings not as a sequence of bytes, but as a sequence of UTF-8 characters.
1130-- (Currently, the only practical effect of setting this option is that Unicode LINE and PARAGRAPH
1131-- separators, if found in a string, are encoded with a JSON escape instead of as raw UTF-8.
1132-- The JSON is valid either way, but encoding this way, apparently, allows the resulting JSON
1133-- to also be valid Java.)
1134--
1135--
1136local encode_value -- must predeclare because it calls itself
1137function encode_value(self, value, parents, etc, options, indent, for_key)
1138
1139 --
1140 -- keys in a JSON object can never be null, so we don't even consider options.null when converting a key value
1141 --
1142 if value == nil or (not for_key and options and options.null and value == options.null) then
1143 return 'null'
1144
1145 elseif type(value) == 'string' then
1146 return json_string_literal(value, options)
1147
1148 elseif type(value) == 'number' then
1149 if value ~= value then
1150 --
1151 -- NaN (Not a Number).
1152 -- JSON has no NaN, so we have to fudge the best we can. This should really be a package option.
1153 --
1154 return "null"
1155 elseif value >= math.huge then
1156 --
1157 -- Positive infinity. JSON has no INF, so we have to fudge the best we can. This should
1158 -- really be a package option. Note: at least with some implementations, positive infinity
1159 -- is both ">= math.huge" and "<= -math.huge", which makes no sense but that's how it is.
1160 -- Negative infinity is properly "<= -math.huge". So, we must be sure to check the ">="
1161 -- case first.
1162 --
1163 return "1e+9999"
1164 elseif value <= -math.huge then
1165 --
1166 -- Negative infinity.
1167 -- JSON has no INF, so we have to fudge the best we can. This should really be a package option.
1168 --
1169 return "-1e+9999"
1170 else
1171 return tostring(value)
1172 end
1173
1174 elseif type(value) == 'boolean' then
1175 return tostring(value)
1176
1177 elseif type(value) ~= 'table' then
1178 self:onEncodeError("can't convert " .. type(value) .. " to JSON", etc)
1179
1180 elseif getmetatable(value) == isNumber then
1181 return tostring(value)
1182 else
1183 --
1184 -- A table to be converted to either a JSON object or array.
1185 --
1186 local T = value
1187
1188 if type(options) ~= 'table' then
1189 options = {}
1190 end
1191 if type(indent) ~= 'string' then
1192 indent = ""
1193 end
1194
1195 if parents[T] then
1196 self:onEncodeError("table " .. tostring(T) .. " is a child of itself", etc)
1197 else
1198 parents[T] = true
1199 end
1200
1201 local result_value
1202
1203 local object_keys, maximum_number_key, map = object_or_array(self, T, etc)
1204 if maximum_number_key then
1205 --
1206 -- An array...
1207 --
1208 local ITEMS = { }
1209 for i = 1, maximum_number_key do
1210 table.insert(ITEMS, encode_value(self, T[i], parents, etc, options, indent))
1211 end
1212
1213 if options.pretty then
1214 result_value = "[ " .. table.concat(ITEMS, ", ") .. " ]"
1215 else
1216 result_value = "[" .. table.concat(ITEMS, ",") .. "]"
1217 end
1218
1219 elseif object_keys then
1220 --
1221 -- An object
1222 --
1223 local TT = map or T
1224
1225 if options.pretty then
1226
1227 local KEYS = { }
1228 local max_key_length = 0
1229 for _, key in ipairs(object_keys) do
1230 local encoded = encode_value(self, tostring(key), parents, etc, options, indent, true)
1231 if options.align_keys then
1232 max_key_length = math.max(max_key_length, #encoded)
1233 end
1234 table.insert(KEYS, encoded)
1235 end
1236 local key_indent = indent .. tostring(options.indent or "")
1237 local subtable_indent = key_indent .. string.rep(" ", max_key_length) .. (options.align_keys and " " or "")
1238 local FORMAT = "%s%" .. string.format("%d", max_key_length) .. "s: %s"
1239
1240 local COMBINED_PARTS = { }
1241 for i, key in ipairs(object_keys) do
1242 local encoded_val = encode_value(self, TT[key], parents, etc, options, subtable_indent)
1243 table.insert(COMBINED_PARTS, string.format(FORMAT, key_indent, KEYS[i], encoded_val))
1244 end
1245 result_value = "{\n" .. table.concat(COMBINED_PARTS, ",\n") .. "\n" .. indent .. "}"
1246
1247 else
1248
1249 local PARTS = { }
1250 for _, key in ipairs(object_keys) do
1251 local encoded_val = encode_value(self, TT[key], parents, etc, options, indent)
1252 local encoded_key = encode_value(self, tostring(key), parents, etc, options, indent, true)
1253 table.insert(PARTS, string.format("%s:%s", encoded_key, encoded_val))
1254 end
1255 result_value = "{" .. table.concat(PARTS, ",") .. "}"
1256
1257 end
1258 else
1259 --
1260 -- An empty array/object... we'll treat it as an array, though it should really be an option
1261 --
1262 result_value = "[]"
1263 end
1264
1265 parents[T] = false
1266 return result_value
1267 end
1268end
1269
1270
1271function OBJDEF:encode(value, etc, options)
1272 if type(self) ~= 'table' or self.__index ~= OBJDEF then
1273 OBJDEF:onEncodeError("JSON:encode must be called in method format", etc)
1274 end
1275
1276 --
1277 -- If the user didn't pass in a table of decode options, make an empty one.
1278 --
1279 if type(options) ~= 'table' then
1280 options = {}
1281 end
1282
1283 return encode_value(self, value, {}, etc, options)
1284end
1285
1286function OBJDEF:encode_pretty(value, etc, options)
1287 if type(self) ~= 'table' or self.__index ~= OBJDEF then
1288 OBJDEF:onEncodeError("JSON:encode_pretty must be called in method format", etc)
1289 end
1290
1291 --
1292 -- If the user didn't pass in a table of decode options, use the default pretty ones
1293 --
1294 if type(options) ~= 'table' then
1295 options = default_pretty_options
1296 end
1297
1298 return encode_value(self, value, {}, etc, options)
1299end
1300
1301function OBJDEF.__tostring()
1302 return "JSON encode/decode package"
1303end
1304
1305OBJDEF.__index = OBJDEF
1306
1307function OBJDEF:new(args)
1308 local new = { }
1309
1310 if args then
1311 for key, val in pairs(args) do
1312 new[key] = val
1313 end
1314 end
1315
1316 return setmetatable(new, OBJDEF)
1317end
1318
1319return OBJDEF:new()
1320
1321--
1322-- Version history:
1323--
1324-- 20160728.17 Added concatenation to the metatable for JSON:asNumber().
1325--
1326-- 20160709.16 Could crash if not passed an options table (thanks jarno heikkinen <jarnoh@capturemonkey.com>).
1327--
1328-- Made JSON:asNumber() a bit more resilient to being passed the results of itself.
1329--
1330-- 20160526.15 Added the ability to easily encode null values in JSON, via the new "null" encoding option.
1331-- (Thanks to Adam B for bringing up the issue.)
1332--
1333-- Added some support for very large numbers and precise floats via
1334-- JSON.decodeNumbersAsObjects
1335-- JSON.decodeIntegerStringificationLength
1336-- JSON.decodeDecimalStringificationLength
1337--
1338-- Added the "stringsAreUtf8" encoding option. (Hat tip to http://lua-users.org/wiki/JsonModules )
1339--
1340-- 20141223.14 The encode_pretty() routine produced fine results for small datasets, but isn't really
1341-- appropriate for anything large, so with help from Alex Aulbach I've made the encode routines
1342-- more flexible, and changed the default encode_pretty() to be more generally useful.
1343--
1344-- Added a third 'options' argument to the encode() and encode_pretty() routines, to control
1345-- how the encoding takes place.
1346--
1347-- Updated docs to add assert() call to the loadfile() line, just as good practice so that
1348-- if there is a problem loading JSON.lua, the appropriate error message will percolate up.
1349--
1350-- 20140920.13 Put back (in a way that doesn't cause warnings about unused variables) the author string,
1351-- so that the source of the package, and its version number, are visible in compiled copies.
1352--
1353-- 20140911.12 Minor lua cleanup.
1354-- Fixed internal reference to 'JSON.noKeyConversion' to reference 'self' instead of 'JSON'.
1355-- (Thanks to SmugMug's David Parry for these.)
1356--
1357-- 20140418.11 JSON nulls embedded within an array were being ignored, such that
1358-- ["1",null,null,null,null,null,"seven"],
1359-- would return
1360-- {1,"seven"}
1361-- It's now fixed to properly return
1362-- {1, nil, nil, nil, nil, nil, "seven"}
1363-- Thanks to "haddock" for catching the error.
1364--
1365-- 20140116.10 The user's JSON.assert() wasn't always being used. Thanks to "blue" for the heads up.
1366--
1367-- 20131118.9 Update for Lua 5.3... it seems that tostring(2/1) produces "2.0" instead of "2",
1368-- and this caused some problems.
1369--
1370-- 20131031.8 Unified the code for encode() and encode_pretty(); they had been stupidly separate,
1371-- and had of course diverged (encode_pretty didn't get the fixes that encode got, so
1372-- sometimes produced incorrect results; thanks to Mattie for the heads up).
1373--
1374-- Handle encoding tables with non-positive numeric keys (unlikely, but possible).
1375--
1376-- If a table has both numeric and string keys, or its numeric keys are inappropriate
1377-- (such as being non-positive or infinite), the numeric keys are turned into
1378-- string keys appropriate for a JSON object. So, as before,
1379-- JSON:encode({ "one", "two", "three" })
1380-- produces the array
1381-- ["one","two","three"]
1382-- but now something with mixed key types like
1383-- JSON:encode({ "one", "two", "three", SOMESTRING = "some string" }))
1384-- instead of throwing an error produces an object:
1385-- {"1":"one","2":"two","3":"three","SOMESTRING":"some string"}
1386--
1387-- To maintain the prior throw-an-error semantics, set
1388-- JSON.noKeyConversion = true
1389--
1390-- 20131004.7 Release under a Creative Commons CC-BY license, which I should have done from day one, sorry.
1391--
1392-- 20130120.6 Comment update: added a link to the specific page on my blog where this code can
1393-- be found, so that folks who come across the code outside of my blog can find updates
1394-- more easily.
1395--
1396-- 20111207.5 Added support for the 'etc' arguments, for better error reporting.
1397--
1398-- 20110731.4 More feedback from David Kolf on how to make the tests for Nan/Infinity system independent.
1399--
1400-- 20110730.3 Incorporated feedback from David Kolf at http://lua-users.org/wiki/JsonModules:
1401--
1402-- * When encoding lua for JSON, Sparse numeric arrays are now handled by
1403-- spitting out full arrays, such that
1404-- JSON:encode({"one", "two", [10] = "ten"})
1405-- returns
1406-- ["one","two",null,null,null,null,null,null,null,"ten"]
1407--
1408-- In 20100810.2 and earlier, only up to the first non-null value would have been retained.
1409--
1410-- * When encoding lua for JSON, numeric value NaN gets spit out as null, and infinity as "1+e9999".
1411-- Version 20100810.2 and earlier created invalid JSON in both cases.
1412--
1413-- * Unicode surrogate pairs are now detected when decoding JSON.
1414--
1415-- 20100810.2 added some checking to ensure that an invalid Unicode character couldn't leak in to the UTF-8 encoding
1416--
1417-- 20100731.1 initial public release
1418--