From d52f8d91c2349de91adcced0b8b47c791c711706 Mon Sep 17 00:00:00 2001 From: zihang Date: Fri, 27 Jun 2025 11:52:36 +0800 Subject: [PATCH 1/4] refactor: handle any json number --- builtin/builtin.mbti | 2 +- builtin/json.mbt | 14 +++++++------- json/from_json.mbt | 12 +++++++++--- json/internal_types.mbt | 2 +- json/json.mbt | 8 ++++++-- json/json_encode_decode_test.mbt | 4 ++-- json/json_test.mbt | 2 +- json/lex_main.mbt | 16 ++++++++-------- json/lex_number.mbt | 27 +++++++++++++++++---------- json/lex_number_test.mbt | 5 +---- json/parse.mbt | 2 +- json/types.mbt | 2 +- 12 files changed, 55 insertions(+), 41 deletions(-) diff --git a/builtin/builtin.mbti b/builtin/builtin.mbti index 3ae35e46c..a1255cb65 100644 --- a/builtin/builtin.mbti +++ b/builtin/builtin.mbti @@ -239,7 +239,7 @@ pub(all) enum Json { Null True False - Number(Double) + Number(Double, repr~ : String?) String(String) Array(Array[Json]) Object(Map[String, Json]) diff --git a/builtin/json.mbt b/builtin/json.mbt index 4f9483a8b..8ea5d008c 100644 --- a/builtin/json.mbt +++ b/builtin/json.mbt @@ -18,7 +18,7 @@ pub(all) enum Json { Null True False - Number(Double) + Number(Double, repr~ : String?) String(String) Array(Array[Json]) Object(Map[String, Json]) @@ -54,7 +54,7 @@ pub fn Json::null() -> Json { /// inspect(Json::number(3.14), content="Number(3.14)") /// ``` pub fn Json::number(number : Double) -> Json { - return Number(number) + return Number(number, repr=None) } ///| @@ -160,12 +160,12 @@ pub impl ToJson for Bool with to_json(self : Bool) -> Json { ///| pub impl ToJson for Byte with to_json(self : Byte) -> Json { - Number(self.to_double()) + Json::number(self.to_double()) } ///| pub impl ToJson for Int with to_json(self : Int) -> Json { - Number(self.to_double()) + Json::number(self.to_double()) } ///| @@ -175,7 +175,7 @@ pub impl ToJson for Int64 with to_json(self : Int64) -> Json { ///| pub impl ToJson for UInt with to_json(self : UInt) -> Json { - Number(self.to_uint64().to_double()) + Json::number(self.to_uint64().to_double()) } ///| @@ -190,12 +190,12 @@ pub impl ToJson for Double with to_json(self : Double) -> Json { self < 0xFFEFFFFFFFFFFFFFL.reinterpret_as_double() { return Null } - Number(self) + Json::number(self) } ///| pub impl ToJson for Float with to_json(self : Float) -> Json { - Number(self.to_double()) + Json::number(self.to_double()) } ///| diff --git a/json/from_json.mbt b/json/from_json.mbt index f5acf27f0..39530b9d8 100644 --- a/json/from_json.mbt +++ b/json/from_json.mbt @@ -45,7 +45,9 @@ pub impl FromJson for Bool with from_json(json, path) { ///| pub impl FromJson for Int with from_json(json, path) { - guard json is Number(n) else { + guard json is Number(n, ..) && + n != @double.infinity && + n != @double.neg_infinity else { decode_error(path, "Int::from_json: expected number") } n.to_int() @@ -65,7 +67,9 @@ pub impl FromJson for Int64 with from_json(json, path) { ///| pub impl FromJson for UInt with from_json(json, path) { - guard json is Number(n) else { + guard json is Number(n, ..) && + n != @double.infinity && + n != @double.neg_infinity else { decode_error(path, "UInt::from_json: expected number") } n.to_uint() @@ -85,7 +89,9 @@ pub impl FromJson for UInt64 with from_json(json, path) { ///| pub impl FromJson for Double with from_json(json, path) { - guard json is Number(n) else { + guard json is Number(n, ..) && + n != @double.infinity && + n != @double.neg_infinity else { decode_error(path, "Double::from_json: expected number") } n diff --git a/json/internal_types.mbt b/json/internal_types.mbt index f8bc309ea..21cab692d 100644 --- a/json/internal_types.mbt +++ b/json/internal_types.mbt @@ -55,7 +55,7 @@ priv enum Token { Null True False - Number(Double) + Number(Double, String?) String(String) LBrace RBrace diff --git a/json/json.mbt b/json/json.mbt index 466523e94..9118986a7 100644 --- a/json/json.mbt +++ b/json/json.mbt @@ -32,7 +32,7 @@ pub fn as_bool(self : JsonValue) -> Bool? { ///| /// Try to get this element as a Number pub fn as_number(self : JsonValue) -> Double? { - guard self is Number(n) else { return None } + guard self is Number(n, ..) else { return None } Some(n) } @@ -146,7 +146,11 @@ pub fn stringify( ..write_char('\"') .to_string() } - Number(n) => n.to_string() + Number(n, repr~) => + match repr { + None => n.to_string() + Some(r) => r + } True => "true" False => "false" Null => "null" diff --git a/json/json_encode_decode_test.mbt b/json/json_encode_decode_test.mbt index 8e7f4f61c..74965a4ec 100644 --- a/json/json_encode_decode_test.mbt +++ b/json/json_encode_decode_test.mbt @@ -53,13 +53,13 @@ fn of_json(jv : Json) -> AllThree raise DecodeError { let strings_result = [] for n in ints { match n { - Number(n) => ints_result.push(n.to_int()) + Number(n, ..) => ints_result.push(n.to_int()) _ => () // error handling here } } for n in floats { match n { - Number(n) => floats_result.push(n) + Number(n, ..) => floats_result.push(n) _ => () // error handling here } } diff --git a/json/json_test.mbt b/json/json_test.mbt index fcb4d21fc..8ee699258 100644 --- a/json/json_test.mbt +++ b/json/json_test.mbt @@ -248,7 +248,7 @@ test "stringify" { // we do come across issues like ParseError not unified with String let newjson = @json.parse(json.stringify()) match json { - { "key": [_, _, _, _, { "value": Number(i), .. }, ..], .. } => + { "key": [_, _, _, _, { "value": Number(i, ..), .. }, ..], .. } => inspect(i, content="100") _ => fail("Failed to match the JSON") } diff --git a/json/lex_main.mbt b/json/lex_main.mbt index e735223b2..ce465e597 100644 --- a/json/lex_main.mbt +++ b/json/lex_main.mbt @@ -62,25 +62,25 @@ fn ParseContext::lex_value( Some('-') => match ctx.read_char() { Some('0') => { - let n = ctx.lex_zero(start=ctx.offset - 2) - return Number(n) + let (n, repr) = ctx.lex_zero(start=ctx.offset - 2) + return Number(n, repr) } Some(c2) => { if c2 is ('1'..='9') { - let n = ctx.lex_decimal_integer(start=ctx.offset - 2) - return Number(n) + let (n, repr) = ctx.lex_decimal_integer(start=ctx.offset - 2) + return Number(n, repr) } ctx.invalid_char(shift=-1) } None => raise InvalidEof } Some('0') => { - let n = ctx.lex_zero(start=ctx.offset - 1) - return Number(n) + let (n, repr) = ctx.lex_zero(start=ctx.offset - 1) + return Number(n, repr) } Some('1'..='9') => { - let n = ctx.lex_decimal_integer(start=ctx.offset - 1) - return Number(n) + let (n, repr) = ctx.lex_decimal_integer(start=ctx.offset - 1) + return Number(n, repr) } Some('"') => { let s = ctx.lex_string() diff --git a/json/lex_number.mbt b/json/lex_number.mbt index 22ed6d06f..6ccabac3b 100644 --- a/json/lex_number.mbt +++ b/json/lex_number.mbt @@ -16,7 +16,7 @@ fn ParseContext::lex_decimal_integer( ctx : ParseContext, start~ : Int -) -> Double raise ParseError { +) -> (Double, String?) raise ParseError { for { match ctx.read_char() { Some('.') => return ctx.lex_decimal_point(start~) @@ -37,7 +37,7 @@ fn ParseContext::lex_decimal_integer( fn ParseContext::lex_decimal_point( ctx : ParseContext, start~ : Int -) -> Double raise ParseError { +) -> (Double, String?) raise ParseError { match ctx.read_char() { Some(c) => if c >= '0' && c <= '9' { @@ -53,7 +53,7 @@ fn ParseContext::lex_decimal_point( fn ParseContext::lex_decimal_fraction( ctx : ParseContext, start~ : Int -) -> Double raise ParseError { +) -> (Double, String?) raise ParseError { for { match ctx.read_char() { Some('e' | 'E') => return ctx.lex_decimal_exponent(start~) @@ -73,7 +73,7 @@ fn ParseContext::lex_decimal_fraction( fn ParseContext::lex_decimal_exponent( ctx : ParseContext, start~ : Int -) -> Double raise ParseError { +) -> (Double, String?) raise ParseError { match ctx.read_char() { Some('+') | Some('-') => return ctx.lex_decimal_exponent_sign(start~) Some(c) => { @@ -91,7 +91,7 @@ fn ParseContext::lex_decimal_exponent( fn ParseContext::lex_decimal_exponent_sign( ctx : ParseContext, start~ : Int -) -> Double raise ParseError { +) -> (Double, String?) raise ParseError { match ctx.read_char() { Some(c) => { if c >= '0' && c <= '9' { @@ -108,7 +108,7 @@ fn ParseContext::lex_decimal_exponent_sign( fn ParseContext::lex_decimal_exponent_integer( ctx : ParseContext, start~ : Int -) -> Double raise ParseError { +) -> (Double, String?) { for { match ctx.read_char() { Some(c) => { @@ -127,7 +127,7 @@ fn ParseContext::lex_decimal_exponent_integer( fn ParseContext::lex_zero( ctx : ParseContext, start~ : Int -) -> Double raise ParseError { +) -> (Double, String?) raise ParseError { match ctx.read_char() { Some('.') => ctx.lex_decimal_point(start~) Some('e' | 'E') => ctx.lex_decimal_exponent(start~) @@ -148,9 +148,16 @@ fn ParseContext::lex_number_end( ctx : ParseContext, start : Int, end : Int -) -> Double raise ParseError { +) -> (Double, String?) { let s = ctx.input.substring(start~, end~) - @strconv.parse_double(s) catch { - _ => raise InvalidNumber(offset_to_position(ctx.input, start), s) + (@strconv.parse_double(s), None) catch { + _ => + // If parsing fails, determine if it's positive or negative infinity + // based on the first character being '-' + if s is ['-', ..] { + (@double.neg_infinity, Some(s)) + } else { + (@double.infinity, Some(s)) + } } } diff --git a/json/lex_number_test.mbt b/json/lex_number_test.mbt index 1530dcf9d..3ec57b5e4 100644 --- a/json/lex_number_test.mbt +++ b/json/lex_number_test.mbt @@ -33,10 +33,7 @@ test "lex_zero invalid case" { ///| test "invalid number" { - inspect( - try? @json.parse("1e999999999"), - content="Err(Invalid number 1e999999999 at line 1, column 0)", - ) + inspect(try? @json.parse("1e999999999"), content="Ok(Number(Infinity))") } ///| diff --git a/json/parse.mbt b/json/parse.mbt index eda33d37e..8fa8d81c8 100644 --- a/json/parse.mbt +++ b/json/parse.mbt @@ -49,7 +49,7 @@ fn ParseContext::parse_value2( Null => Json::null() True => Json::boolean(true) False => Json::boolean(false) - Number(n) => Json::number(n) + Number(n, repr) => Number(n, repr~) String(s) => Json::string(s) LBrace => ctx.parse_object() LBracket => ctx.parse_array() diff --git a/json/types.mbt b/json/types.mbt index ca68fabec..4a8282574 100644 --- a/json/types.mbt +++ b/json/types.mbt @@ -61,7 +61,7 @@ pub impl Show for JsonValue with output(self, logger) { Null => logger.write_string("Null") True => logger.write_string("True") False => logger.write_string("False") - Number(n) => { + Number(n, repr=_) => { logger.write_string("Number(") Show::output(n, logger) logger.write_string(")") From 3158d8dc7dbaf6214a2855d68738faa0faf9df44 Mon Sep 17 00:00:00 2001 From: zihang Date: Fri, 4 Jul 2025 15:04:55 +0800 Subject: [PATCH 2/4] fix: handle unsafe numbers and update show & util --- builtin/builtin.mbti | 2 +- builtin/json.mbt | 10 +++++--- json/json_test.mbt | 6 +++++ json/lex_number.mbt | 36 +++++++++++++++++++++-------- json/lex_number_test.mbt | 49 ++++++++++++++++++++++++++++++++++++---- json/types.mbt | 6 ++++- 6 files changed, 91 insertions(+), 18 deletions(-) diff --git a/builtin/builtin.mbti b/builtin/builtin.mbti index a1255cb65..2592be490 100644 --- a/builtin/builtin.mbti +++ b/builtin/builtin.mbti @@ -247,7 +247,7 @@ pub(all) enum Json { fn Json::array(Array[Self]) -> Self fn Json::boolean(Bool) -> Self fn Json::null() -> Self -fn Json::number(Double) -> Self +fn Json::number(Double, repr? : String) -> Self fn Json::object(Map[String, Self]) -> Self fn Json::string(String) -> Self impl Default for Json diff --git a/builtin/json.mbt b/builtin/json.mbt index 8ea5d008c..a96b84667 100644 --- a/builtin/json.mbt +++ b/builtin/json.mbt @@ -51,10 +51,14 @@ pub fn Json::null() -> Json { /// Example: /// /// ```moonbit -/// inspect(Json::number(3.14), content="Number(3.14)") +/// inspect(Json::number(3.14), content="Number(3.14)") +/// inspect( +/// Json::number(@double.infinity, repr="1e9999999999999999999999999999999").stringify(), +/// content="1e9999999999999999999999999999999" +/// ) /// ``` -pub fn Json::number(number : Double) -> Json { - return Number(number, repr=None) +pub fn Json::number(number : Double, repr? : String) -> Json { + return Number(number, repr~) } ///| diff --git a/json/json_test.mbt b/json/json_test.mbt index 8ee699258..0fff679e0 100644 --- a/json/json_test.mbt +++ b/json/json_test.mbt @@ -386,6 +386,12 @@ test "stringify number" { for json in nums { match (try? @json.parse(json.stringify())) { Err(e) => err.push(e.to_string()) + Ok(Number(_, repr~) as newjson) => + if repr is Some(_) { + assert_eq(newjson.stringify(), json.stringify()) + } else { + assert_eq(newjson, json) + } Ok(newjson) => assert_eq(newjson, json) } } diff --git a/json/lex_number.mbt b/json/lex_number.mbt index 6ccabac3b..dfd98a684 100644 --- a/json/lex_number.mbt +++ b/json/lex_number.mbt @@ -150,14 +150,32 @@ fn ParseContext::lex_number_end( end : Int ) -> (Double, String?) { let s = ctx.input.substring(start~, end~) - (@strconv.parse_double(s), None) catch { - _ => - // If parsing fails, determine if it's positive or negative infinity - // based on the first character being '-' - if s is ['-', ..] { - (@double.neg_infinity, Some(s)) - } else { - (@double.infinity, Some(s)) - } + if not(s.contains(".")) && not(s.contains("e")) && not(s.contains("E")) { + // If the string does not contain a decimal point or exponent, it is likely an integer + // We can try to parse it as an integer first + let parsed_int = try? @strconv.parse_int64(s) + match parsed_int { + Ok(i) if i <= 9007199254740991 && i >= -9007199254740991 => + return (i.to_double(), None) + _ => + return if s is ['-', ..] { + (@double.neg_infinity, Some(s)) + } else { + (@double.infinity, Some(s)) + } + } + } else { + let parsed_double = try? @strconv.parse_double(s) + match parsed_double { + // For normal values, return without string representation + Ok(d) => (d, None) + // If parsing fails as a double, treat it as infinity and preserve the string + Err(_) => + if s is ['-', ..] { + (@double.neg_infinity, Some(s)) + } else { + (@double.infinity, Some(s)) + } + } } } diff --git a/json/lex_number_test.mbt b/json/lex_number_test.mbt index 3ec57b5e4..115968445 100644 --- a/json/lex_number_test.mbt +++ b/json/lex_number_test.mbt @@ -32,11 +32,52 @@ test "lex_zero invalid case" { } ///| -test "invalid number" { - inspect(try? @json.parse("1e999999999"), content="Ok(Number(Infinity))") +test "parse numbers" { + // Basic float parsing + inspect(@json.parse("123.45"), content="Number(123.45)") + inspect(@json.parse("-123.45"), content="Number(-123.45)") + + // Exponential notation + // Note: The actual format depends on the implementation details of Double.to_string() + let exp_plus = @json.parse("123.45e+10") + inspect(exp_plus, content="Number(1234500000000)") + inspect(exp_plus.stringify(), content="1234500000000") + let exp_minus = @json.parse("123.45e-10") + inspect(exp_minus, content="Number(1.2345e-8)") + inspect(exp_minus.stringify(), content="1.2345e-8") + let exp_upper_plus = @json.parse("123.45E+10") + inspect(exp_upper_plus.stringify(), content="1234500000000") + let exp_upper_minus = @json.parse("123.45E-10") + inspect(exp_upper_minus.stringify(), content="1.2345e-8") + + // Very large number + inspect( + try? @json.parse("1e999999999"), + content= + #|Ok(Number(Infinity, repr=Some("1e999999999"))) + , + ) } ///| -test "parse incomplete exponent sign" { - inspect(try? @json.parse("1e+"), content="Err(Unexpected end of file)") +test "parse and stringify large integers" { + // Test integers at Int boundaries + let min_int = "-2147483648" // Int.min_value + let parsed_min_str = @json.parse("\"" + min_int + "\"").stringify() + assert_eq(parsed_min_str, "\"" + min_int + "\"") + let max_int = "2147483647" // Int.max_value + let parsed_max_str = @json.parse("\"" + max_int + "\"").stringify() + assert_eq(parsed_max_str, "\"" + max_int + "\"") + + // Test integers beyond safe JavaScript integer precision (±2^53) + let beyond_js_safe = "9007199254740993" // 2^53 + 1 + let json_beyond = "\"" + beyond_js_safe + "\"" + let parsed_beyond_str = @json.parse(json_beyond).stringify() + assert_eq(parsed_beyond_str, json_beyond) + + // Test very large integers + let very_large = "12345678901234567890123456789" + let json_large = "\"" + very_large + "\"" + let parsed_large_str = @json.parse(json_large).stringify() + assert_eq(parsed_large_str, json_large) } diff --git a/json/types.mbt b/json/types.mbt index 4a8282574..744676177 100644 --- a/json/types.mbt +++ b/json/types.mbt @@ -61,9 +61,13 @@ pub impl Show for JsonValue with output(self, logger) { Null => logger.write_string("Null") True => logger.write_string("True") False => logger.write_string("False") - Number(n, repr=_) => { + Number(n, repr~) => { logger.write_string("Number(") Show::output(n, logger) + if repr is Some(_) { + logger.write_string(", repr=") + Show::output(repr, logger) + } logger.write_string(")") } String(s) => { From 7d06eb966a0d8de2b805b243149491d2f83e637c Mon Sep 17 00:00:00 2001 From: zihang Date: Fri, 4 Jul 2025 20:22:46 +0800 Subject: [PATCH 3/4] fix: adjust visibility and adjust equality --- builtin/builtin.mbti | 2 +- builtin/json.mbt | 21 +++++++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/builtin/builtin.mbti b/builtin/builtin.mbti index 2592be490..3cbf89bce 100644 --- a/builtin/builtin.mbti +++ b/builtin/builtin.mbti @@ -235,7 +235,7 @@ pub(all) enum IterResult { } impl Eq for IterResult -pub(all) enum Json { +pub enum Json { Null True False diff --git a/builtin/json.mbt b/builtin/json.mbt index a96b84667..b6d87944c 100644 --- a/builtin/json.mbt +++ b/builtin/json.mbt @@ -13,16 +13,29 @@ // limitations under the License. ///| -#visibility(change_to="readonly", "Use helper functions like `Json::object(...)` instead") -pub(all) enum Json { +pub enum Json { Null True False - Number(Double, repr~ : String?) + Number(Double, repr~ : String?) // 1.0000000000000000000e100 String(String) Array(Array[Json]) Object(Map[String, Json]) -} derive(Eq) +} + +///| +pub impl Eq for Json with op_equal(a, b) { + match (a, b) { + (Null, Null) => true + (True, True) => true + (False, False) => true + (Number(a_num, ..), Number(b_num, ..)) => a_num == b_num + (String(a_str), String(b_str)) => a_str == b_str + (Array(a_arr), Array(b_arr)) => a_arr == b_arr + (Object(a_obj), Object(b_obj)) => a_obj == b_obj + _ => false + } +} ///| /// Creates a JSON null value. From 5fab74ba3e20e9b33111194a051dbb0b06f0d42d Mon Sep 17 00:00:00 2001 From: zihang Date: Tue, 8 Jul 2025 10:20:06 +0800 Subject: [PATCH 4/4] chore: update tests and function calling --- json/lex_number_test.mbt | 44 +++++++++++++++++++++++++++++++--------- json/parse.mbt | 2 +- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/json/lex_number_test.mbt b/json/lex_number_test.mbt index 115968445..4f8f0c060 100644 --- a/json/lex_number_test.mbt +++ b/json/lex_number_test.mbt @@ -63,21 +63,45 @@ test "parse numbers" { test "parse and stringify large integers" { // Test integers at Int boundaries let min_int = "-2147483648" // Int.min_value - let parsed_min_str = @json.parse("\"" + min_int + "\"").stringify() - assert_eq(parsed_min_str, "\"" + min_int + "\"") + let parsed_min = @json.parse("\{min_int}") + inspect(parsed_min, content="Number(-2147483648)") + inspect(parsed_min.stringify(), content="-2147483648") let max_int = "2147483647" // Int.max_value - let parsed_max_str = @json.parse("\"" + max_int + "\"").stringify() - assert_eq(parsed_max_str, "\"" + max_int + "\"") + let parsed_max = @json.parse("\{max_int}") + inspect(parsed_max, content="Number(2147483647)") + inspect(parsed_max.stringify(), content="2147483647") // Test integers beyond safe JavaScript integer precision (±2^53) let beyond_js_safe = "9007199254740993" // 2^53 + 1 - let json_beyond = "\"" + beyond_js_safe + "\"" - let parsed_beyond_str = @json.parse(json_beyond).stringify() - assert_eq(parsed_beyond_str, json_beyond) + let parsed_beyond = @json.parse(beyond_js_safe) + inspect( + parsed_beyond, + content= + #|Number(Infinity, repr=Some("9007199254740993")) + , + ) + inspect(parsed_beyond.stringify(), content="9007199254740993") // Test very large integers let very_large = "12345678901234567890123456789" - let json_large = "\"" + very_large + "\"" - let parsed_large_str = @json.parse(json_large).stringify() - assert_eq(parsed_large_str, json_large) + let parsed_large = @json.parse(very_large) + inspect( + parsed_large, + content= + #|Number(Infinity, repr=Some("12345678901234567890123456789")) + , + ) + inspect(parsed_large.stringify(), content="12345678901234567890123456789") +} + +///| +test "parse and stringify large double" { + let very_large = "10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003.141592653589793238462643383279" + let parsed_large = @json.parse(very_large) + inspect( + parsed_large, + content= + #|Number(Infinity, repr=Some("10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003.141592653589793238462643383279")) + , + ) } diff --git a/json/parse.mbt b/json/parse.mbt index 8fa8d81c8..f71c47a77 100644 --- a/json/parse.mbt +++ b/json/parse.mbt @@ -49,7 +49,7 @@ fn ParseContext::parse_value2( Null => Json::null() True => Json::boolean(true) False => Json::boolean(false) - Number(n, repr) => Number(n, repr~) + Number(n, repr) => Json::number(n, repr?) String(s) => Json::string(s) LBrace => ctx.parse_object() LBracket => ctx.parse_array()