diff --git a/Changes b/Changes index abfb03e..7a79d83 100644 --- a/Changes +++ b/Changes @@ -2,6 +2,22 @@ Revision history for Perl extension Cpanel::JSON::XS TODO: http://stevehanov.ca/blog/index.php?id=104 compression +4.39 2024-12-12 (rurban) + - Fix Windows -Dusequadmath (sisyphus GH #235, GH #229) + - Fix inconsistent behavior between decoding escaped and unescaped + surrogates, and escaped non-characters vs non-escaped non-characters. + Now aligned to JSON::PP (Gavin Hayes GH #233, GH #227) + - Add type_all_string tests (Bernhard Schmalhofer GH #236) + - Silence UV to char cast warnings (bulk88 GH #232) + - Fix MSVC preprocessor errors (bulk88 GH #232) + - Fix -Wformat warnings on Windows (sisyphus GH #228) + - Clarify BigInt decoding (GH #226) + +4.38 2024-05-27 (rurban) + - Encode real core booleans as boolean notation (PR #224 leonerd) + - Minor test fixes by leonerd + - Fix docs typo (PR #225 karenetheridge) + 4.37 2023-07-04 (rurban) - Fix NAN/INF for AIX (Tux: AIX-5.3, tested by XSven on AIX-7.3) GH #165 - Fix empty string result in object stringification (PR #221 jixam) diff --git a/META.json b/META.json index dc6b168..6c0e964 100644 --- a/META.json +++ b/META.json @@ -100,7 +100,7 @@ "url" : "https://github.com/rurban/Cpanel-JSON-XS" } }, - "version" : "4.37", + "version" : "4.39", "x_contributors" : [ "Ashley Willis ", "Chip Salzenberg ", diff --git a/META.yml b/META.yml index 4d29d20..d2c8884 100644 --- a/META.yml +++ b/META.yml @@ -48,7 +48,7 @@ resources: bugtracker: https://github.com/rurban/Cpanel-JSON-XS/issues license: http://dev.perl.org/licenses/ repository: https://github.com/rurban/Cpanel-JSON-XS -version: '4.37' +version: '4.39' x_contributors: - 'Ashley Willis ' - 'Chip Salzenberg ' diff --git a/Makefile.PL b/Makefile.PL index 236d424..8462641 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -13,6 +13,10 @@ if ($Config{gccversion} and $Config{gccversion} =~ /^(\d+\.\d+)\./) { } elsif ($gccver >= 3.4) { $define = '-Wall -Wdeclaration-after-statement -Wextra -W'; } + + # -Wformat does not support the Windows specific "%I64u" format + # and will cause false warnings in compiling 'encode_sv'. + if($define && $^O =~/MSWin32/i) { $define .= ' -Wno-format' } } if ($] < 5.022 && $Config{d_setlocale} && $Config{usethreads}) { if (-e "/usr/include/xlocale.h") { diff --git a/README b/README index 7b1cacf..542433d 100644 --- a/README +++ b/README @@ -280,10 +280,11 @@ A FEW NOTES ON UNICODE AND PERL 5. A string containing "high" (> 255) character values is *not* a UTF-8 string. - 6. Unicode noncharacters only warn, as in core. + 6. Raw non-Unicode characters below U+10FFFF are allowed. The 66 Unicode noncharacters U+FDD0..U+FDEF, and U+*FFFE, U+*FFFF - just warn, see . - But illegal surrogate pairs fail to parse. + are allowed without warning, as JSON::PP does, see + . But illegal + surrogate pairs fail to parse. 7. Raw non-Unicode characters above U+10FFFF are disallowed. Raw non-Unicode characters outside the valid unicode range fail to @@ -292,6 +293,16 @@ A FEW NOTES ON UNICODE AND PERL Unicode RFC 7159 section 8.1. We use now the UTF8_DISALLOW_SUPER flag when parsing unicode. + 8. Lone surrogates or illegal surrogate pairs are disallowed. + Since RFC 3629, U+D800 through U+DFFF are not legal Unicode values + and their UTF-8 encodings must be treated as an invalid byte + sequence. RFC 8259 section 8.2 admits the spec allows string values + that contain bit sequences that cannot encode Unicode characters and + that the behavior of software that receives such values is + unpredictable. To avoid introducing non-Unicode strings into Perl we + use the UTF8_DISALLOW_SURROGATE flag when parsing Unicode and verify + escaped surrogates form valid pairs. + I hope this helps :) OBJECT-ORIENTED INTERFACE @@ -633,9 +644,10 @@ OBJECT-ORIENTED INTERFACE $json = $json->unblessed_bool([$enable]) If $enable is true (or missing), then "decode" will return Perl - non-object boolean variables (1 and 0) for JSON booleans ("true" and - "false"). If $enable is false, then "decode" will return - "JSON::PP::Boolean" objects for JSON booleans. + non-object boolean variables (1 and 0 as numbers or "1" and "" as + strings) for JSON booleans ("true" and "false"). If $enable is + false, then "decode" will return "JSON::PP::Boolean" objects for + JSON booleans. $json = $json->allow_singlequote ([$enable]) $enabled = $json->get_allow_singlequote @@ -672,6 +684,10 @@ OBJECT-ORIENTED INTERFACE integer Perl cannot handle as integer into a Math::BigInt object and convert a floating number (any) into a Math::BigFloat. + $int = $json->allow_nonref->allow_bignum->decode(1); # => 1 + $bigint = $json->allow_bignum->decode('100000000000000000000000000000000000000'); + $bigfloat = $json->allow_bignum->decode(1.0); + On the contrary, "encode" converts "Math::BigInt" objects and "Math::BigFloat" objects into JSON numbers with "allow_blessed" enable. @@ -1066,7 +1082,7 @@ INCREMENTAL PARSING parentheses. The only thing it guarantees is that it starts decoding as soon as a syntactically valid JSON text has been seen. This means you need to set resource limits (e.g. "max_size") to ensure the parser will - stop parsing in the presence if syntax errors. + stop parsing in the presence of syntax errors. The following methods implement this incremental parser. @@ -1797,17 +1813,21 @@ ENCODING/CODESET FLAG NOTES This works because "__proto__" is not valid outside of strings, so every occurrence of ""__proto__"\s*:" must be a string used as property name. - Unicode non-characters between U+FFFD and U+10FFFF are decoded either to - the recommended U+FFFD REPLACEMENT CHARACTER (see Unicode PR #121: - Recommended Practice for Replacement Characters), or in the binary or - relaxed mode left as is, keeping the illegal non-characters as before. - Raw non-Unicode characters outside the valid unicode range fail now to parse, because "A string is a sequence of zero or more Unicode characters" RFC 7159 section 1 and "JSON text SHALL be encoded in Unicode RFC 7159 section 8.1. We use now the UTF8_DISALLOW_SUPER flag when parsing unicode. + Since RFC 3629, U+D800 through U+DFFF are not legal Unicode values and + their UTF-8 encodings must be treated as an invalid byte sequence. RFC + 8259 section 8.2 admits the spec allows string values that contain bit + sequences that cannot encode Unicode characters and that the behavior of + software that receives such values is unpredictable. To avoid + introducing non-Unicode strings into Perl we use the + UTF8_DISALLOW_SURROGATE flag when parsing Unicode and verify escaped + surrogates form valid pairs. + If you know of other incompatibilities, please let me know. JSON and YAML @@ -2134,9 +2154,9 @@ LICENSE SEE ALSO The cpanel_json_xs command line utility for quick experiments. - JSON, JSON::XS, JSON::MaybeXS, Mojo::JSON, Mojo::JSON::MaybeXS, - JSON::SL, JSON::DWIW, JSON::YAJL, JSON::Any, Test::JSON, - Locale::Wolowitz, + JSON::PP, JSON, JSON::XS, JSON::MaybeXS, Mojo::JSON, + Mojo::JSON::MaybeXS, JSON::SL, JSON::DWIW, JSON::YAJL, JSON::Any, + Test::JSON, Locale::Wolowitz, diff --git a/SIGNATURE b/SIGNATURE index 01b6c04..2228edf 100644 --- a/SIGNATURE +++ b/SIGNATURE @@ -1,5 +1,5 @@ This file contains message digests of all files listed in MANIFEST, -signed via the Module::Signature module, version 0.88. +signed via the Module::Signature module, version 0.89. To verify the content in this distribution, first make sure you have Module::Signature installed, then type: @@ -16,25 +16,25 @@ Hash: RIPEMD160 SHA256 aac2b4bbaa7b93eaf72300f60e167a17e05adcd721087f735ba55d2900f31490 .appveyor.yml SHA256 082201a3cbd62a55f2e58ffbb991c4b2bb806de0009bc9497ffcc07202f60855 .github/FUNDING.yml -SHA256 136c5a00ee7c7425150f18013743e7b0d30339eca0f08901d4cf6b8731a6b017 .github/workflows/testsuite.yml +SHA256 724cc9e03083bc4d51ec2659118a9bb7ca66ca5f640a1d249ae3732d68511380 .github/workflows/testsuite.yml SHA256 a3c34aba52e269e6cec558ecf9cff393138574189fdff26b183bee9cc2e0434f .travis.yml SHA256 c3f2a1a4f66382f796f71a571946722edba53cf3238152b26fd325f4c2f1a20f .whitesource SHA256 8de3540a3cd7ecc9a9dcb48975fe852c082fe17d4462f87bb72aa7cc47f083ad COPYING -SHA256 a0c1df3ac89bec04046fe55c39cd20773a4d6f104e76508ef2c688dfc3db6dfa Changes +SHA256 ee990945735ae8b4b7b264dbd98ac361fa4b5ef59d2893343c53c2b09b3971e1 Changes SHA256 a5378ebe65273d49047a21e94af087f70a303793ffed2a695c800ed965ac185d MANIFEST -SHA256 ec3a3db51c59096cc14f7712986e3ab2e4027a40f3cc1faf9be69553c172e32c META.json -SHA256 c76fb8cf23082da8da4f9ffff46a25501af0c2769a6dbf9fbeeaf0f32d536b0a META.yml -SHA256 31d74c68c91639bc34e18541091616d226713c6c49168d42eefab58535f5cd4a Makefile.PL -SHA256 2eb1d8124526462c3afb4089a09a36d2e22da1b931a859f1ec06bec81d8a4245 README -SHA256 cf86fbae8a2abeadbd634c40b126f390077dd4a97bd54e2a8ec7311e9e9d71ae XS.pm -SHA256 21e00e79d18a7320df51dbc7a053052602d30659475a95bbbd366cb1d61a3f21 XS.xs +SHA256 445f18f73b799c4a86167412664e567fd0b6aa1d3d61fdf61dc74a1143e5dcd5 META.json +SHA256 be01f9e157d49e4cc37184bcc5e73cbc0befb3b441bc6b1ae647e838427c2e61 META.yml +SHA256 1ad5eab412179cae0dceb6fd9346aa20650988dd6eee23c3482a21d8714dd5bf Makefile.PL +SHA256 c62f5a06dffaa850fe7d55cad1c5ce3fbdb5504031b73b6043aa2974708c9293 README +SHA256 54c8486f5141eae12a12db7bb92d5883605edfd30de355a531ba267ebbdb4e5e XS.pm +SHA256 b71b7e0cadd6dfefb8f593d85de68d4d7b4d36b73e2e75344b79be377d98a61e XS.xs SHA256 c95e4b970183cbd6d1ec9c88c37a80f12bd2b66ed6be3c008ffd578d2f622c01 XS/Boolean.pm SHA256 20596259e7e399ed1984a469a9a907be878499406d5285a11f1ab98f93aff44f XS/Type.pm SHA256 2f34a530b7ce981b0df9aacd2b6944ccf74c4a7f8bb49fde57b342663e7feb26 bin/cpanel_json_xs SHA256 0d188abe82c2270e7bc5fc21de1d8210bfc52118a834b22592781bb2879a6065 eg/bench SHA256 3290077eba2e57ff1d2bf46c2a7d34a3b9c7f9b24fe517a3943430f5720da95f ppport.h SHA256 8bd5ef4d15ed3a9b2e641cc04549d6eed1532c86bba907e2b035d80c8dd5ac2c t/00_load.t -SHA256 07c0d02dcabd1d0996e6ab852df867fe7f91d551579832760262f586b56a1808 t/01_utf8.t +SHA256 0cefac61a4f61481fb66be51997b99b0e6da84f62b4976686d9ef87284ca5378 t/01_utf8.t SHA256 d2ec342bd9183411689ae9fee51b4e6730f6e5dce95f49cb59b4b38237337f1e t/02_error.t SHA256 e02e3fc388734af0fb8f7b8c8ebe4163ba4cfe5db94cc46b67776db0774ac716 t/03_types.t SHA256 36a9a87f4e143077195add1e8d931ff49d7e71f93f66326def22703158c268e1 t/04_dwiw_encode.t @@ -51,7 +51,7 @@ SHA256 16f34295a33f59b8fe7a4f70b701df03fc866d77eac300ca0503a98875675569 t/107_al SHA256 f2047975a3b8392feb6a87d782ecc7746ae2117bde57f716cc90877c8850f2e0 t/108_decode.t SHA256 e6f7738431bc8d77ad0b8ad2db9ab54426f7bbc86eb5f5794b1a4616f454baef t/109_encode.t SHA256 d6c467d647ab46c64734d6c6913ff262e854e00073a804da186caf894c5367d0 t/10_pc_keysort.t -SHA256 256e0dcaae188387bc0db211dd3b713576d73c01d95c5429417ed9c622640b4d t/110_bignum.t +SHA256 b83ef117bbbadaa4cdeccf0a0dec6087bd97094980638d982f7f805cf006f988 t/110_bignum.t SHA256 9c6d125de04ea14d24bbc96aafcbe8fbad75811ce4bf4e4d2d569eee8f195de0 t/112_upgrade.t SHA256 b207546715e27fe738f448fb0e1087b73bfd2a002c0254f656e6b4d47e154f58 t/113_overloaded_eq.t SHA256 89a6817b7b7ad584200b65c4b17e0c3960637162308d7bfd6d74756754235cc5 t/114_decode_prefix.t @@ -61,7 +61,7 @@ SHA256 cd69ce6908281737a22df45ce40e0cca3080312faa666ef68701c33fefc145c9 t/117_nu SHA256 34a7fd54a9c17af5bc643bf237d334b9f664c4af9fc699e8af2d6347696ef13e t/118_type.t SHA256 5f4f0f1d4221f5b5c28c1988f4d127462a42f36ae82fedd7319c8e0fbfbd57eb t/119_type_decode.t SHA256 8f0f898f0499424740eea5e2537e97faee949b1e3981a1588b1a61645d942d3c t/11_pc_expo.t -SHA256 67295534f9f44b6c2fd9375a4baff3b959318ce2468d466204efd1aeb8caadae t/120_type_all_string.t +SHA256 5b27401bd1e20eeabdff15e9453d67806bba74ccd6a9f4d1d934a25e8126e0f3 t/120_type_all_string.t SHA256 af3adbcc14e32df9fc2ef3f9a1502c1335a9e2da36ac54119be1f98fcabb4264 t/121_memleak.t SHA256 782bc33e7b6e46d42a168713b0828db134c7885f67fdd35ac53619ba6476aef6 t/125_shared_boolean.t SHA256 a1249dee56939f6577f385b1b5942f57009f1b5bd1d785616bc62802146f41ec t/12_blessed.t @@ -73,14 +73,14 @@ SHA256 398e5ff51603a52de901f4c1934265601e226d05b88ac604dcd9e9d179a0344f t/17_rel SHA256 1585a6aecec5c73b7a6f70982b3bcc1edc1d63ca55467223ab0d6f0956254bc4 t/18_json_checker.t SHA256 9f9006c1f925d9ace7d1621e1479c21f9b50459ab57aa0046209fed2b3e66530 t/19_incr.t SHA256 dde73ed3cfc0e28d064f61fc08871accf88b780aee06a3cb0040f59f04c1ff36 t/20_faihu.t -SHA256 fcac4365f8d9ba099ea90b6442eb205433d8419ffda5be82d66f5d69c3cdefe3 t/20_unknown.t +SHA256 56e11977ce3d544f8c8e62a38cbcc4f58f7f1d53b71918f803536acd62122713 t/20_unknown.t SHA256 388f8e0f0e41c9921aedc67313f8b89bdd08b95ced0dba242986d3b76d9a1688 t/21_evans.t SHA256 3da823eab55abb6dca05e8bc6111d3b59ea18c4ee270baf6413d9a45042ff48c t/22_comment_at_eof.t SHA256 2a6506fb07b27b1fef52b251d3876d23bd572596ff487d37c2f6597be554836c t/23_array_ctx.t SHA256 a8dfccba0b60b0fc91812fcfd96656e993abb74970509926d738c67a58641f01 t/24_freeze_recursion.t -SHA256 3d81e94b5d3407ba3df47ccace0aaf8f16bad9da3016e74f653e150629ce5b36 t/25_boolean.t +SHA256 d6e46428bf221ea9bca6f8c0a9d14ee76305d91e01d9946b570aac125d392ba8 t/25_boolean.t SHA256 e7297f97fe3fea65c865658675b72e667b37b201e7fec8b8128f2006f8999d86 t/26_duplicate.t -SHA256 03a2061b4742ea591961a4ce7403feac91998c0909dbde982c465ce3d2c39706 t/30_jsonspec.t +SHA256 814438975ce229d4ff0deb6a9aef967b7f088e03894b8b8e6ca1ccfb6d953117 t/30_jsonspec.t SHA256 cf2181a691d5e1862d71e4e902928a0d899b9671e3633584efa8ae3f5cc0d174 t/31_bom.t SHA256 59c743137453c8c4e9e785a15dcd057b0209d5ce160d683d7ab416dc37a92b6d t/52_object.t SHA256 3b9ce402e2d0cae8a525df4beca05f2656ba5cf02b074d02fd690fe97773d2d7 t/53_readonly.t @@ -88,7 +88,7 @@ SHA256 a08be137b59c9cd58410bc41969e1e9e9fa2159469523394b6bfd0c798c00908 t/54_str SHA256 f542b8cfd2bee5ab9ae92dd042e3bbf68a968ae959d0eeada43cd5cd53c8337a t/55_modifiable.t SHA256 7e825a17dc348ddee2b61e686a670115c31d80f372a7614e27811b9f3d795c79 t/96_interop.t SHA256 f720857c5fb03c62aab4765610ab05326e5139505f823400605abaebedffeb32 t/96_interop_pp.t -SHA256 ddf64cc8fddcf0d2b32c34b2969730a955869edf51b4157a3d3591334dd6b8de t/96_mojo.t +SHA256 40683e1922b62e46053ee60bf4c94e2ca9dbbe0da9d3131c16b4ef49045feb0a t/96_mojo.t SHA256 f847d17e014f19232281a3f44184da5f6dc0d1efb2d817d03156d1ff3d152afa t/97_unshare_hek.t SHA256 6b6c59c5f260f28afca3ccfe0785fb8da328ee3aa259079ef5dd982822862726 t/98_56only.t SHA256 fc880ee039642bcb1517cdb4afaa8060471785b521df45d295b50041137211e5 t/99_binary.t @@ -446,12 +446,15 @@ SHA256 5a4c8ec0943d40da2d65d3e4325ba55168f0f4a243dad391431df3d89d73e3d3 xt/pod-s SHA256 aca6f846869ab2e4881e807739086e1535b1438bd0e23d7a021360742736a6a9 xt/pod.t -----BEGIN PGP SIGNATURE----- -iQEzBAEBAwAdFiEEKJWogdNCcPq/6PdHtPYzOeZdZBQFAmSj9eoACgkQtPYzOeZd -ZBTP2wf8DlABhDoQ9mJCIILTVANIlJzE4HcTDwVczvLVSo9q7nBkuRC0HQoh8tfg -vl6zPgFLNEpjES2kYUQsjNxdrOJ06lB1n2Bm5G3BrSxCHg5oEQ5RPukDcTbWF1Mx -6uZvtUWCoP5lsUdJ5QqM4fhNDf9iXkY+14EZGMwPj2ia07gkbo7vf2v6vpSIvmze -dKRP7finLl+mPpBTXZtJ6REva/Ip5BFfxKdCpbVqOSmcCJ3bBkw1MT4UDalhmr9n -3bOliBpZa2Wx12jWk6Iewqw68p+jCjT6dNmA1Q3Ct+ii9ihYkVuCV6vGY1WCHpCl -C4xy/13TfjfQ5ZjfhVOeXF8OAHiIrA== -=bfLU +iQGzBAEBAwAdFiEEOKQWew22nknF9yFsuMKIZqsnp6IFAmdbUkEACgkQuMKIZqsn +p6J51Av/YTs0fXeOQoB8pgohlUf8bZAmOAguteMvxaI+rrkDjVbHLIR53fC/TXEF +OCPQqrqFO6aayuUlU3R1rK8LnGjtwBtdo47C548gxIsORITTM8+iQRjt5bnOfpV8 +pSHdY9x6HxYWdvEEWa3zOxqr6KOy4yZWtf2Jnki1VUMrsriD9rxJ9nGlA1quULXX +GMILyv1hkDdMC+wdkbY5YECJfPZo1xvRl8uIciFKNKb+Aysl1SJ/PTzP4lgoHTP4 +Hxu9B0QG72JXh2ZacpmqnlXfNAIJaA1WrmiQf+mPrriC8xyu9s/wsaLcoc1ndvqH +sWXiRyjfEw+0+B8VeIK/NEeJ9dnLRpWx0/IBvtnjQS3afnlflooaf8pB7D6SHBY4 +nS4vipjB3UUw+vA4wO3up9WHDuG7WlucO7k9nqSHcZSorCub7z2mB/TL5Dd88h/s +2X0wBnuqgVjY8UYvnbr4BeSVRdhMwSNsJbyQ3J5XVDAj86N7P9o0xYLYuTlVS1fK +rfVlKCoA +=Rrxl -----END PGP SIGNATURE----- diff --git a/XS.pm b/XS.pm index 8872858..457175e 100644 --- a/XS.pm +++ b/XS.pm @@ -1,5 +1,5 @@ package Cpanel::JSON::XS; -our $VERSION = '4.37'; +our $VERSION = '4.39'; our $XS_VERSION = $VERSION; # $VERSION = eval $VERSION; @@ -347,11 +347,12 @@ a Unicode string encoded in UTF-8, giving you a binary string. =item 5. A string containing "high" (> 255) character values is I a UTF-8 string. -=item 6. Unicode noncharacters only warn, as in core. +=item 6. Raw non-Unicode characters below U+10FFFF are allowed. -The 66 Unicode noncharacters U+FDD0..U+FDEF, and U+*FFFE, U+*FFFF just -warn, see L. But -illegal surrogate pairs fail to parse. +The 66 Unicode noncharacters U+FDD0..U+FDEF, and U+*FFFE, U+*FFFF are +allowed without warning, as JSON::PP does, see +L. But illegal +surrogate pairs fail to parse. =item 7. Raw non-Unicode characters above U+10FFFF are disallowed. @@ -361,6 +362,17 @@ characters" RFC 7159 section 1 and "JSON text SHALL be encoded in Unicode RFC 7159 section 8.1. We use now the UTF8_DISALLOW_SUPER flag when parsing unicode. +=item 8. Lone surrogates or illegal surrogate pairs are disallowed. + +Since RFC 3629, U+D800 through U+DFFF are not legal Unicode values and +their UTF-8 encodings must be treated as an invalid byte sequence. +RFC 8259 section 8.2 admits the spec allows string values that contain +bit sequences that cannot encode Unicode characters and that the +behavior of software that receives such values is unpredictable. To +avoid introducing non-Unicode strings into Perl we use the +UTF8_DISALLOW_SURROGATE flag when parsing Unicode and verify escaped +surrogates form valid pairs. + =back I hope this helps :) @@ -731,10 +743,11 @@ This setting has no effect when decoding JSON texts. $json = $json->unblessed_bool([$enable]) -If C<$enable> is true (or missing), then C will return -Perl non-object boolean variables (1 and 0) for JSON booleans -(C and C). If C<$enable> is false, then C -will return C objects for JSON booleans. +If C<$enable> is true (or missing), then C will return Perl +non-object boolean variables (1 and 0 as numbers or "1" and "" as +strings) for JSON booleans (C and C). If C<$enable> is +false, then C will return C objects for +JSON booleans. =item $json = $json->allow_singlequote ([$enable]) @@ -780,6 +793,10 @@ If C<$enable> is true (or missing), then C will convert the big integer Perl cannot handle as integer into a L object and convert a floating number (any) into a L. + $int = $json->allow_nonref->allow_bignum->decode(1); # => 1 + $bigint = $json->allow_bignum->decode('100000000000000000000000000000000000000'); + $bigfloat = $json->allow_bignum->decode(1.0); + On the contrary, C converts C objects and C objects into JSON numbers with C enable. @@ -1205,7 +1222,7 @@ as early as the full parser, for example, it doesn't detect mismatched parentheses. The only thing it guarantees is that it starts decoding as soon as a syntactically valid JSON text has been seen. This means you need to set resource limits (e.g. C) to ensure the -parser will stop parsing in the presence if syntax errors. +parser will stop parsing in the presence of syntax errors. The following methods implement this incremental parser. @@ -1985,17 +2002,21 @@ output for these property strings, e.g.: This works because C<__proto__> is not valid outside of strings, so every occurrence of C<"__proto__"\s*:> must be a string used as property name. -Unicode non-characters between U+FFFD and U+10FFFF are decoded either -to the recommended U+FFFD REPLACEMENT CHARACTER (see Unicode PR #121: -Recommended Practice for Replacement Characters), or in the binary or -relaxed mode left as is, keeping the illegal non-characters as before. - Raw non-Unicode characters outside the valid unicode range fail now to parse, because "A string is a sequence of zero or more Unicode characters" RFC 7159 section 1 and "JSON text SHALL be encoded in Unicode RFC 7159 section 8.1. We use now the UTF8_DISALLOW_SUPER flag when parsing unicode. +Since RFC 3629, U+D800 through U+DFFF are not legal Unicode values and +their UTF-8 encodings must be treated as an invalid byte sequence. +RFC 8259 section 8.2 admits the spec allows string values that contain +bit sequences that cannot encode Unicode characters and that the +behavior of software that receives such values is unpredictable. To +avoid introducing non-Unicode strings into Perl we use the +UTF8_DISALLOW_SURROGATE flag when parsing Unicode and verify escaped +surrogates form valid pairs. + If you know of other incompatibilities, please let me know. @@ -2414,9 +2435,9 @@ XSLoader::load 'Cpanel::JSON::XS', $XS_VERSION; The F command line utility for quick experiments. -L, L, L, L, L, -L, L, L, L, L, -L, +L, L, L, L, L, +L, L, L, L, +L, L, L, L L diff --git a/XS.xs b/XS.xs old mode 100644 new mode 100755 index f9d2d0a..9b1ce2b --- a/XS.xs +++ b/XS.xs @@ -242,6 +242,7 @@ mingw_modfl(long double x, long double *ip) #ifndef HvNAMEUTF8 # define HvNAMEUTF8(hv) 0 #endif +#if 0 /* since 5.14 check use warnings 'nonchar' */ #ifdef WARN_NONCHAR #define WARNER_NONCHAR(hi) \ @@ -259,6 +260,9 @@ mingw_modfl(long double x, long double *ip) Perl_warner(aTHX_ packWARN(WARN_UTF8), \ "Unicode non-character U+%04lX is illegal", (unsigned long)hi) #endif +#else +#define WARNER_NONCHAR(hi) +#endif /* since 5.16 */ #ifndef GV_NO_SVGMAGIC @@ -303,6 +307,10 @@ mingw_modfl(long double x, long double *ip) # endif #endif +#if (PERL_REVISION > 5) || (PERL_REVISION == 5 && PERL_VERSION >= 36) +# define PERL_HAVE_BOOLEANS +#endif + // i.e. "JSON" in big-endian #define JSON_MAGIC 0x4A534F4E @@ -543,17 +551,19 @@ decode_utf8 (pTHX_ unsigned char *s, STRLEN len, int relaxed, STRLEN *clen) return ((s[0] & 0x1f) << 6) | (s[1] & 0x3f); } else { -/* Since perl 5.14 we can disallow illegal unicode above U+10FFFF. +/* Since perl 5.14 we can disallow surrogates and illegal unicode above + U+10FFFF. Before we could only warn with warnings 'utf8'. - We accept only valid unicode, unless we are in the relaxed mode, - which allows SUPER, above U+10FFFF. + Surrogates are never allowed for consistency with unpaired escaped surrogate + handling. + SUPER, above U+10FFFF is not allowed, unless we are in the relaxed mode. */ #if PERL_VERSION > 36 UV c = utf8n_to_uvchr (s, len, clen, - UTF8_CHECK_ONLY | (relaxed ? 0 : UTF8_DISALLOW_SUPER)); + UTF8_CHECK_ONLY | UTF8_DISALLOW_SURROGATE | (relaxed ? 0 : UTF8_DISALLOW_SUPER)); #elif PERL_VERSION > 12 UV c = utf8n_to_uvuni (s, len, clen, - UTF8_CHECK_ONLY | (relaxed ? 0 : UTF8_DISALLOW_SUPER)); + UTF8_CHECK_ONLY | UTF8_DISALLOW_SURROGATE | (relaxed ? 0 : UTF8_DISALLOW_SUPER)); #elif PERL_VERSION >= 8 UV c = utf8n_to_uvuni (s, len, clen, UTF8_CHECK_ONLY); #endif @@ -618,21 +628,33 @@ decode_utf8 (pTHX_ unsigned char *s, STRLEN len, int relaxed, STRLEN *clen) INLINE unsigned char * encode_utf8 (unsigned char *s, UV ch) { + UV uv_ch; if (UNLIKELY(ch < 0x000080)) - *s++ = ch; - else if (LIKELY(ch < 0x000800)) - *s++ = 0xc0 | ( ch >> 6), - *s++ = 0x80 | ( ch & 0x3f); - else if (ch < 0x010000) - *s++ = 0xe0 | ( ch >> 12), - *s++ = 0x80 | ((ch >> 6) & 0x3f), - *s++ = 0x80 | ( ch & 0x3f); - else if (ch < 0x110000) - *s++ = 0xf0 | ( ch >> 18), - *s++ = 0x80 | ((ch >> 12) & 0x3f), - *s++ = 0x80 | ((ch >> 6) & 0x3f), - *s++ = 0x80 | ( ch & 0x3f); - + *s++ = (unsigned char) ch; + else if (LIKELY(ch < 0x000800)) { + uv_ch = 0xc0 | ( ch >> 6); + *s++ = (unsigned char) uv_ch; + uv_ch = 0x80 | ( ch & 0x3f); + *s++ = (unsigned char) uv_ch; + } + else if (ch < 0x010000) { + uv_ch = 0xe0 | ( ch >> 12); + *s++ = (unsigned char) uv_ch; + uv_ch = 0x80 | ((ch >> 6) & 0x3f); + *s++ = (unsigned char) uv_ch; + uv_ch = 0x80 | ( ch & 0x3f); + *s++ = (unsigned char) uv_ch; + } + else if (ch < 0x110000) { + uv_ch = 0xf0 | ( ch >> 18); + *s++ = (unsigned char) uv_ch; + uv_ch = 0x80 | ((ch >> 12) & 0x3f); + *s++ = (unsigned char) uv_ch; + uv_ch = 0x80 | ((ch >> 6) & 0x3f); + *s++ = (unsigned char) uv_ch; + uv_ch = 0x80 | ( ch & 0x3f); + *s++ = (unsigned char) uv_ch; + } return s; } @@ -1019,13 +1041,13 @@ encode_str (pTHX_ enc_t *enc, char *str, STRLEN len, int is_utf8) else if (enc->json.flags & F_LATIN1) { need (aTHX_ enc, 1); - *enc->cur++ = uch; + *enc->cur++ = (unsigned char)uch; str += clen; } else if (enc->json.flags & F_BINARY) { need (aTHX_ enc, 1); - *enc->cur++ = uch; + *enc->cur++ = (unsigned char)uch; str += clen; } else if (is_utf8) @@ -1445,7 +1467,7 @@ encode_hv (pTHX_ enc_t *enc, HV *hv, SV *typesv) retrieve_hk (aTHX_ he, &key, &klen); encode_hk (aTHX_ enc, key, klen); - if (UNLIKELY (PTR2ul (typehv))) + if (UNLIKELY (PTR2UV (typehv))) { SV **typesv_ref = hv_fetch (typehv, key, klen, 0); if (UNLIKELY (!typesv_ref)) @@ -1487,7 +1509,7 @@ encode_hv (pTHX_ enc_t *enc, HV *hv, SV *typesv) retrieve_hk (aTHX_ he, &key, &klen); encode_hk (aTHX_ enc, key, klen); - if (UNLIKELY (PTR2ul (typehv))) + if (UNLIKELY (PTR2UV (typehv))) { SV **typesv_ref = hv_fetch (typehv, key, klen, 0); if (UNLIKELY (!typesv_ref)) @@ -1626,7 +1648,8 @@ encode_stringify(pTHX_ enc_t *enc, SV *sv, int isref) SvREFCNT_dec(rv); } } - if (UNLIKELY(isref == 1 && (enc->json.flags & F_ALLOW_BIGNUM) && str && str[0] == '+')) { + if (UNLIKELY(isref == 1 + && (enc->json.flags & F_ALLOW_BIGNUM) && str && str[0] == '+')) { str++; len--; } @@ -1852,9 +1875,21 @@ encode_bool (pTHX_ enc_t *enc, SV *sv) if (!SvROK (sv)) { - if (UNLIKELY (sv == &PL_sv_yes)) + if ( +#ifdef PERL_HAVE_BOOLEANS + UNLIKELY (sv == &PL_sv_yes) || (SvIsBOOL(sv) && SvTRUE(sv)) +#else + UNLIKELY (sv == &PL_sv_yes) +#endif + ) encode_const_str (aTHX_ enc, "true", 4, 0); - else if (UNLIKELY (sv == &PL_sv_no)) + else if ( +#ifdef PERL_HAVE_BOOLEANS + UNLIKELY (sv == &PL_sv_no) || (SvIsBOOL(sv) && !SvTRUE(sv)) +#else + UNLIKELY (sv == &PL_sv_no) +#endif + ) encode_const_str (aTHX_ enc, "false", 5, 0); else if (!SvOK (sv)) encode_const_str (aTHX_ enc, "false", 5, 0); @@ -1979,7 +2014,13 @@ encode_sv (pTHX_ enc_t *enc, SV *sv, SV *typesv) } else { - if (UNLIKELY (sv == &PL_sv_yes || sv == &PL_sv_no)) type = JSON_TYPE_BOOL; + if ( +#ifdef PERL_HAVE_BOOLEANS + UNLIKELY (sv == &PL_sv_yes || sv == &PL_sv_no || SvIsBOOL(sv)) +#else + UNLIKELY (sv == &PL_sv_yes || sv == &PL_sv_no) +#endif + ) type = JSON_TYPE_BOOL; else if (SvNOKp (sv)) type = JSON_TYPE_FLOAT; else if (SvIOKp (sv)) type = JSON_TYPE_INT; else if (SvPOKp (sv)) type = JSON_TYPE_STRING; @@ -2136,13 +2177,19 @@ encode_sv (pTHX_ enc_t *enc, SV *sv, SV *typesv) if (force_conversion) { had_nokp = 0; -#if defined(USE_QUADMATH) && defined(HAVE_ISINFL) +#if defined(USE_QUADMATH) && defined(WIN32) /* Use Perl_isinf */ + if (UNLIKELY(Perl_isinf(nv))) + +#elif defined(USE_QUADMATH) && defined(HAVE_ISINFL) if (UNLIKELY(isinfl(nv))) #else if (UNLIKELY(isinf(nv))) #endif nv = (nv > 0) ? NV_MAX : -NV_MAX; -#if defined(USE_QUADMATH) && defined(HAVE_ISNANL) +#if defined(USE_QUADMATH) && defined(WIN32) /* Use Perl_isnan */ + if (UNLIKELY(Perl_isnan(nv))) + +#elif defined(USE_QUADMATH) && defined(HAVE_ISNANL) if (UNLIKELY(isnanl(nv))) #else if (UNLIKELY(isnan(nv))) @@ -2152,7 +2199,10 @@ encode_sv (pTHX_ enc_t *enc, SV *sv, SV *typesv) /* With no stringify_infnan we can skip the conversion, returning null. */ else if (enc->json.infnan_mode == 0) { -#if defined(USE_QUADMATH) && defined(HAVE_ISINFL) +#if defined(USE_QUADMATH) && defined(WIN32) /* Use Perl_isinf */ + if (UNLIKELY(Perl_isinf(nv))) + +#elif defined(USE_QUADMATH) && defined(HAVE_ISINFL) if (UNLIKELY(isinfl(nv))) #else if (UNLIKELY(isinf(nv))) @@ -2161,7 +2211,10 @@ encode_sv (pTHX_ enc_t *enc, SV *sv, SV *typesv) inf_or_nan = (nv > 0) ? 1 : 2; goto is_inf_or_nan; } -#if defined(USE_QUADMATH) && defined(HAVE_ISNANL) +#if defined(USE_QUADMATH) && defined(WIN32) /* Use Perl_isnan */ + if (UNLIKELY(Perl_isnan(nv))) + +#elif defined(USE_QUADMATH) && defined(HAVE_ISNANL) if (UNLIKELY(isnanl(nv))) #else if (UNLIKELY(isnan(nv))) @@ -2333,8 +2386,11 @@ encode_sv (pTHX_ enc_t *enc, SV *sv, SV *typesv) } else { NV intpart; - if (!( inf_or_nan || (had_nokp && Perl_modf(SvNVX(sv), &intpart)) || (!force_conversion && SvIOK(sv)) - || strchr(enc->cur,'e') || strchr(enc->cur,'E') || strchr(savecur,'.') + if (!( inf_or_nan || (had_nokp && Perl_modf(SvNVX(sv), &intpart)) + || (!force_conversion && SvIOK(sv)) + || strchr(enc->cur,'e') + || strchr(enc->cur,'E') + || strchr(savecur,'.') #if PERL_VERSION < 10 /* !!1 with 5.8 */ || (SvPOKp(sv) && strEQc(SvPVX(sv), "1") @@ -2358,7 +2414,8 @@ encode_sv (pTHX_ enc_t *enc, SV *sv, SV *typesv) IV iv = 0; int is_neg = 0; - if (UNLIKELY (SvROK (sv) && SvOBJECT (SvRV (sv))) && (enc->json.flags & F_ALLOW_BIGNUM)) + if (UNLIKELY (SvROK (sv) && SvOBJECT (SvRV (sv))) + && (enc->json.flags & F_ALLOW_BIGNUM)) { HV *stash = SvSTASH (SvRV (sv)); int is_bigint = (stash && stash == gv_stashpvs ("Math::BigInt", 0)); @@ -2487,7 +2544,8 @@ encode_sv (pTHX_ enc_t *enc, SV *sv, SV *typesv) } } - if ((numtype & (IS_NUMBER_GREATER_THAN_UV_MAX|IS_NUMBER_NOT_INT)) && (enc->json.flags & F_ALLOW_BIGNUM)) + if ((numtype & (IS_NUMBER_GREATER_THAN_UV_MAX|IS_NUMBER_NOT_INT)) + && (enc->json.flags & F_ALLOW_BIGNUM)) { STRLEN len; char *str; @@ -2554,11 +2612,7 @@ encode_sv (pTHX_ enc_t *enc, SV *sv, SV *typesv) { #if PERL_VERSION < 8 /* SvIV() and SvUV() in Perl 5.6 does not handle Inf and NaN in NV slot */ -# if defined(USE_QUADMATH) && defined(HAVE_ISINFL) && defined(HAVE_ISNANL) - if (SvNOKp (sv) && UNLIKELY (isinfl (SvNVX (sv)))) -# else if (SvNOKp (sv) && UNLIKELY (isinf (SvNVX (sv)))) -# endif { if (SvNVX (sv) < 0) { @@ -2572,11 +2626,7 @@ encode_sv (pTHX_ enc_t *enc, SV *sv, SV *typesv) iv = (IV)uv; } } -# if defined(USE_QUADMATH) && defined(HAVE_ISINFL) && defined(HAVE_ISNANL) - else if (!SvNOKp (sv) || LIKELY (!isnanl (SvNVX (sv)))) -# else else if (!SvNOKp (sv) || LIKELY (!isnan (SvNVX (sv)))) -# endif #endif sv_to_ivuv (aTHX_ sv, &is_neg, &iv, &uv); } @@ -3401,13 +3451,13 @@ _decode_str (pTHX_ dec_t *dec, char endstr) } case 'x': { - UV c; + unsigned char c; if (!(dec->json.flags & F_BINARY)) ERR ("illegal hex character in non-binary string"); ++dec_cur; dec->cur = dec_cur; - c = decode_2hex (dec); - if (c == (UV)-1) + c = (unsigned char)decode_2hex (dec); + if (c == (unsigned char)((UV)-1)) goto fail; *cur++ = c; dec_cur += 2; @@ -3416,12 +3466,12 @@ _decode_str (pTHX_ dec_t *dec, char endstr) case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': { - UV c; + char c; if (!(dec->json.flags & F_BINARY)) ERR ("illegal octal character in non-binary string"); dec->cur = dec_cur; - c = decode_3oct (dec); - if (c == (UV)-1) + c = (char)decode_3oct (dec); + if (c == (char)-1) goto fail; *cur++ = c; dec_cur += 3; @@ -3479,15 +3529,6 @@ _decode_str (pTHX_ dec_t *dec, char endstr) The WG's consensus was to leave the full range present in the ABNF and add the interoperability guidance about values outside the Unicode accepted range. - - http://seriot.ch/parsing_json.html#25 According to the Unicode - standard, illformed subsequences should be replaced by U+FFFD - REPLACEMENT CHARACTER. (See Unicode PR #121: Recommended Practice - for Replacement Characters). Several parsers use replacement - characters, while other keep the escaped form or produce an - non-Unicode character (see Section 5 - Parsing Contents). This - values are not for interchange, only for application internal use. - They are different from private use. Most parsers accept these. */ if (UNLIKELY( !(dec->json.flags & F_RELAXED) @@ -3502,7 +3543,7 @@ _decode_str (pTHX_ dec_t *dec, char endstr) cur = (char*)encode_utf8 ((U8*)cur, hi); } else - *cur++ = hi; + *cur++ = (unsigned char)hi; } break; @@ -4112,7 +4153,7 @@ decode_hv (pTHX_ dec_t *dec, SV *typesv) /* the next line creates a mortal sv each time it's called. */ /* might want to optimise this for common cases. */ - if (LIKELY((long)he)) + if (LIKELY((UV)he)) cb = hv_fetch_ent (dec->json.cb_sk_object, hv_iterkeysv (he), 0, 0); if (cb) diff --git a/debian/changelog b/debian/changelog index 3e83927..da75e93 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,25 @@ +libcpanel-json-xs-perl (4.39-2) unstable; urgency=medium + + * Team upload. + * Fix json_atof_scan1 overflows (CVE-2025-40929) + + -- Salvatore Bonaccorso Mon, 08 Sep 2025 21:26:54 +0200 + +libcpanel-json-xs-perl (4.39-1) unstable; urgency=medium + + * Team upload. + * Import upstream version 4.39. + + -- gregor herrmann Sat, 11 Jan 2025 02:09:43 +0100 + +libcpanel-json-xs-perl (4.38-1) unstable; urgency=medium + + * Team upload. + * Import upstream version 4.38. + * Declare compliance with Debian Policy 4.7.0. + + -- gregor herrmann Thu, 30 May 2024 18:47:08 +0200 + libcpanel-json-xs-perl (4.37-1) unstable; urgency=medium * Team upload. diff --git a/debian/control b/debian/control index 6a759f4..be19359 100644 --- a/debian/control +++ b/debian/control @@ -13,7 +13,7 @@ Build-Depends: debhelper-compat (= 13), libtie-ixhash-perl , perl-xs-dev, perl:native -Standards-Version: 4.6.2 +Standards-Version: 4.7.0 Vcs-Browser: https://salsa.debian.org/perl-team/modules/packages/libcpanel-json-xs-perl Vcs-Git: https://salsa.debian.org/perl-team/modules/packages/libcpanel-json-xs-perl.git Homepage: https://metacpan.org/release/Cpanel-JSON-XS @@ -21,10 +21,10 @@ Rules-Requires-Root: no Package: libcpanel-json-xs-perl Architecture: any +Multi-Arch: same Depends: ${misc:Depends}, ${perl:Depends}, ${shlibs:Depends} -Multi-Arch: same Description: module for fast and correct serialising to JSON Cpanel::JSON::XS converts Perl data structures to JSON and vice versa. Its primary goal is to be correct and its secondary goal is to be fast. To reach diff --git a/debian/patches/fix-json_atof_scan1-overflows.patch b/debian/patches/fix-json_atof_scan1-overflows.patch new file mode 100644 index 0000000..b0ec121 --- /dev/null +++ b/debian/patches/fix-json_atof_scan1-overflows.patch @@ -0,0 +1,48 @@ +From: Marc Lehmann +Date: Sat, 6 Sep 2025 11:31:36 +0200 +Subject: fix json_atof_scan1 overflows +Origin: https://github.com/rurban/Cpanel-JSON-XS/commit/378236219eaa35742c3962ecbdee364903b0a1f2 +Bug-Debian-Security: https://security-tracker.debian.org/tracker/CVE-2025-40929 + +with fuzzed overlong numbers. CVE-2025-40928 +Really the comparisons were wrong. +--- + XS.xs | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +diff --git a/XS.xs b/XS.xs +index 9b1ce2bd5f28..2b9900f62285 100755 +--- a/XS.xs ++++ b/XS.xs +@@ -710,16 +710,16 @@ json_atof_scan1 (const char *s, NV *accum, int *expo, int postdp, int maxdepth) + /* if we recurse too deep, skip all remaining digits */ + /* to avoid a stack overflow attack */ + if (UNLIKELY(--maxdepth <= 0)) +- while (((U8)*s - '0') < 10) ++ while (*s >= '0' && *s <= '9') + ++s; + + for (;;) + { +- U8 dig = (U8)*s - '0'; ++ U8 dig = (U8)(*s - '0'); + + if (UNLIKELY(dig >= 10)) + { +- if (dig == (U8)((U8)'.' - (U8)'0')) ++ if (dig == (U8)('.' - '0')) + { + ++s; + json_atof_scan1 (s, accum, expo, 1, maxdepth); +@@ -739,7 +739,7 @@ json_atof_scan1 (const char *s, NV *accum, int *expo, int postdp, int maxdepth) + else if (*s == '+') + ++s; + +- while ((dig = (U8)*s - '0') < 10) ++ while (*s >= '0' && *s <= '9') + exp2 = exp2 * 10 + *s++ - '0'; + + *expo += neg ? -exp2 : exp2; +-- +2.51.0 + diff --git a/debian/patches/series b/debian/patches/series new file mode 100644 index 0000000..9644795 --- /dev/null +++ b/debian/patches/series @@ -0,0 +1 @@ +fix-json_atof_scan1-overflows.patch diff --git a/t/01_utf8.t b/t/01_utf8.t index 49f7418..9b5d515 100644 --- a/t/01_utf8.t +++ b/t/01_utf8.t @@ -1,4 +1,4 @@ -use Test::More tests => 162; +use Test::More tests => 152; use utf8; use Cpanel::JSON::XS; use warnings; @@ -43,56 +43,6 @@ SKIP: { # TODO: test utf8 hash keys, # test utf8 strings without any char > 0x80. -# warn on the 66 non-characters as in core -{ - BEGIN { 'warnings'->import($] < 5.014 ? 'utf8' : 'nonchar') } - my $w = ''; - $SIG{__WARN__} = sub { $w = shift }; - my $d = Cpanel::JSON::XS->new->allow_nonref->decode('"\ufdd0"'); - my $warn = $w; - { - no warnings 'utf8'; - is ($d, "\x{fdd0}", substr($warn,0,31)."..."); - } - like ($warn, qr/^Unicode non-character U\+FDD0 is/); - $w = ''; - # higher planes - $d = Cpanel::JSON::XS->new->allow_nonref->decode('"\ud83f\udfff"'); - $warn = $w; - { - no warnings 'utf8'; - is ($d, "\x{1ffff}", substr($warn,0,31)."..."); - } - like ($w, qr/^Unicode non-character U\+1FFFF is/); - $w = ''; - $d = Cpanel::JSON::XS->new->allow_nonref->decode('"\ud87f\udffe"'); - $warn = $w; - { - no warnings 'utf8'; - is ($d, "\x{2fffe}", substr($warn,0,31)."..."); - } - like ($w, qr/^Unicode non-character U\+2FFFE is/); - - $w = ''; - $d = Cpanel::JSON::XS->new->allow_nonref->decode('"\ud8a4\uddd1"'); - $warn = $w; - is ($d, "\x{391d1}", substr($warn,0,31)."..."); - is ($w, ''); -} -{ - my $w; - BEGIN { 'warnings'->import($] < 5.014 ? 'utf8' : 'nonchar') } - $SIG{__WARN__} = sub { $w = shift }; - # no warning with relaxed - my $d = Cpanel::JSON::XS->new->allow_nonref->relaxed->decode('"\ufdd0"'); - my $warn = $w; - { - no warnings 'utf8'; - is ($d, "\x{fdd0}", "no warning with relaxed"); - } - is($w, undef); -} - # security exploits via ill-formed subsequences # see http://unicode.org/reports/tr36/#UTF-8_Exploit # testcases from Encode/t/utf8strict.t diff --git a/t/110_bignum.t b/t/110_bignum.t index 4d8f604..5532c23 100644 --- a/t/110_bignum.t +++ b/t/110_bignum.t @@ -1,4 +1,4 @@ - +#!/usr/bin/env perl use strict; my $has_bignum; BEGIN { @@ -6,15 +6,15 @@ BEGIN { $has_bignum = $@ ? 0 : 1; } use Test::More $has_bignum - ? (tests => 17) + ? (tests => 20) : (skip_all => "Can't load Math::BigInt"); use Cpanel::JSON::XS; +use Scalar::Util (); use Devel::Peek; my $json = new Cpanel::JSON::XS; - -$json->allow_nonref->allow_bignum; -$json->convert_blessed->allow_blessed; +$json->allow_bignum; # is implicitly allow_nonref and convert_blessed + # $json->convert_blessed->allow_blessed; my $num = $json->decode(q|100000000000000000000000000000000000000|); @@ -71,3 +71,16 @@ is( "$inf", 'null', '-inf default' ); $exp = "$biginf" =~ /nan/i ? "nan" : "-inf"; $inf = $json->stringify_infnan(3)->encode($biginf); is( "$inf", $exp, '-inf stringify' ); + +# see if allow_bignum is enough, always decodes to a BigFloat +my $num = $json->decode(4.5); +isa_ok( $num, 'Math::BigFloat' ); +is( + $num->bcmp('4.5'), + 0, + 'decode simple bigfloat' +) or Dump($num); + +# But a short int will not decode to a BigInt +$num = $json->decode(q|[4]|)->[0]; +ok( Scalar::Util::looks_like_number($num), 'simple IV') or Dump($num); diff --git a/t/120_type_all_string.t b/t/120_type_all_string.t index 582dc34..940dee8 100644 --- a/t/120_type_all_string.t +++ b/t/120_type_all_string.t @@ -3,7 +3,7 @@ use warnings; use Cpanel::JSON::XS; -use Test::More tests => 5; +use Test::More tests => 8; my $sjson = Cpanel::JSON::XS->new->canonical->require_types->type_all_string->allow_nonref; @@ -12,3 +12,6 @@ is($sjson->encode("0"), '"0"'); is($sjson->encode(0.5), '"0.5"'); is($sjson->encode("0.5"), '"0.5"'); is($sjson->encode([ 1, "2", { key1 => 3.5 }, [ "string", -10 ] ]), '["1","2",{"key1":"3.5"},["string","-10"]]'); +is($sjson->encode([ Cpanel::JSON::XS::false, Cpanel::JSON::XS::true ]), '["false","true"]'); +is($sjson->encode([ 1 < 0, 1 > 0 ]), '["","1"]'); +is($sjson->encode(undef), 'null'); diff --git a/t/20_unknown.t b/t/20_unknown.t index b3240fb..cfd7b48 100755 --- a/t/20_unknown.t +++ b/t/20_unknown.t @@ -7,6 +7,7 @@ BEGIN { or plan skip_all => 'JSON::PP 2.09 required for cross testing'; $ENV{PERL_JSON_BACKEND} = 'JSON::PP'; } +use constant HAVE_BOOLEANS => ($^V ge v5.36); plan tests => 32; use JSON::PP (); use Cpanel::JSON::XS (); @@ -64,9 +65,16 @@ is( $pp->encode( {false => \!!""} ), '{"false":null}', 'pp \sv_no' ); is( $json->encode( {null => \"some"} ), '{"null":null}', 'js unknown' ); is( $json->encode( {null => \""} ), '{"null":null}', 'js unknown' ); -is( $json->encode( {true => !!1} ), '{"true":1}', 'js sv_yes' ); -is( $json->encode( {false => !!0} ), '{"false":""}', 'js sv_no' ); -is( $json->encode( {false => !!""} ), '{"false":""}', 'js sv_no' ); +if(HAVE_BOOLEANS) { + is( $json->encode( {true => !!1} ), '{"true":true}', 'js sv_yes' ); + is( $json->encode( {false => !!0} ), '{"false":false}', 'js sv_no' ); + is( $json->encode( {false => !!""} ), '{"false":false}', 'js sv_no' ); +} +else { + is( $json->encode( {true => !!1} ), '{"true":1}', 'js sv_yes' ); + is( $json->encode( {false => !!0} ), '{"false":""}', 'js sv_no' ); + is( $json->encode( {false => !!""} ), '{"false":""}', 'js sv_no' ); +} is( $json->encode( {true => \!!1} ), '{"true":true}', 'js \sv_yes' ); is( $json->encode( {false => \!!0} ), '{"false":null}', 'js \sv_no' ); is( $json->encode( {false => \!!""} ), '{"false":null}', 'js \sv_no' ); diff --git a/t/25_boolean.t b/t/25_boolean.t index 89f7f54..4b67c2b 100644 --- a/t/25_boolean.t +++ b/t/25_boolean.t @@ -1,5 +1,6 @@ use strict; -use Test::More tests => 42; +use constant HAVE_BOOLEANS => ($^V ge v5.36); +use Test::More tests => 42 + (HAVE_BOOLEANS ? 2 : 0); use Cpanel::JSON::XS (); use Config; @@ -125,3 +126,11 @@ is($cjson->encode(do { my $struct = $unblessed_bool_cjson->decode($truefalse, my $js = $unblessed_bool_cjson->decode($truefalse); ok eval { $js->[0] = "new value 0" }, "decoded 'true' is modifiable" or diag($@); ok eval { $js->[1] = "new value 1" }, "decoded 'false' is modifiable" or diag($@); + +if(HAVE_BOOLEANS) { + no if HAVE_BOOLEANS, warnings => "experimental::builtin"; + is($cjson->encode({t => builtin::true}), q({"t":true}), + 'true core booleans encode as boolean'); + is($cjson->encode({f => builtin::false}), q({"f":false}), + 'false core booleans encode as boolean'); +} diff --git a/t/30_jsonspec.t b/t/30_jsonspec.t index 1557b62..5786c0b 100644 --- a/t/30_jsonspec.t +++ b/t/30_jsonspec.t @@ -1,6 +1,6 @@ # regressions and differences from the JSON Specs and JSON::PP # detected by http://seriot.ch/json/parsing.html -use Test::More ($] >= 5.008) ? (tests => 686) : (skip_all => "needs 5.8"); +use Test::More ($] >= 5.008) ? (tests => 678) : (skip_all => "needs 5.8"); use Cpanel::JSON::XS; BEGIN { require Encode if $] >= 5.008 && $] < 5.020; # Currently required for <5.20 @@ -9,14 +9,14 @@ my $json = Cpanel::JSON::XS->new->utf8->allow_nonref; my $relaxed = Cpanel::JSON::XS->new->utf8->allow_nonref->relaxed; # fixme: -# n_string_UTF8_surrogate_U+D800 ["EDA080"] <=> [""] unicode # done: # i_string_unicode_*_nonchar ["\uDBFF\uDFFE"] (add warning as in core) # i_string_not_in_unicode_range Code point 0x13FFFF is not Unicode UTF8_DISALLOW_SUPER # y_string_utf16, y_string_utf16be, y_string_utf32, y_string_utf32be fixed with 3.0222 +# n_string_UTF8_surrogate_U+D800 Code point 0xD800 is not Unicode UTF8_DISALLOW_SURROGATE my %todo; +$todo{'n_string_UTF8_surrogate_U+D800'}++ if $] < 5.014; $todo{'y_string_nonCharacterInUTF-8_U+FFFF'}++ if $] < 5.013; -$todo{'n_string_UTF8_surrogate_U+D800'}++ if $] >= 5.012; if ($] < 5.008) { # 5.6 has no multibyte support $todo{$_}++ for qw( @@ -107,32 +107,16 @@ sub i_undefined { sub i_pass { my ($str, $name) = @_; $@ = ''; - my $w; - if ($name =~ /nonchar/) { # check the warning - require warnings; - warnings->import($] < 5.014 ? 'utf8' : 'nonchar'); - $SIG{__WARN__} = sub { $w = shift }; - } my $result = $todo{$name} ? eval { $json->decode($str) } : $json->decode($str); - my $warn = $w; TODO: { local $TODO = "$name" if exists $todo{$name}; is($@, '', "no parsing error with undefined $name ".substr($@,0,40)); isnt($result, undef, "valid result with undefined $name"); - if ($name =~ /nonchar/) { - like ($warn, qr/^Unicode non-character U\+[10DFE]+ is/); - $w = ''; - } $@ = ''; #diag "$name $str"; $result = eval { $relaxed->decode($str) }; - $warn = $w; is($@, '', "no parsing error with undefined $name relaxed ".substr($@,0,40)); isnt($result, undef, "valid result with undefined $name relaxed"); - if ($name =~ /nonchar/) { - is($warn, ''); - $w = ''; - } } } # result undefined, parsing failed diff --git a/t/96_mojo.t b/t/96_mojo.t index 86c7bfe..daf31c1 100644 --- a/t/96_mojo.t +++ b/t/96_mojo.t @@ -9,7 +9,7 @@ BEGIN { plan skip_all => "Mojo::JSON::decode_json required for testing interop"; exit 0; } - plan tests => 9; + plan tests => 12; } use Mojo::JSON (); @@ -35,18 +35,10 @@ ok( !$js->{is_false}, 'ok !false'); my $mj = Mojo::JSON::encode_json( $yesno ); $js = $cjson->decode( $mj ); -# fragile -ok( $js->[0] eq '' or $js->[0] == 0 or !$js->[0], 'can decode Mojo false' ); -is( $js->[1], 1, 'can decode Mojo true' ); -# Note this is fragile. it depends on the internal representation of booleans. -# It can also be ['0', '1'] -if ($js->[0] eq '') { - is_deeply( $js, ['', 1], 'can decode Mojo booleans' ) - or diag( $mj, $js ); -} else { - TODO: { - local $TODO = 'fragile false => "0"'; - is_deeply( $js, ['', 1], 'can decode Mojo booleans' ) - or diag( $mj, $js ); - } -} +ok( !$js->[0], 'decoded Mojo false is false' ); +ok( $js->[0] == 0, 'decoded Mojo false is zero' ); +ok( $js->[0] eq "", 'decoded Mojo false is empty string' ); + +ok( $js->[1], 'decoded Mojo true is true' ); +ok( $js->[1] == 1, 'decoded Mojo true is one' ); +ok( $js->[1] eq "1", 'decoded Mojo true is "1" string' );