Updated patch for github issue #88 "Latin 1 text could end up as malf…

…ormed UTF-8 on output" (nc)

Now using U8 casts
-            *(d++) = UTF8_EIGHT_BIT_HI(c);
-            *(d++) = UTF8_EIGHT_BIT_LO(c);
+            *(d++) = UTF8_EIGHT_BIT_HI((U8) c);
+            *(d++) = UTF8_EIGHT_BIT_LO((U8) c);

Minor POD improvement

MANIFEST

@@ -311,6 +311,7 @@ t/900_bugs/042_perl59_issue.t
 t/900_bugs/043_issue107.t
 t/900_bugs/044_empty_result.t
 t/900_bugs/045_issue130.t
+t/900_bugs/046_issue88.t
 t/900_bugs/issue79/tmpl/contentA.tt
 t/900_bugs/issue79/tmpl/contentB.tt
 t/900_bugs/issue79/tmpl/wrapperA.tt

lib/Text/Xslate/PP/State.pm

@@ -177,7 +177,7 @@ sub print {
         if(defined ${$sv}) {
             $st->{output} .=
                 (utf8::is_utf8($st->{output}) && !utf8::is_utf8(${$sv}))
-                 ? $st->encoding->decode(${$sv})
+                 ? eval {$st->encoding->decode(${$sv}, Encode::FB_CROAK())} || ${$sv}
                  : ${$sv};
         }
         else {
@@ -188,7 +188,7 @@ sub print {
         $sv =~ s/($Text::Xslate::PP::html_metachars)/$Text::Xslate::PP::html_escape{$1}/xmsgeo;
         $st->{output} .=
             (utf8::is_utf8($st->{output}) && !utf8::is_utf8($sv))
-             ? $st->encoding->decode($sv)
+             ? eval {$st->encoding->decode($sv, Encode::FB_CROAK())} || $sv
              : $sv;
     }
     else {

script/xslate

@@ -42,7 +42,7 @@ xslate - Process Xslate Templates
 
 =head1 DESCRIPTION
 
-The xslate script is used to process entire directory trees containing
+The C<xslate> script is used to process entire directory trees containing
 template files, or to process one liners.
 
 =head1 ARGUMENTS
@@ -51,7 +51,7 @@ template files, or to process one liners.
 
 Specifies the target to be processed by Xslate.
 
-If the target is a file, the file is processed, and xslate will exit immediately. If the target is a directory, then the directory is traversed and each file found is processed via xslate.
+If the target is a file, the file is processed, and C<xslate> will exit immediately. If the target is a directory, then the directory is traversed and each file found is processed via C<xslate>.
 
 =head1 AUTHOR
 

src/Text-Xslate.xs

@@ -541,19 +541,39 @@ tx_unmark_raw(pTHX_ SV* const str) {
 /* does sv_catsv_nomg(dest, src), but significantly faster */
 STATIC_INLINE void
 tx_sv_cat(pTHX_ SV* const dest, SV* const src) {
+    STRLEN len;
+    const char* pv = SvPV_const(src, len);
+
     if(!SvUTF8(dest) && SvUTF8(src)) {
         sv_utf8_upgrade(dest);
     }
 
-    {
-        STRLEN len;
-        const char* const pv  = SvPV_const(src, len);
+    if(SvUTF8(dest) == SvUTF8(src)
+       || is_utf8_string((const U8 *)pv, len)) {
         STRLEN const dest_cur = SvCUR(dest);
         char* const d         = SvGROW(dest, dest_cur + len + 1 /* count '\0' */);
 
         SvCUR_set(dest, dest_cur + len);
         Copy(pv, d + dest_cur, len + 1 /* copy '\0' */, char);
     }
+    else {
+        STRLEN const dest_cur = SvCUR(dest);
+        /* Longest UTF-8 representation of each char is 2 octets. */
+        char* const d_start   = SvGROW(dest, dest_cur + 2 * len + 1 /* count '\0' */);
+        char* d = d_start + dest_cur;
+
+        while(len--) {
+            const U8 c = *pv++;
+            if (UTF8_IS_INVARIANT(c)) {
+                *(d++) = c;
+            } else {
+                *(d++) = UTF8_EIGHT_BIT_HI(c);
+                *(d++) = UTF8_EIGHT_BIT_LO(c);
+            }
+        }
+        *d = '\0';
+        SvCUR_set(dest, d - d_start);
+    }
 }
 
 static void /* doesn't care about raw-ness */
@@ -563,6 +583,8 @@ tx_sv_cat_with_html_escape_force(pTHX_ SV* const dest, SV* const src) {
     const char* const end = cur + len;
     STRLEN const dest_cur = SvCUR(dest);
     char* d;
+    const U32 upgrade_on_copy = SvUTF8(dest) && !SvUTF8(src)
+        && !is_utf8_string((const U8 *)cur, len);
 
     (void)SvGROW(dest, dest_cur + ( len * ( sizeof(""") - 1) ) + 1);
     if(!SvUTF8(dest) && SvUTF8(src)) {
@@ -595,6 +617,10 @@ tx_sv_cat_with_html_escape_force(pTHX_ SV* const dest, SV* const src) {
             // CopyToken("'", d);
             CopyToken("'", d);
         }
+        else if (upgrade_on_copy && !UTF8_IS_INVARIANT(c)) {
+            *(d++) = UTF8_EIGHT_BIT_HI((U8) c);
+            *(d++) = UTF8_EIGHT_BIT_LO((U8) c);
+        }
         else {
             *(d++) = c;
         }

t/900_bugs/046_issue88.t

@@ -0,0 +1,23 @@
+#!perl
+# https://github.com/xslate/p5-Text-Xslate/issues/88
+use strict;
+use warnings;
+use Test::More;
+
+use utf8;
+use Text::Xslate 'mark_raw';
+my $xslate = Text::Xslate->new();
+
+is $xslate->render_string('<: $string :>', {string => "Ä"})      => 'Ä';
+is $xslate->render_string('<: $string :>', {string => "\x{c4}"}) => 'Ä';
+
+is $xslate->render_string('あ<: $string :>', {string => "Ä"})      => 'あÄ';
+is $xslate->render_string('あ<: $string :>', {string => "\x{c4}"}) => 'あÄ';
+
+is $xslate->render_string('<: $string :>', {string => mark_raw("Ä")})      => 'Ä';
+is $xslate->render_string('<: $string :>', {string => mark_raw("\x{c4}")}) => 'Ä';
+
+is $xslate->render_string('あ<: $string :>', {string => mark_raw("Ä")})      => 'あÄ';
+is $xslate->render_string('あ<: $string :>', {string => mark_raw("\x{c4}")}) => 'あÄ';
+
+done_testing();