From afdc66c9ef4a6317bdd601800be025cca25692b8 Mon Sep 17 00:00:00 2001 From: "Michael G. Schwern" Date: Wed, 10 Aug 2016 11:44:25 -0700 Subject: [PATCH] Use PerlIO::utf8_strict as the default. Leon suggested using PerlIO::utf8_strict as the default. https://github.com/doherty/utf8-all/issues/42#issuecomment-238928373 It seems to work. I'm assuming there's still value in trying to use :encoding(UTF-8)? Maybe it's faster? --- Makefile.PL | 2 ++ lib/utf8/all.pm | 15 +++++++++++++-- t/lexical-again.t | 4 +++- t/open.t | 2 ++ t/utf8.t | 6 ++++-- 5 files changed, 24 insertions(+), 5 deletions(-) diff --git a/Makefile.PL b/Makefile.PL index 0905347..259d22f 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -20,6 +20,7 @@ my %WriteMakefileArgs = ( "Carp" => 0, "Encode" => 0, "Import::Into" => 0, + "PerlIO::utf8_strict" => 0, "Symbol" => 0, "charnames" => 0, "feature" => 0, @@ -60,6 +61,7 @@ my %FallbackPrereqs = ( "IPC::Open3" => 0, "Import::Into" => 0, "PerlIO" => 0, + "PerlIO::utf8_strict" => 0, "Symbol" => 0, "Test::Exception" => 0, "Test::Fatal" => 0, diff --git a/lib/utf8/all.pm b/lib/utf8/all.pm index c9f1cc5..3aa3244 100644 --- a/lib/utf8/all.pm +++ b/lib/utf8/all.pm @@ -167,7 +167,17 @@ sub import { my $utf8_IO_encoding = $class->_choose_utf8_IO_encoding; 'utf8'->import::into($target); - 'open'->import::into($target => $utf8_IO_encoding, ':std'); + 'open'->import::into($target => "IO" => $utf8_IO_encoding); + + # use open ':std' only works with some encodings. + state $have_encoded_std = 0; + if( !$have_encoded_std ) { + binmode STDERR, $utf8_IO_encoding; + binmode STDOUT, $utf8_IO_encoding; + binmode STDIN, $utf8_IO_encoding; + $have_encoded_std = 1; + } + 'charnames'->import::into($target, qw{:full :short}); 'warnings'->import::into($target, qw{FATAL utf8}); 'feature'->import::into($target, qw{unicode_strings}) if $^V >= v5.11.0; @@ -272,7 +282,8 @@ sub _choose_utf8_IO_encoding { return ':encoding(UTF-8)' if $^V >= v5.24.0 || (!$Config{usethreads} && !$Config{useithreads}); # A safe default. - return ':utf8'; + require PerlIO::utf8_strict; + return ':utf8_strict'; } =head1 INTERACTION WITH AUTODIE diff --git a/t/lexical-again.t b/t/lexical-again.t index 6f2877a..1c8082a 100644 --- a/t/lexical-again.t +++ b/t/lexical-again.t @@ -30,7 +30,7 @@ is "ใƒ†ใ‚นใƒˆ" => $expected_unicode, 'Literal string should be characters under my @layers = PerlIO::get_layers($handles{$fh}); ok(!grep(m/utf8/, @layers), "$fh: utf8 does not appear in the perlio layers") or diag explain { $fh => \@layers }; - ok(!grep(m/utf-8-strict/, @layers), "$fh: utf-8-strict does not appear in the perlio layers") + ok(!grep(m/utf-?8[-_]strict/, @layers), "$fh: utf-?8[-_]strict does not appear in the perlio layers") or diag explain { $fh => \@layers }; } @@ -52,6 +52,8 @@ for my $fh (keys %handles) { my @layers = PerlIO::get_layers($handles{$fh}); ok(grep(m/utf8/, @layers), "$fh: utf8 does appear in the perlio layers") or diag explain { $fh => \@layers }; + ok(grep(m/utf-?8[-_]strict/, @layers), "$fh: utf-?8[-_]strict does appear in the perlio layers") + or diag explain { $fh => \@layers }; } done_testing; diff --git a/t/open.t b/t/open.t index 227fa8d..662a680 100644 --- a/t/open.t +++ b/t/open.t @@ -8,6 +8,8 @@ ok open my $in, '<', 'corpus/testfile'; my @layers = PerlIO::get_layers($in); ok(grep(m/utf8/, @layers), 'utf8 appears in the perlio layers') or diag explain { $fh => \@layers }; +ok(grep(m/utf-?8[-_]strict/, @layers), 'utf-?8[-_]strict appears in the perlio layers') + or diag explain { $fh => \@layers }; my $contents = do { local $/; <$in>}; is $contents, "\x{30c6}\x{30b9}\x{30c8}\n", 'unicode retrieved OK'; diff --git a/t/utf8.t b/t/utf8.t index fba32b2..9b1930d 100644 --- a/t/utf8.t +++ b/t/utf8.t @@ -19,6 +19,8 @@ use Test::More; my @layers = PerlIO::get_layers($fh); ok(grep(m/utf8/, @layers), 'utf8 appears in the perlio layers') or diag explain { $fh => \@layers }; + ok(grep(m/utf-?8[-_]strict/, @layers), 'utf-?8[-_]strict appears in the perlio layers') + or diag explain { $fh => \@layers }; } } @@ -31,14 +33,14 @@ use Test::More; END { unlink "perlio_test2" } my @layers = PerlIO::get_layers($test_fh); - SKIP: { + SKIP: { # If we have the Perl Unicode flag set that adds the UTF-8 layer, # we need to skip this test. skip 'Perl Unicode flag set that always adds UTF-8 layer to output', 1 if (${^UNICODE} & 16); ok( !grep(/utf8/, @layers), q{utf8 doesn't appear in perlio layers}) or diag explain { $test_fh => \@layers }; } - ok( !grep(m/utf-8-strict/, @layers), q{utf-8-strict doesn't appear in the perlio layers}) + ok( !grep(m/utf-?8[-_]strict/, @layers), q{utf-?8[-_]strict doesn't appear in the perlio layers}) or diag explain { $test_fh => \@layers }; }