Skip to content

Commit

Permalink
Use PerlIO::utf8_strict as the default.
Browse files Browse the repository at this point in the history
Leon suggested using PerlIO::utf8_strict as the default.
doherty#42 (comment)

It seems to work.

I'm assuming there's still value in trying to use :encoding(UTF-8)?
Maybe it's faster?
  • Loading branch information
schwern committed Aug 10, 2016
1 parent 145c46a commit afdc66c
Show file tree
Hide file tree
Showing 5 changed files with 24 additions and 5 deletions.
2 changes: 2 additions & 0 deletions Makefile.PL
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ my %WriteMakefileArgs = (
"Carp" => 0,
"Encode" => 0,
"Import::Into" => 0,
"PerlIO::utf8_strict" => 0,
"Symbol" => 0,
"charnames" => 0,
"feature" => 0,
Expand Down Expand Up @@ -60,6 +61,7 @@ my %FallbackPrereqs = (
"IPC::Open3" => 0,
"Import::Into" => 0,
"PerlIO" => 0,
"PerlIO::utf8_strict" => 0,
"Symbol" => 0,
"Test::Exception" => 0,
"Test::Fatal" => 0,
Expand Down
15 changes: 13 additions & 2 deletions lib/utf8/all.pm
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,17 @@ sub import {
my $utf8_IO_encoding = $class->_choose_utf8_IO_encoding;

'utf8'->import::into($target);
'open'->import::into($target => $utf8_IO_encoding, ':std');
'open'->import::into($target => "IO" => $utf8_IO_encoding);

# use open ':std' only works with some encodings.
state $have_encoded_std = 0;
if( !$have_encoded_std ) {
binmode STDERR, $utf8_IO_encoding;
binmode STDOUT, $utf8_IO_encoding;
binmode STDIN, $utf8_IO_encoding;
$have_encoded_std = 1;
}

'charnames'->import::into($target, qw{:full :short});
'warnings'->import::into($target, qw{FATAL utf8});
'feature'->import::into($target, qw{unicode_strings}) if $^V >= v5.11.0;
Expand Down Expand Up @@ -272,7 +282,8 @@ sub _choose_utf8_IO_encoding {
return ':encoding(UTF-8)' if $^V >= v5.24.0 || (!$Config{usethreads} && !$Config{useithreads});

# A safe default.
return ':utf8';
require PerlIO::utf8_strict;
return ':utf8_strict';
}

=head1 INTERACTION WITH AUTODIE
Expand Down
4 changes: 3 additions & 1 deletion t/lexical-again.t
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ is "テスト" => $expected_unicode, 'Literal string should be characters under
my @layers = PerlIO::get_layers($handles{$fh});
ok(!grep(m/utf8/, @layers), "$fh: utf8 does not appear in the perlio layers")
or diag explain { $fh => \@layers };
ok(!grep(m/utf-8-strict/, @layers), "$fh: utf-8-strict does not appear in the perlio layers")
ok(!grep(m/utf-?8[-_]strict/, @layers), "$fh: utf-?8[-_]strict does not appear in the perlio layers")
or diag explain { $fh => \@layers };
}

Expand All @@ -52,6 +52,8 @@ for my $fh (keys %handles) {
my @layers = PerlIO::get_layers($handles{$fh});
ok(grep(m/utf8/, @layers), "$fh: utf8 does appear in the perlio layers")
or diag explain { $fh => \@layers };
ok(grep(m/utf-?8[-_]strict/, @layers), "$fh: utf-?8[-_]strict does appear in the perlio layers")
or diag explain { $fh => \@layers };
}

done_testing;
2 changes: 2 additions & 0 deletions t/open.t
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ ok open my $in, '<', 'corpus/testfile';
my @layers = PerlIO::get_layers($in);
ok(grep(m/utf8/, @layers), 'utf8 appears in the perlio layers')
or diag explain { $fh => \@layers };
ok(grep(m/utf-?8[-_]strict/, @layers), 'utf-?8[-_]strict appears in the perlio layers')
or diag explain { $fh => \@layers };

my $contents = do { local $/; <$in>};
is $contents, "\x{30c6}\x{30b9}\x{30c8}\n", 'unicode retrieved OK';
Expand Down
6 changes: 4 additions & 2 deletions t/utf8.t
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ use Test::More;
my @layers = PerlIO::get_layers($fh);
ok(grep(m/utf8/, @layers), 'utf8 appears in the perlio layers')
or diag explain { $fh => \@layers };
ok(grep(m/utf-?8[-_]strict/, @layers), 'utf-?8[-_]strict appears in the perlio layers')
or diag explain { $fh => \@layers };
}
}

Expand All @@ -31,14 +33,14 @@ use Test::More;
END { unlink "perlio_test2" }

my @layers = PerlIO::get_layers($test_fh);
SKIP: {
SKIP: {
# If we have the Perl Unicode flag set that adds the UTF-8 layer,
# we need to skip this test.
skip 'Perl Unicode flag set that always adds UTF-8 layer to output', 1 if (${^UNICODE} & 16);
ok( !grep(/utf8/, @layers), q{utf8 doesn't appear in perlio layers})
or diag explain { $test_fh => \@layers };
}
ok( !grep(m/utf-8-strict/, @layers), q{utf-8-strict doesn't appear in the perlio layers})
ok( !grep(m/utf-?8[-_]strict/, @layers), q{utf-?8[-_]strict doesn't appear in the perlio layers})
or diag explain { $test_fh => \@layers };

}
Expand Down

0 comments on commit afdc66c

Please sign in to comment.