Merge branch 'develop'

Dockerfile

@@ -17,7 +17,7 @@ FROM perl:5.38.2-slim-threaded-buster
 #
 
 ARG LATEXINDENT_VERSION
-ENV LATEXINDENT_VERSION ${LATEXINDENT_VERSION:-V3.24.1}
+ENV LATEXINDENT_VERSION ${LATEXINDENT_VERSION:-V3.24.2}
 
 RUN apt-get update \
     && apt-get install \

LatexIndent/GetYamlSettings.pm

@@ -389,36 +389,8 @@ sub yaml_read_settings {
                 # output the contents of indentconfig to the log file
                 $logger->info( Dump \%{ $userSettings->[0] } );
 
-                # change the encoding of the paths according to the field `encoding`
-                if ( $userSettings and ( ref( $userSettings->[0] ) eq 'HASH' ) and $userSettings->[0]->{encoding} ) {
-                    use Encode;
-                    my $encoding       = $userSettings->[0]->{encoding};
-                    my $encodingObject = find_encoding($encoding);
-
-                    # Check if the encoding is valid.
-                    if ( ref($encodingObject) ) {
-                        $logger->info("*Encoding of the paths is $encoding");
-                        foreach ( @{ $userSettings->[0]->{paths} } ) {
-                            my $temp = $encodingObject->encode("$_");
-                            $logger->info("Transform file encoding: $_ -> $temp");
-                            push( @absPaths, $temp );
-                        }
-                    }
-                    else {
-                        $logger->warn("*encoding \"$encoding\" not found");
-                        $logger->warn("Ignore this setting and will take the default encoding.");
-                        @absPaths = @{ $userSettings->[0]->{paths} };
-                        foreach ( @{ $userSettings->[0]->{paths} } ) {
-                            push( @absPaths, $_ );
-                        }
-                    }
-                }
-                else    # No such setting, and will take the default
-                {
-                    # $logger->info("*Encoding of the paths takes the default.");
-                    foreach ( @{ $userSettings->[0]->{paths} } ) {
-                        push( @absPaths, $_ );
-                    }
+                foreach ( @{ $userSettings->[0]->{paths} } ) {
+                    push( @absPaths, $_ );
                 }
             }
 

LatexIndent/Logger.pm

@@ -18,7 +18,7 @@ package LatexIndent::Logger;
 
 use strict;
 use warnings;
-use Exporter;
+use Exporter qw/import/;
 use LatexIndent::Switches qw/%switches/;
 our @ISA       = "LatexIndent::Document";    # class inheritance, Programming Perl, pg 321
 our @EXPORT_OK = qw/@logFileLines/;

LatexIndent/Special.pm

@@ -17,14 +17,14 @@ package LatexIndent::Special;
 #	For all communication, please visit: https://github.com/cmhughes/latexindent.pl
 use strict;
 use warnings;
+use Exporter                      qw/import/;
 use LatexIndent::Tokens           qw/%tokens/;
 use LatexIndent::TrailingComments qw/$trailingCommentRegExp/;
 use LatexIndent::GetYamlSettings  qw/%mainSettings/;
 use LatexIndent::Switches         qw/$is_t_switch_active $is_tt_switch_active/;
 use LatexIndent::LogFile          qw/$logger/;
-use LatexIndent::IfElseFi         qw/$ifElseFiBasicRegExp/;
+use LatexIndent::IfElseFi;
 use Data::Dumper;
-use Exporter qw/import/;
 our @ISA = "LatexIndent::Document";    # class inheritance, Programming Perl, pg 321
 our @EXPORT_OK
     = qw/find_special construct_special_begin $specialBeginAndBracesBracketsBasicRegExp $specialBeginBasicRegExp/;

LatexIndent/TrailingComments.pm

@@ -27,12 +27,20 @@ our @EXPORT_OK
     = qw/remove_trailing_comments put_trailing_comments_back_in $trailingCommentRegExp add_comment_symbol construct_trailing_comment_regexp @trailingComments/;
 our @trailingComments;
 our $commentCounter = 0;
+our $notPrecededByRegExp;
 our $trailingCommentRegExp;
 
 sub construct_trailing_comment_regexp {
-    my $notPreceededBy = qr/${${$mainSettings{fineTuning}}{trailingComments}}{notPreceededBy}/;
+    $notPrecededByRegExp = qr/${${$mainSettings{fineTuning}}{trailingComments}}{notPrecededBy}/;
+    my $notPreceededBy = ${${mainSettings{fineTuning}}{trailingComments}}{notPreceededBy};
 
-    $trailingCommentRegExp = qr/$notPreceededBy%$tokens{trailingComment}\d+$tokens{endOfToken}/;
+    if ( $notPreceededBy ) {
+        $logger->warn(
+            "*fineTuning:trailingComments:notPreceededBy is ok for now, but in future versions, fineTuning:trailingComments:notPrecededBy will be used" );
+        $notPrecededByRegExp = qr/$notPreceededBy/;
+    }
+
+    $trailingCommentRegExp = qr/$notPrecededByRegExp%$tokens{trailingComment}\d+$tokens{endOfToken}/;
 }
 
 sub add_comment_symbol {
@@ -66,17 +74,16 @@ sub remove_trailing_comments {
 
     $logger->trace("*Storing trailing comments") if $is_t_switch_active;
 
-    my $notPreceededBy = qr/${${$mainSettings{fineTuning}}{trailingComments}}{notPreceededBy}/;
-    my $afterComment   = qr/${${$mainSettings{fineTuning}}{trailingComments}}{afterComment}/;
+    my $afterComment = qr/${${$mainSettings{fineTuning}}{trailingComments}}{afterComment}/;
 
     # perform the substitution
     ${$self}{body} =~ s/
-                            $notPreceededBy   # not preceded by a \
-                            %                 # % 
+                            $notPrecededByRegExp  # not preceded by a \
+                            %                     # % 
                             (
-                                $afterComment # anything else
+                                $afterComment     # anything else
                             )
-                            $                 # up to the end of a line
+                            $                     # up to the end of a line
                         /   
                             # increment comment counter and store comment
                             $commentCounter++;
@@ -134,14 +141,13 @@ sub put_trailing_comments_back_in {
             # replace the line-broken trailing comment ID with a non-broken trailing comment ID
             ${$self}{body} =~ s/%\R?$trailingcommentIDwithLineBreaksRegExp/%$trailingcommentID/s;
         }
-        my $notPreceededBy = qr/${${$mainSettings{fineTuning}}{trailingComments}}{notPreceededBy}/;
         if (${$self}{body} =~ m/%$trailingcommentID
                               (
-                                  (?!                  # not immediately preceded by 
-                                      $notPreceededBy  # \
-                                      %                # %
+                                  (?!                       # not immediately preceded by 
+                                      $notPrecededByRegExp  # \
+                                      %                     # %
                                   ).*?
-                              )                        # captured into $1
+                              )                             # captured into $1
                               (\h*)?$                
                           /mx and $1 ne ''
             )

LatexIndent/UTF8CmdLineArgsFileOperation.pm

@@ -5,43 +5,61 @@ use warnings;
 use feature qw( say state );
 use utf8;
 use Config qw( %Config );
-use Encode qw( decode encode );
+use Encode qw(find_encoding decode encode );
 
 use Exporter qw/import/;
 our @EXPORT_OK
     = qw/commandlineargs_with_encode @new_args copy_with_encode exist_with_encode open_with_encode zero_with_encode read_yaml_with_encode isdir_with_encode mkdir_with_encode/;
 
+our $encodingObject;  
+
+if ($^O eq 'MSWin32') {
+    my $encoding_sys = 'cp' . Win32::GetACP();
+    $encodingObject = find_encoding( $encoding_sys );
+
+    # Check if the encoding is valid.
+    unless ( ref($encodingObject) ) {
+        $encodingObject = find_encoding( 'utf-8' );
+    }
+}
+else {
+    $encodingObject = find_encoding( 'utf-8' );
+}
+
 sub copy_with_encode {
     use File::Copy;
     my ( $source, $destination ) = @_;
 
-    if ( $FindBin::Script eq 'latexindent.exe' ) {
+    if ( $FindBin::Script =~ /\.exe$/ ) {
         require Win32::Unicode::File;
         Win32::Unicode::File->import(qw(copyW));
         copyW( $source, $destination, 1 );
     }
     else {
+        $source = $encodingObject->encode($source);
+        $destination = $encodingObject->encode($destination);
         copy( $source, $destination );
     }
 }
 
 sub exist_with_encode {
     my ($filename) = @_;
 
-    if ( $FindBin::Script eq 'latexindent.exe' ) {
+    if ( $FindBin::Script =~ /\.exe$/ ) {
         require Win32::Unicode::File;
         Win32::Unicode::File->import(qw(statW));
         return statW($filename);
     }
     else {
+        $filename = $encodingObject->encode($filename);
         return -e $filename;
     }
 }
 
 sub zero_with_encode {
     my ($filename) = @_;
 
-    if ( $FindBin::Script eq 'latexindent.exe' ) {
+    if ( $FindBin::Script =~ /\.exe$/ ) {
         require Win32::Unicode::File;
         Win32::Unicode::File->import(qw(file_size));
         my $size = file_size($filename);
@@ -53,6 +71,7 @@ sub zero_with_encode {
         }
     }
     else {
+        $filename = $encodingObject->encode($filename);
         return -z $filename;
     }
 }
@@ -62,7 +81,7 @@ sub open_with_encode {
     my $filename = shift;
     my $fh;
 
-    if ( $FindBin::Script eq 'latexindent.exe' ) {
+    if ( $FindBin::Script =~ /\.exe$/ ) {
         require Win32::Unicode::File;
         Win32::Unicode::File->import;
         $fh = Win32::Unicode::File->new;
@@ -74,6 +93,7 @@ sub open_with_encode {
         }
     }
     else {
+        $filename = $encodingObject->encode($filename);
         if ( open( $fh, $mode, $filename ) ) {
             return $fh;
         }
@@ -95,21 +115,22 @@ sub read_yaml_with_encode {
 sub isdir_with_encode {
     my $path = shift;
 
-    if ( $FindBin::Script eq 'latexindent.exe' ) {
+    if ( $FindBin::Script =~ /\.exe$/ ) {
         require Win32::Unicode::File;
         Win32::Unicode::File->import(qw(file_type));
 
         return file_type( 'd', $path );
     }
     else {
+        $path = $encodingObject->encode($path);
         return -d $path;
     }
 }
 
 sub mkdir_with_encode {
     my $path = shift;
 
-    if ( $FindBin::Script eq 'latexindent.exe' ) {
+    if ( $FindBin::Script =~ /\.exe$/ ) {
         require Win32::Unicode::Dir;
         Win32::Unicode::Dir->import(qw(mkdirW));
 
@@ -118,15 +139,15 @@ sub mkdir_with_encode {
     else {
         require File::Path;
         File::Path->import(qw(make_path));
-
+        $path = $encodingObject->encode($path);
         make_path($path);
     }
 }
 
 #https://stackoverflow.com/a/63868721
 #https://stackoverflow.com/a/44489228
 sub commandlineargs_with_encode {
-    if ( $FindBin::Script eq 'latexindent.exe' ) {
+    if ( $FindBin::Script =~ /\.exe$/ ) {
         require Win32::API;
         import Win32::API qw( ReadMemory );
 
@@ -207,7 +228,6 @@ sub commandlineargs_with_encode {
         @ARGV = @{$args};
     }
     else {
-        my $encodingObject = "utf-8";
         @ARGV = map { decode( $encodingObject, $_ ) } @ARGV;
         our @new_args = @ARGV;
     }

LatexIndent/Version.pm

@@ -20,6 +20,6 @@ use warnings;
 use Exporter qw/import/;
 our @EXPORT_OK = qw/$versionNumber $versionDate/;
 
-our $versionNumber = '3.24.1';
-our $versionDate   = '2024-05-12';
+our $versionNumber = '3.24.2';
+our $versionDate   = '2024-06-15';
 1

defaultSettings.yaml

@@ -1,5 +1,5 @@
 #
-# latexindent.pl, version 3.24.1, 2024-05-12
+# latexindent.pl, version 3.24.2, 2024-06-15
 #
 # defaultSettings.yaml, the default settings for latexindent.pl
 #
@@ -649,7 +649,7 @@ fineTuning:
     before: (?:#\d\h*;?,?\/?)+|\<.*?\>
     between: _|\^|\*
   trailingComments:
-    notPreceededBy: (?<!\\)
+    notPrecededBy: (?<!\\)
     afterComment: .*?
   modifyLineBreaks:
     doubleBackSlash: \\\\(?:\h*\[\h*\d+\h*[a-zA-Z]+\h*\])?

documentation/changelog.md

@@ -1,6 +1,9 @@
-<!-- announcement: oneSentencePerLine par issue bug fix-->
+<!-- announcement: encoding bug fix-->
 
 # changelog.md
+## V3.24.2, June 15, 2024
+encoding bug fix, see [issue 547](https://github.com/cmhughes/latexindent.pl/issues/547), thanks to @fengzyf
+
 ## V3.24.1, May 12, 2024
 oneSentencePerLine par issue bug fix, see [issue 527](https://github.com/cmhughes/latexindent.pl/issues/527)
 

documentation/conf.py

@@ -63,9 +63,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'3.24.1'
+version = u'3.24.2'
 # The full version, including alpha/beta/rc tags.
-release = u'3.24.1'
+release = u'3.24.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

documentation/contributors.bib

@@ -224,3 +224,14 @@ @online{jessestricker
 	author   = "Jesse Stricker",
 	urldate  = {2023-07-12},
 	keywords = {contributor},}
+
+%
+% 2024
+%
+@online{fengzyf,
+	title    = "Encoding work",
+	url      = "https://github.com/cmhughes/latexindent.pl/pull/548",
+	date     = {2024-06-15},
+	author   = "fengzyf",
+	urldate  = {2024-06-15},
+	keywords = {contributor},}

documentation/demonstrations/fine-tuning3.yaml

@@ -1,6 +1,6 @@
 fineTuning:
     trailingComments:
-      notPreceededBy: (?<!\\)
+      notPrecededBy: (?<!\\)
       afterComment: (?!(?:\hend)).*?
 
 specialBeginEnd:

documentation/demonstrations/href2.yaml

@@ -1,6 +1,6 @@
 fineTuning:
     trailingComments:
-      notPreceededBy: '(?:(?<!Handbook)(?<!for)(?<!Spoken))'
+      notPrecededBy: '(?:(?<!Handbook)(?<!for)(?<!Spoken))'
 
 modifyLineBreaks:
     textWrapOptions:

documentation/demonstrations/pre-commit-config-conda.yaml

@@ -1,5 +1,5 @@
 - repo: https://github.com/cmhughes/latexindent.pl
-  rev: V3.24.1
+  rev: V3.24.2
   hooks:
   - id: latexindent-conda
     args: [-s]

documentation/demonstrations/pre-commit-config-cpan.yaml

@@ -1,5 +1,5 @@
 - repo: https://github.com/cmhughes/latexindent.pl
-  rev: V3.24.1
+  rev: V3.24.2
   hooks:
   - id: latexindent
     args: [-s]

documentation/demonstrations/pre-commit-config-demo.yaml

@@ -1,5 +1,5 @@
 - repo: https://github.com/cmhughes/latexindent.pl
-  rev: V3.24.1
+  rev: V3.24.2
   hooks:
   - id: latexindent
     args: [-l, -m, -s, -w]

documentation/demonstrations/pre-commit-config-docker.yaml

@@ -1,5 +1,5 @@
 - repo: https://github.com/cmhughes/latexindent.pl
-  rev: V3.24.1
+  rev: V3.24.2
   hooks:
     - id: latexindent-docker
       args: [-s]