Merge branch 'develop'

cmhughes · Mar 16, 2024 · 16c9bd6 · 16c9bd6
2 parents 9052310 + 7b5ad07
commit 16c9bd6
Show file tree

Hide file tree

Showing 86 changed files with 1,404 additions and 84 deletions.
diff --git a/.github/workflows/batch-check.yaml b/.github/workflows/batch-check.yaml
@@ -29,8 +29,8 @@ name: Batch check of latexindent.pl
 
 on:
   push:
-    branches:
-      - develop
+    branches-ignore:
+      - main
 
 jobs:
   check-pm-files-changed:
@@ -50,7 +50,7 @@ jobs:
         with:
           fetch-depth: 2
       - name: Check for changed pm files, latexindent.pl, defaultSettings.yaml
-        uses: tj-actions/changed-files@v41
+        uses: tj-actions/changed-files@v42
         id: changed-files
         with:
           files: |
@@ -102,7 +102,12 @@ jobs:
         with:
           perl-version: ${{ matrix.perl }}
           distribution: strawberry
-      - name: installing perl modules for latexindent.pl
+      - if: ${{ matrix.platform == 'windows-latest' }}
+        name: installing perl modules for latexindent.pl on windows-latest
+        run: |
+          cpanm --force Win32::Unicode::File
+          cpanm --force Win32::API
+      - name: installing perl modules for latexindent.pl 
         run: |
           cpanm YAML::Tiny
           cpanm -f File::HomeDir
@@ -145,6 +150,19 @@ jobs:
           cpanm YAML::Tiny
           cpanm File::HomeDir
           cpanm Unicode::GCString
+          cpanm --force Win32::Unicode::File
+          cpanm --force Win32::API
+      - name: rebuild and install PAR-Packer-master
+      #If the version number of PAR::Packer does not exceed 1.061, it is necessary to rebuild and install PAR-Packer-master
+      #https://github.com/rschupp/PAR-Packer/issues/84
+      #Issue #84: pass UTF-16 command line down to my_perl (using _wspawnvp)
+        run: |
+          Invoke-WebRequest -Uri https://github.com/rschupp/PAR-Packer/archive/refs/heads/master.zip -OutFile PAR-Packer-master.zip
+          unzip PAR-Packer-master.zip
+          cd PAR-Packer-master
+          perl Makefile.PL
+          gmake
+          gmake install
       - name: preparations for PAR packer
         run: |
           sed -i'.bak' -r 's,eval\s\"use\sUnicode::GCString\"\sif\s\$switches\{GCString\},use Unicode::GCString,' latexindent.pl
@@ -155,7 +173,9 @@ jobs:
       #
       - name: create latexindent.exe using PAR Packer
         run: >-
-          pp 
+          pp
+          -T
+          -M Win32::Unicode
           --addfile="defaultSettings.yaml;lib/LatexIndent/defaultSettings.yaml"
           --cachedeps=scancache
           --output latexindent.exe
@@ -168,8 +188,19 @@ jobs:
         run: |
           ./latexindent.exe -l test-cases/replacements/issue-503b.yaml -r test-cases/replacements/issue-503a.tex 
           cat test-cases/replacements/indent.log
-      ### - name: latexindent.exe cjk test, issue 505
-      ###   # https://github.com/cmhughes/latexindent.pl/issues/505
-      ###   run: |
-      ###     ./latexindent.exe -w test-cases/back-up-tests/新建.tex -y 'onlyOneBackUp:1'
-      ###     cat test-cases/back-up-tests/indent.log
+      - name: latexindent.exe one-sentence-per-line test, issue 514
+        # https://github.com/cmhughes/latexindent.pl/issues/514
+        run: |
+          ./latexindent.exe -m -l test-cases/oneSentencePerLine/issue-514.yaml test-cases/oneSentencePerLine/issue-514.tex
+          cat test-cases/oneSentencePerLine/indent.log
+      - name: latexindent.exe cjk test, issue 505
+        # https://github.com/cmhughes/latexindent.pl/issues/505
+        run: |
+          ./latexindent.exe -w test-cases/back-up-tests/新建äö.tex -y 'onlyOneBackUp:1' 
+          cat test-cases/back-up-tests/indent.log
+          ./latexindent.exe test-cases/back-up-tests/新建äö.tex -o=++ -y 'onlyOneBackUp:1'
+          cat test-cases/back-up-tests/indent.log
+          ./latexindent.exe test-cases/back-up-tests/新建äö.tex -o=+-新建 -l="test-cases/back-up-tests/新建äö.yaml" -r -y 'onlyOneBackUp:1'
+          cat test-cases/back-up-tests/indent.log
+          ./latexindent.exe test-cases/back-up-tests/新建äö.tex -g="新建äö.log" -c="D:/新建"
+          cat D:/新建/新建äö.log
diff --git a/.github/workflows/build-documentation-and-executables.yaml b/.github/workflows/build-documentation-and-executables.yaml
@@ -86,6 +86,19 @@ jobs:
           cpanm YAML::Tiny
           cpanm File::HomeDir
           cpanm Unicode::GCString
+          cpanm --force Win32::Unicode::File
+          cpanm --force Win32::API
+      - name: rebuild and install PAR-Packer-master
+      #If the version number of PAR::Packer does not exceed 1.061, it is necessary to rebuild and install PAR-Packer-master
+      #https://github.com/rschupp/PAR-Packer/issues/84
+      #Issue #84: pass UTF-16 command line down to my_perl (using _wspawnvp)
+        run: |
+          Invoke-WebRequest -Uri https://github.com/rschupp/PAR-Packer/archive/refs/heads/master.zip -OutFile PAR-Packer-master.zip
+          unzip PAR-Packer-master.zip
+          cd PAR-Packer-master
+          perl Makefile.PL
+          gmake
+          gmake install
       - name: preparations for PAR packer
         run: |
           sed -i'.bak' -r 's,eval\s\"use\sUnicode::GCString\"\sif\s\$switches\{GCString\},use Unicode::GCString,' latexindent.pl
@@ -96,7 +109,9 @@ jobs:
       #
       - name: create latexindent.exe using PAR Packer
         run: >-
-          pp 
+          pp
+          -T
+          -M Win32::Unicode
           --addfile="defaultSettings.yaml;lib/LatexIndent/defaultSettings.yaml"
           --cachedeps=scancache
           --output latexindent.exe
@@ -363,7 +378,7 @@ jobs:
           mv cmh01 latexindent-release.md
           cat latexindent-release.md
       - name: latexindent.pl release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         with:
           body_path: latexindent-release.md
           files: |

diff --git a/.github/workflows/test-cases-on-pull.yaml b/.github/workflows/test-cases-on-pull.yaml
@@ -38,7 +38,7 @@ jobs:
           fetch-depth: 0
       - name: Get changed files, latexindent.pl
         id: changed-files
-        uses: tj-actions/changed-files@v41
+        uses: tj-actions/changed-files@v42
         with:
           files: |
             latexindent.pl
@@ -74,6 +74,7 @@ jobs:
         run: |
           cpanm YAML::Tiny
           cpanm File::HomeDir
+          cpanm Unicode::GCString
       # checkout github.com/cmhughes/latexindent.pl
       #
       # https://github.com/actions/checkout
@@ -91,7 +92,7 @@ jobs:
           sed -i.bak "s/checkgitdiff//" test-cases.sh
           ./test-cases.sh -s
       - name: Check for changed files in test-cases
-        uses: tj-actions/changed-files@v41
+        uses: tj-actions/changed-files@v42
         id: changed-files
         with:
           files: |

diff --git a/Dockerfile b/Dockerfile
@@ -17,7 +17,7 @@ FROM perl:5.38.2-slim-threaded-buster
 #
 
 ARG LATEXINDENT_VERSION
-ENV LATEXINDENT_VERSION ${LATEXINDENT_VERSION:-V3.23.6}
+ENV LATEXINDENT_VERSION ${LATEXINDENT_VERSION:-V3.23.7}
 
 RUN apt-get update \
     && apt-get install \

diff --git a/LatexIndent/BackUpFileProcedure.pm b/LatexIndent/BackUpFileProcedure.pm
@@ -26,6 +26,10 @@ use Exporter qw/import/;
 use Encode   qw/decode/;
 our @EXPORT_OK = qw/create_back_up_file check_if_different/;
 
+use LatexIndent::UTF8CmdLineArgsFileOperation
+    qw/copy_with_encode exist_with_encode open_with_encode  zero_with_encode read_yaml_with_encode/;
+use utf8;
+
 # copy main file to a backup in the case of the overwrite switch being active
 
 sub create_back_up_file {
@@ -36,7 +40,7 @@ sub create_back_up_file {
     # if we want to overwrite the current file create a backup first
     $logger->info("*Backup procedure (-w flag active):");
 
-    my $fileName = decode( "utf-8", ${$self}{fileName} );
+    my $fileName = ${$self}{fileName};
 
     # grab the file extension preferences
     my %fileExtensionPreference = %{ $mainSettings{fileExtensionPreference} };
@@ -49,7 +53,7 @@ sub create_back_up_file {
     my $backupFileNoExt = basename( ${$self}{fileName}, @fileExtensions );
 
     # add the user's backup directory to the backup path
-    $backupFileNoExt = decode( "utf-8", "${$self}{cruftDirectory}/$backupFileNoExt" );
+    $backupFileNoExt = "${$self}{cruftDirectory}/$backupFileNoExt";
 
     # local variables, determined from the YAML settings
     my $onlyOneBackUp       = $mainSettings{onlyOneBackUp};
@@ -74,7 +78,7 @@ sub create_back_up_file {
 
         # if the file already exists, increment the number until either
         # the file does not exist, or you reach the maximal number of backups
-        while ( -e ( $backupFile . $backupCounter ) and $backupCounter != ( $maxNumberOfBackUps - 1 ) ) {
+        while ( exist_with_encode( $backupFile . $backupCounter ) and $backupCounter != ( $maxNumberOfBackUps - 1 ) ) {
             $logger->info("$backupFile$backupCounter already exists, incrementing by 1 (see maxNumberOfBackUps)");
             $backupCounter++;
         }
@@ -83,7 +87,7 @@ sub create_back_up_file {
 
     # if the backup file already exists, output some information in the log file
     # and proceed to cycleThroughBackUps if the latter is set
-    if ( -e $backupFile ) {
+    if ( exist_with_encode($backupFile) ) {
         if ($onlyOneBackUp) {
             $logger->info("$backupFile will be overwritten (see onlyOneBackUp)");
         }
@@ -105,9 +109,9 @@ sub create_back_up_file {
                     $newBackupFile = $backupFileNoExt . $backupExtension . ( $i - 1 );
 
                     # check that the oldBackupFile exists
-                    if ( -e $oldBackupFile ) {
+                    if ( exist_with_encode($oldBackupFile) ) {
                         $logger->info("Copying $oldBackupFile to $newBackupFile...");
-                        if ( !( copy( $oldBackupFile, $newBackupFile ) ) ) {
+                        if ( !( copy_with_encode( $oldBackupFile, $newBackupFile ) ) ) {
                             $logger->fatal("*Could not write to backup file $newBackupFile. Please check permissions.");
                             $logger->fatal("Exiting, no indentation done.");
                             $self->output_logfile();
@@ -122,7 +126,7 @@ sub create_back_up_file {
     # output these lines to the log file
     $logger->info("Backing up $fileName to $backupFile...");
     $logger->info("$fileName will be overwritten after indentation");
-    if ( !( copy( $fileName, $backupFile ) ) ) {
+    if ( !( copy_with_encode( $fileName, $backupFile ) ) ) {
         $logger->fatal("*Could not write to backup file $backupFile. Please check permissions.");
         $logger->fatal("Exiting, no indentation done.");
         $self->output_logfile();

diff --git a/LatexIndent/Document.pm b/LatexIndent/Document.pm
@@ -19,7 +19,8 @@ use strict;
 use warnings;
 use Data::Dumper;
 use File::Basename;    # to get the filename and directory path
-use open ':std', ':encoding(UTF-8)';
+
+#use open ':std', ':encoding(UTF-8)';
 use Encode qw/decode/;
 
 # gain access to subroutines in the following modules
@@ -73,6 +74,10 @@ use LatexIndent::Heading      qw/find_heading construct_headings_levels $allHead
 use LatexIndent::FileContents qw/find_file_contents_environments_and_preamble/;
 use LatexIndent::Preamble;
 
+use LatexIndent::UTF8CmdLineArgsFileOperation
+    qw/copy_with_encode exist_with_encode open_with_encode  zero_with_encode read_yaml_with_encode/;
+use utf8;
+
 sub new {
 
     # Create new objects, with optional key/value pairs
@@ -98,7 +103,7 @@ sub latexindent {
 
     # one-time operations
     $self->store_switches;
-    ${$self}{fileName} = decode( "utf-8", $fileNames[0] );
+    ${$self}{fileName} = $fileNames[0];
     $self->process_switches( \@fileNames );
     $self->yaml_read_settings;
 
@@ -231,19 +236,18 @@ sub output_indented_text {
     if ( ${$self}{overwrite} ) {
 
         # diacritics in file names (highlighted in https://github.com/cmhughes/latexindent.pl/pull/439)
-        ${$self}{fileName} = decode( "utf-8", ${$self}{fileName} );
+        ${$self}{fileName} = ${$self}{fileName};
 
         $logger->info("Overwriting file ${$self}{fileName}");
-        open( OUTPUTFILE, ">", ${$self}{fileName} );
-        print OUTPUTFILE ${$self}{body};
-        close(OUTPUTFILE);
+        my $OUTPUTFILE = open_with_encode( '>:encoding(UTF-8)', ${$self}{fileName} );
+        print $OUTPUTFILE ${$self}{body};
+        close($OUTPUTFILE);
     }
     elsif ( $switches{outputToFile} ) {
-
         $logger->info("Outputting to file ${$self}{outputToFile}");
-        open( OUTPUTFILE, ">", ${$self}{outputToFile} );
-        print OUTPUTFILE ${$self}{body};
-        close(OUTPUTFILE);
+        my $OUTPUTFILE = open_with_encode( '>:encoding(UTF-8)', ${$self}{outputToFile} );
+        print $OUTPUTFILE ${$self}{body};
+        close($OUTPUTFILE);
     }
     else {
         $logger->info("Not outputting to file; see -w and -o switches for more options.");
@@ -268,10 +272,10 @@ sub output_logfile {
         if ${ $mainSettings{logFilePreferences} }{showGitHubInfoFooter};
 
     # open log file
-    my $logfileName = $switches{logFileName} || "indent.log";
-    my $logfile;
+    my $logfileName     = $switches{logFileName} || "indent.log";
     my $logfilePossible = 1;
-    open( $logfile, ">", "${$self}{cruftDirectory}/$logfileName" ) or $logfilePossible = 0;
+    my $logfile         = open_with_encode( '>:encoding(UTF-8)', "${$self}{cruftDirectory}/$logfileName" )
+        or $logfilePossible = 0;
 
     if ($logfilePossible) {
         foreach my $line ( @{LatexIndent::Logger::logFileLines} ) {