Add unit tests for text UTF text encoding

.gitattributes

@@ -20,3 +20,8 @@
 # Force bash scripts to always use LF line endings so that if a repo is accessed
 # in Unix via a file share from Windows, the scripts will work.
 *.sh text eol=lf
+
+
+# Preserve text encoding, bom-fields & line-endings in test playlist files
+src/test/resources/playlists/m3u/*.m3u binary
+src/test/resources/playlists/m3u/*.m3u8 binary

src/main/java/listfix/io/playlists/PlaylistReader.java

@@ -17,7 +17,6 @@
 import java.nio.charset.Charset;
 import java.nio.file.Path;
 import java.util.List;
-import java.util.Objects;
 import java.util.StringTokenizer;
 
 public abstract class PlaylistReader implements IPlaylistReader
@@ -48,7 +47,7 @@ protected void processEntry(List<PlaylistEntry> results, String L2, String cid,
   {
     StringTokenizer pathTokenizer = null;
     StringBuilder path = new StringBuilder();
-    if (OperatingSystem.isLinux()) // OS Specific Hack
+     if (OperatingSystem.isLinux()) // OS Specific Hack
     {
       if (!L2.startsWith("\\\\") && !L2.startsWith(".") && !L2.startsWith(Constants.FS))
       {
@@ -105,7 +104,7 @@ else if (L2.startsWith(Constants.FS))
           // This token is the closest thing we have to the notion of a 'drive' on any OS...
           // make a file out of this and see if it has any files.
           File testFile = new File(tempPath);
-          if (testFile.isAbsolute() && !(testFile.isDirectory() && Objects.requireNonNull(testFile.list()).length > 0))
+          if (!(testFile.exists() && testFile.isDirectory() && testFile.list().length > 0) && testFile.isAbsolute())
           {
             PlaylistEntry.NonExistentDirectories.add(tempPath);
           }

src/main/java/listfix/io/playlists/m3u/M3UReader.java

@@ -25,7 +25,6 @@
 import listfix.io.playlists.PlaylistReader;
 import listfix.model.enums.PlaylistType;
 import listfix.model.playlists.PlaylistEntry;
-import listfix.util.UnicodeUtils;
 import listfix.view.support.IProgressObserver;
 import listfix.view.support.ProgressAdapter;
 import org.apache.logging.log4j.LogManager;
@@ -59,17 +58,11 @@ public M3UReader(IPlaylistOptions playListOptions, Path m3uPath) throws FileNotF
 
     File m3uFile = m3uPath.toFile();
 
-    encoding = UnicodeUtils.getEncoding(m3uFile);
-    final Charset defaultEncoding = StandardCharsets.UTF_8;
-    if (encoding.equals(defaultEncoding) || m3uFile.getName().toLowerCase().endsWith(".m3u8"))
-    {
-      buffer = new BufferedReader(new InputStreamReader(new UnicodeInputStream(new FileInputStream(m3uFile), defaultEncoding), defaultEncoding));
-      encoding = StandardCharsets.UTF_8;
-    }
-    else
-    {
-      buffer = new BufferedReader(new FileReader(m3uFile));
-    }
+    // encoding = UnicodeUtils.getEncoding(m3uFile);
+    UnicodeInputStream unicodeInputStream = new UnicodeInputStream(new FileInputStream(m3uFile), StandardCharsets.UTF_8);
+    this.encoding = Charset.forName(unicodeInputStream.getEncoding());
+    _logger.info(String.format("Detected M3U encoding for \"%s\": %s", m3uPath.getFileName().toString(), this.encoding.name()));
+    buffer = new BufferedReader(new InputStreamReader(unicodeInputStream, this.encoding));
     fileLength = m3uFile.length();
   }
 

src/main/java/listfix/util/UnicodeUtils.java

@@ -24,6 +24,7 @@
 
 import java.io.*;
 import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 
 /**
@@ -86,7 +87,7 @@ public static Charset getEncoding(Path path)
 
   public static Charset getEncoding(File input)
   {
-    try (UnicodeInputStream stream = new UnicodeInputStream(new FileInputStream(input), "ASCII"))
+    try (UnicodeInputStream stream = new UnicodeInputStream(new FileInputStream(input), StandardCharsets.UTF_8))
     {
       return Charset.forName(stream.getEncoding());
     }

src/test/java/listfix/io/playlists/m3u/M3UReaderTests.java

@@ -0,0 +1,65 @@
+package listfix.io.playlists.m3u;
+
+import listfix.io.IPlaylistOptions;
+import listfix.json.JsonAppOptions;
+import listfix.model.playlists.PlaylistEntry;
+import listfix.util.TestUtil;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.jupiter.api.DisplayName;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+public class M3UReaderTests
+{
+  private IPlaylistOptions playlistOptions;
+
+  @Before
+  public void initOptions() {
+    this.playlistOptions = new JsonAppOptions();
+  }
+
+  private M3UReader buildM3uReader(String playlist) throws IOException
+  {
+    File m3uPlaylistFile = TestUtil.createFileFromResource(this, "/playlists/m3u/" + playlist);
+    return new M3UReader(this.playlistOptions, m3uPlaylistFile.toPath());
+  }
+
+  @Test
+  @DisplayName("Read MP3U Playlist UTF-8 no BOM")
+  public void readPlaylistM3uUtf8NoBom() throws IOException
+  {
+    M3UReader m3UReader = buildM3uReader("playlist-utf8.m3u");
+    List<PlaylistEntry> m3uPlaylist = m3UReader.readPlaylist();
+    assertNotNull(m3uPlaylist, "PlaylistFactory should read and construct M3U playlist");
+    assertEquals(2, m3uPlaylist.size(), "M3U playlist contains 2 tracks");
+    assertEquals(StandardCharsets.UTF_8, m3UReader.getEncoding());
+  }
+
+  @Test
+  @DisplayName("Read MP3U Playlist UTF-16-BE with BOM")
+  public void readPlaylistM3uUtf16BeWithBom() throws IOException
+  {
+    M3UReader m3UReader = buildM3uReader("playlist-utf16be-bom.m3u");
+    List<PlaylistEntry> m3uPlaylist = m3UReader.readPlaylist();
+    assertNotNull(m3uPlaylist, "PlaylistFactory should read and construct M3U playlist");
+    assertEquals(2, m3uPlaylist.size(), "M3U playlist contains 2 tracks");
+    assertEquals(StandardCharsets.UTF_16BE, m3UReader.getEncoding());
+  }
+
+  @Test
+  @DisplayName("Read MP3U Playlist UTF-16-LE with BOM")
+  public void readPlaylistM3uUtf16LeWithBom() throws IOException
+  {
+    M3UReader m3UReader = buildM3uReader("playlist-utf16le-bom.m3u");
+    List<PlaylistEntry> m3uPlaylist = m3UReader.readPlaylist();
+    assertNotNull(m3uPlaylist, "PlaylistFactory should read and construct M3U playlist");
+    assertEquals(2, m3uPlaylist.size(), "M3U playlist contains 2 tracks");
+    assertEquals(StandardCharsets.UTF_16LE, m3UReader.getEncoding());
+  }
+}

src/test/java/listfix/model/playlists/PlaylistFactoryTest.java → src/test/java/listfix/model/playlists/PlaylistFactoryTests.java

@@ -11,7 +11,7 @@
 
 import static org.junit.jupiter.api.Assertions.*;
 
-public class PlaylistFactoryTest
+public class PlaylistFactoryTests
 {
   private IPlaylistOptions playlistOptions;
 
@@ -23,7 +23,7 @@ public void initOptions() {
   @Test
   public void readPlaylistM3u() throws IOException
   {
-    File m3uPlaylistFile = TestUtil.createFileFromResource(this, "/playlists/m3u/playlist.m3u");
+    File m3uPlaylistFile = TestUtil.createFileFromResource(this, "/playlists/m3u/playlist-utf8.m3u");
 
     Playlist m3uPlaylist = PlaylistFactory.getPlaylist(m3uPlaylistFile.toPath(), null, playlistOptions);
     assertNotNull(m3uPlaylist, "PlaylistFactory should read and construct M3U playlist");

src/test/resources/playlists/m3u/playlist-utf8-bom.m3u

@@ -0,0 +1,7 @@
+#EXTM3U
+
+#EXTINF:111, Sample artist name - Sample track title
+C:\Music\SampleMusic.mp3
+
+#EXTINF:222,Example Artist name - Example track title
+C:\Music\ExampleMusic.mp3

src/test/resources/playlists/m3u/playlist.m3u → src/test/resources/playlists/m3u/playlist-utf8.m3u

@@ -1,7 +1,7 @@
-#EXTM3U
-
-#EXTINF:111, Sample artist name - Sample track title
-C:\Music\SampleMusic.mp3
-
-#EXTINF:222,Example Artist name - Example track title
+#EXTM3U
+
+#EXTINF:111, Sample artist name - Sample track title
+C:\Music\SampleMusic.mp3
+
+#EXTINF:222,Example Artist name - Example track title
 C:\Music\ExampleMusic.mp3