fix: handle tilde/paths in the query string, cleanup tests (#43)

videojs · Nov 20, 2020 · 4a528ce · 4a528ce
1 parent ca1b6e2
commit 4a528ce
Show file tree

Hide file tree

Showing 10 changed files with 1,487 additions and 1,018 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -7,6 +7,7 @@
     "docs:toc": "doctoc README.md",
     "lint": "vjsstandard",
     "pretest": "npm run lint",
+    "start": "npm run test -- --watch",
     "test": "NODE_ENV=test mocha --opts test/opts/unit.opts test/unit",
     "update-changelog": "conventional-changelog -p videojs -i CHANGELOG.md -s",
     "preversion": "npm test",
@@ -47,7 +48,7 @@
     "husky": "^1.0.0-rc.13",
     "lint-staged": "^7.2.2",
     "mocha": "^5.2.0",
-    "nock": "^9.3.3",
+    "nock": "^13.0.4",
     "not-prerelease": "^1.0.1",
     "npm-merge-driver-install": "^1.0.0",
     "videojs-generator-verify": "~1.0.4",

diff --git a/src/walk-manifest.js b/src/walk-manifest.js
@@ -19,6 +19,15 @@ const fsSanitize = function(filepath) {
     .join(path.sep);
 };
 
+const urlBasename = function(uri) {
+  const parsed = url.parse(uri);
+  const pathname = parsed.pathname || parsed.path.replace(parsed.query || '', '');
+  const query = (parsed.query || '').split(/\\\\|\\|\//).join('');
+  const basename = path.basename(pathname) + query;
+
+  return fsSanitize(basename);
+};
+
 const joinURI = function(absolute, relative) {
   const parse = url.parse(absolute);
 
@@ -129,7 +138,7 @@ const parseKey = function(requestOptions, basedir, decrypt, resources, manifest,
       if (parent) {
         key.file = path.dirname(parent.file);
       }
-      key.file = path.join(key.file, path.basename(fsSanitize(key.uri)));
+      key.file = path.join(key.file, urlBasename(key.uri));
 
       manifest.content = Buffer.from(manifest.content.toString().replace(
         key.uri,
@@ -207,31 +216,43 @@ const walkPlaylist = function(options) {
     let resources = [];
     const manifest = {parent};
 
-    manifest.uri = uri;
-    manifest.file = path.join(basedir, path.basename(fsSanitize(uri)));
+    if (uri) {
+      manifest.uri = uri;
+      manifest.file = path.join(basedir, urlBasename(uri));
+    }
+
+    let existingManifest;
 
     // if we are not the master playlist
     if (dashPlaylist && parent) {
       manifest.file = parent.file;
       manifest.uri = parent.uri;
+      existingManifest = visitedUrls[manifest.uri];
     } else if (parent) {
       manifest.file = path.join(
-        path.dirname(parent.file),
+        basedir,
+        path.dirname(path.relative(basedir, parent.file)),
         'manifest' + manifestIndex,
-        path.basename(fsSanitize(manifest.file))
+        path.basename(manifest.file)
       );
       // get the real uri of this playlist
       if (!isAbsolute(manifest.uri)) {
         manifest.uri = joinURI(path.dirname(parent.uri), manifest.uri);
       }
+      existingManifest = visitedUrls[manifest.uri];
+
+      const file = existingManifest && existingManifest.file || manifest.file;
+      const relativePath = path.relative(path.dirname(parent.file), file);
+
       // replace original uri in file with new file path
-      parent.content = Buffer.from(parent.content.toString().replace(uri, path.relative(path.dirname(parent.file), manifest.file)));
+      parent.content = Buffer.from(parent.content.toString().replace(uri, relativePath));
     }
 
-    if (!dashPlaylist && visitedUrls.includes(manifest.uri)) {
+    if (!dashPlaylist && existingManifest) {
       console.error(`[WARN] Trying to visit the same uri again; skipping to avoid getting stuck in a cycle: ${manifest.uri}`);
       return resolve(resources);
     }
+    visitedUrls[manifest.uri] = manifest;
 
     let requestPromise;
 
@@ -258,7 +279,6 @@ const walkPlaylist = function(options) {
 
       if (!dashPlaylist) {
         resources.push(manifest);
-        visitedUrls.push(manifest.uri);
 
         manifest.content = response.body;
         if ((/^application\/dash\+xml/i).test(response.headers['content-type']) || (/^\<\?xml/i).test(response.body)) {
@@ -298,7 +318,7 @@ const walkPlaylist = function(options) {
             return;
           }
           // put segments in manifest-name/segment-name.ts
-          s.file = path.join(path.dirname(manifest.file), path.basename(fsSanitize(s.uri)));
+          s.file = path.join(path.dirname(manifest.file), urlBasename(s.uri));
 
           if (!isAbsolute(s.uri)) {
             s.uri = joinURI(path.dirname(manifest.uri), s.uri);

diff --git a/test/resources/duplicate-manifests/manifest0/rendition.m3u8 b/test/resources/duplicate-manifests/manifest0/rendition.m3u8
@@ -0,0 +1,24 @@
+#EXTM3U
+#EXT-X-VERSION:3
+#EXT-X-PLAYLIST-TYPE:VOD
+#EXT-X-MEDIA-SEQUENCE:0
+#EXT-X-TARGETDURATION:10
+#EXTINF:9.985,
+segment0.ts
+#EXTINF:9.985,
+segment1.ts
+#EXTINF:9.985,
+segment2.ts
+#EXTINF:9.985,
+segment3.ts
+#EXTINF:9.985,
+segment4.ts
+#EXTINF:9.985,
+segment5.ts
+#EXTINF:9.985,
+segment6.ts
+#EXTINF:9.985,
+segment7.ts
+#EXTINF:6.177,
+segment8.ts
+#EXT-X-ENDLIST
diff --git a/test/resources/duplicate-manifests/manifest1/rendition.m3u8 b/test/resources/duplicate-manifests/manifest1/rendition.m3u8
@@ -0,0 +1,24 @@
+#EXTM3U
+#EXT-X-VERSION:3
+#EXT-X-PLAYLIST-TYPE:VOD
+#EXT-X-MEDIA-SEQUENCE:0
+#EXT-X-TARGETDURATION:10
+#EXTINF:9.985,
+segment0.ts
+#EXTINF:9.985,
+segment1.ts
+#EXTINF:9.985,
+segment2.ts
+#EXTINF:9.985,
+segment3.ts
+#EXTINF:9.985,
+segment4.ts
+#EXTINF:9.985,
+segment5.ts
+#EXTINF:9.985,
+segment6.ts
+#EXTINF:9.985,
+segment7.ts
+#EXTINF:6.177,
+segment8.ts
+#EXT-X-ENDLIST
diff --git a/test/resources/duplicate-manifests/manifest2/rendition.m3u8 b/test/resources/duplicate-manifests/manifest2/rendition.m3u8
@@ -0,0 +1,24 @@
+#EXTM3U
+#EXT-X-VERSION:3
+#EXT-X-PLAYLIST-TYPE:VOD
+#EXT-X-MEDIA-SEQUENCE:0
+#EXT-X-TARGETDURATION:10
+#EXTINF:9.985,
+segment0.ts
+#EXTINF:9.985,
+segment1.ts
+#EXTINF:9.985,
+segment2.ts
+#EXTINF:9.985,
+segment3.ts
+#EXTINF:9.985,
+segment4.ts
+#EXTINF:9.985,
+segment5.ts
+#EXTINF:9.985,
+segment6.ts
+#EXTINF:9.985,
+segment7.ts
+#EXTINF:6.177,
+segment8.ts
+#EXT-X-ENDLIST
diff --git a/test/resources/duplicate-manifests/master.m3u8 b/test/resources/duplicate-manifests/master.m3u8
@@ -0,0 +1,11 @@
+#EXTM3U
+#EXT-X-VERSION:4
+#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio-0",NAME="en",DEFAULT=YES,AUTOSELECT=YES,LANGUAGE="en",URI="manifest0/rendition.m3u8"
+#EXT-X-STREAM-INF:PROGRAM-ID=0,BANDWIDTH=70400,CODECS="mp4a.40.2",AUDIO="audio-0",CLOSED-CAPTIONS=NONE
+manifest0/rendition.m3u8
+#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio-1",NAME="en",DEFAULT=YES,AUTOSELECT=YES,LANGUAGE="en",URI="manifest1/rendition.m3u8"
+#EXT-X-STREAM-INF:PROGRAM-ID=0,BANDWIDTH=105600,CODECS="mp4a.40.2",AUDIO="audio-1",CLOSED-CAPTIONS=NONE
+manifest1/rendition.m3u8
+#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio-2",NAME="en",DEFAULT=YES,AUTOSELECT=YES,LANGUAGE="en",URI="manifest2/rendition.m3u8"
+#EXT-X-STREAM-INF:PROGRAM-ID=0,BANDWIDTH=139700,CODECS="mp4a.40.2",AUDIO="audio-2",CLOSED-CAPTIONS=NONE
+manifest2/rendition.m3u8
diff --git a/test/resources/path-query.m3u8 b/test/resources/path-query.m3u8
@@ -0,0 +1,24 @@
+#EXTM3U
+#EXT-X-VERSION:3
+#EXT-X-PLAYLIST-TYPE:VOD
+#EXT-X-MEDIA-SEQUENCE:0
+#EXT-X-TARGETDURATION:10
+#EXTINF:9.985,
+https://example.com/segment0.ts?foo=/do-not/include-me/in-output
+#EXTINF:9.985,
+https://example.com/segment1.ts?foo=/do-not/include-me/in-output
+#EXTINF:9.985,
+https://example.com/segment2.ts?foo=/do-not/include-me/in-output
+#EXTINF:9.985,
+https://example.com/segment3.ts?foo=/do-not/include-me/in-output
+#EXTINF:9.985,
+https://example.com/segment4.ts?foo=/do-not/include-me/in-output
+#EXTINF:9.985,
+https://example.com/segment5.ts?foo=/do-not/include-me/in-output
+#EXTINF:9.985,
+https://example.com/segment6.ts?foo=/do-not/include-me/in-output
+#EXTINF:9.985,
+https://example.com/segment7.ts?foo=/do-not/include-me/in-output
+#EXTINF:6.177,
+https://example.com/segment8.ts?foo=/do-not/include-me/in-output
+#EXT-X-ENDLIST
diff --git a/test/resources/tilde-query-param.m3u8 b/test/resources/tilde-query-param.m3u8
@@ -0,0 +1,24 @@
+#EXTM3U
+#EXT-X-VERSION:3
+#EXT-X-PLAYLIST-TYPE:VOD
+#EXT-X-MEDIA-SEQUENCE:0
+#EXT-X-TARGETDURATION:10
+#EXTINF:9.985,
+https://example.com/segment0.ts?foo_token=exp=4444444444~pool=/123456788787387878728/*~nice=some-uuid-my-dude
+#EXTINF:9.985,
+https://example.com/segment1.ts?foo_token=exp=4444444444~pool=/123456788787387878728/*~nice=some-uuid-my-dude
+#EXTINF:9.985,
+https://example.com/segment2.ts?foo_token=exp=4444444444~pool=/123456788787387878728/*~nice=some-uuid-my-dude
+#EXTINF:9.985,
+https://example.com/segment3.ts?foo_token=exp=4444444444~pool=/123456788787387878728/*~nice=some-uuid-my-dude
+#EXTINF:9.985,
+https://example.com/segment4.ts?foo_token=exp=4444444444~pool=/123456788787387878728/*~nice=some-uuid-my-dude
+#EXTINF:9.985,
+https://example.com/segment5.ts?foo_token=exp=4444444444~pool=/123456788787387878728/*~nice=some-uuid-my-dude
+#EXTINF:9.985,
+https://example.com/segment6.ts?foo_token=exp=4444444444~pool=/123456788787387878728/*~nice=some-uuid-my-dude
+#EXTINF:9.985,
+https://example.com/segment7.ts?foo_token=exp=4444444444~pool=/123456788787387878728/*~nice=some-uuid-my-dude
+#EXTINF:6.177,
+https://example.com/segment8.ts?foo_token=exp=4444444444~pool=/123456788787387878728/*~nice=some-uuid-my-dude
+#EXT-X-ENDLIST