feat: .video() now returns mediaDefinition as an alternative to `…

…videos` closes #90
pionxzh · Oct 7, 2023 · ef1069e · ef1069e
1 parent 3f5ecd8
commit ef1069e
Show file tree

Hide file tree

Showing 8 changed files with 103 additions and 20 deletions.
diff --git a/README.md b/README.md
@@ -88,7 +88,28 @@ console.log(video)
   "provider": { "username": "wowgirls", "url": "/users/wowgirls" },
   "tags": ["hardcore", "hentai", "memes"],
   "categories": ["HD-Porn", "SFW"],
-  "pornstars": ["pig"]
+  "pornstars": ["pig"],
+  "mediaDefinitions": [
+    {
+      "defaultQuality": true,
+      "format": "hls",
+      "quality": 720,
+      "videoUrl": "https://cv.phncdn.com/videos/.../720P_1500K_.m3u8?validfrom=...&validto=...&rate=...&burst=...&ip=...&ipa=...&hash=..."
+    },
+    {
+      "defaultQuality": false,
+      "format": "hls",
+      "quality": [1080, 720, 480, 240],
+      "videoUrl": "https://cv.phncdn.com/videos/.../480P_600K_.m3u8?validfrom=...&validto=...&rate=...&burst=...&ip=...&ipa=...&hash=..."
+    },
+    {
+      "defaultQuality": false,
+      "format": "hls",
+      "quality": 240,
+      "videoUrl": "https://cv.phncdn.com/videos/.../240P_400K_.m3u8?validfrom=...&validto=...&rate=...&burst=...&ip=...&ipa=...&hash=...",
+      "remote": true
+    }
+  ]
 }
 ```
 </details>
@@ -97,8 +118,7 @@ console.log(video)
 See [DOCUMENT](/doc/DOCUMENT.md) for more information.
 
 ## Notice
-* Please note that video downloading is **no longer supported**. We are unable to provide a download link due to the changes made by PornHub. However, you can still download videos using alternative tools such as [yt-dlp](https://github.com/yt-dlp/yt-dlp).
-
+* Please note that video downloading is **no longer supported**. We are unable to provide a download link due to the changes made by PornHub. Instead, the raw `mediaDefinitions` are provided. You can try to download videos via the m3u8 information in `mediaDefinitions` or use alternative tools such as [yt-dlp](https://github.com/yt-dlp/yt-dlp). But that would be out of the scope of this project.
 
 ## License
 [MIT](LICENSE)
diff --git a/doc/Page.md b/doc/Page.md
@@ -48,12 +48,27 @@ console.log(video)
   "tags": ["hardcore", "hentai", "memes"],
   "categories": ["HD-Porn", "SFW"],
   "pornstars": [],
-  "videos": [{
-    "quality": "720",
-    "filename": "720P_1500K_161102592.mp4",
-    "extension": "mp4",
-    "url": "..."
-  }]
+  "mediaDefinitions": [
+    {
+      "defaultQuality": true,
+      "format": "hls",
+      "quality": 720,
+      "videoUrl": "https://cv.phncdn.com/videos/.../720P_1500K_.m3u8?validfrom=...&validto=...&rate=...&burst=...&ip=...&ipa=...&hash=..."
+    },
+    {
+      "defaultQuality": false,
+      "format": "hls",
+      "quality": [1080, 720, 480, 240],
+      "videoUrl": "https://cv.phncdn.com/videos/.../480P_600K_.m3u8?validfrom=...&validto=...&rate=...&burst=...&ip=...&ipa=...&hash=..."
+    },
+    {
+      "defaultQuality": false,
+      "format": "hls",
+      "quality": 240,
+      "videoUrl": "https://cv.phncdn.com/videos/.../240P_400K_.m3u8?validfrom=...&validto=...&rate=...&burst=...&ip=...&ipa=...&hash=...",
+      "remote": true
+    }
+  ]
 }
 ```
 </details>
@@ -74,7 +89,8 @@ console.log(video)
 | provider          | `object`        | Video provider/uploader                                                                                                            |
 | provider.username | `string`        | Provider's username                                                                                                                |
 | provider.url      | `string`        | Provider's account page                                                                                                            |
-| ~~videos~~ (*deprecated*) | `Array`         | List of the videos. Sorted by quality(low->hight).                                                                                 |
+| ~~videos~~ (*deprecated*) | `Array`         | List of the videos. Sorted by quality(low->hight).                                                                         |
+| mediaDefinitions  | `Array<object>` | List of the media information. Mostly m3u8 related.                                                                                |
 | tags              | `Array<string>` | example: `['couple', 'kissing']`                                                                                                   |
 | pornstars         | `Array<string>` | example: `['Bob', 'John', 'Emily]`                                                                                                 |
 | categories        | `Array<string>` | example: `['HD-Porn', 'Small-Tits']`                                                                                               |

diff --git a/src/scrapers/pages/video.ts b/src/scrapers/pages/video.ts
@@ -6,6 +6,14 @@ import { UrlParser } from '../../utils/url'
 import type { Engine } from '../../core/engine'
 import type { CheerioAPI } from 'cheerio'
 
+export interface MediaDefinition {
+    defaultQuality: boolean | number
+    format: string
+    videoUrl: string
+    quality: number | number[]
+    remote: boolean
+}
+
 export interface VideoPage {
     id: string
     url: string
@@ -29,6 +37,7 @@ export interface VideoPage {
         filename: string
         extension: string
     }>
+    mediaDefinitions: MediaDefinition[]
     provider: {
         username: string
         url: string
@@ -53,6 +62,7 @@ export async function videoPage(engine: Engine, urlOrId: string): Promise<VideoP
     return {
         id,
         url,
+        mediaDefinitions: parseMediaDefinition(html),
         ...parseByDom(html, $),
     }
 }
@@ -124,3 +134,38 @@ function parseByLdJson($: CheerioAPI) {
         }
     }
 }
+
+const mediaDefinitionRegex = /{"defaultQuality":(true|false|\d+),"format":"(\w+)","videoUrl":"(.+?)","quality":(("\d+")|(\[[\d,]*\]))(,"remote":(true|false))?}/g
+function parseMediaDefinition(html: string): MediaDefinition[] {
+    const mediaDefinitions: MediaDefinition[] = []
+
+    while (true) {
+        const match = mediaDefinitionRegex.exec(html)
+        if (!match) break
+
+        try {
+            const [, _defaultQuality, format, videoUrl, _quality, ,_qualityArray, , , _remote] = match
+            const defaultQuality = _defaultQuality === 'true'
+                ? true
+                : _defaultQuality === 'false'
+                    ? false
+                    : +_defaultQuality
+            const quality = _qualityArray ? JSON.parse(_qualityArray) as number[] : +_quality
+            const remote = _remote === 'true'
+
+            mediaDefinitions.push({
+                defaultQuality,
+                format,
+                videoUrl,
+                quality,
+                remote,
+            })
+        }
+        catch (error) {
+            console.warn(`Failed to parse media definition from input: "${match}"`)
+            console.warn(error)
+        }
+    }
+
+    return mediaDefinitions
+}
diff --git a/test/entity/info/__snapshots__/pornstar.spec.ts.snap b/test/entity/info/__snapshots__/pornstar.spec.ts.snap
@@ -16,14 +16,14 @@ Stop wasting your time - my tiny pussy is waiting for your sweet hot cum.",
   "eyeColor": "Green",
   "fakeBoobs": false,
   "featuredIn": [
-    {
-      "name": "Ultra Films",
-      "url": "/channels/ultra-films",
-    },
     {
       "name": "Wow Girls",
       "url": "/channels/wow-girls",
     },
+    {
+      "name": "Ultra Films",
+      "url": "/channels/ultra-films",
+    },
     {
       "name": "Pornhub Models",
       "url": "/channels/pornhub-models",

diff --git a/test/entity/info/video.spec.ts b/test/entity/info/video.spec.ts
@@ -18,6 +18,8 @@ describe('Video Info', () => {
         expect(result.provider?.username).toBe('Hansel Grettel')
         expect(result.provider?.url).toBe('/model/hansel-grettel')
 
+        expect(result.mediaDefinitions.length).to.be.at.least(2)
+
         expect(result.tags.length).to.be.at.least(10)
 
         expect(result.pornstars).to.be.empty

diff --git a/test/entity/list/pornstarList.spec.ts b/test/entity/list/pornstarList.spec.ts
@@ -17,6 +17,6 @@ describe('PornstarList', () => {
         expect(result.data.length).to.be.at.least(30)
 
         const first = result.data[0]
-        expect(first.name).to.equal('Lina Lynx')
+        expect(first.videoNum).to.be.at.least(3000)
     })
 })
diff --git a/test/entity/search/pornstar.spec.ts b/test/entity/search/pornstar.spec.ts
@@ -6,15 +6,15 @@ const pornhub = new PornHub()
 describe('Pornstar Search', () => {
     it('# run()', async () => {
         const result = await pornhub.searchPornstar('kelly', {
-            page: 2,
+            page: 1,
             order: 'No. of Video',
         })
 
-        expect(result.paging.current).to.equal(2)
+        expect(result.paging.current).to.equal(1)
         expect(result.paging.isEnd).to.equal(false)
 
-        expect(result.counting.from).to.equal(23)
-        expect(result.counting.to).to.equal(44)
+        expect(result.counting.from).to.equal(1)
+        expect(result.counting.to).to.equal(22)
 
         expect(result.data.length).to.equal(22)
     })

diff --git a/test/webmaster/pornstar.spec.ts b/test/webmaster/pornstar.spec.ts
@@ -7,7 +7,7 @@ describe('WebMaster getPornstars', () => {
     it('# run()', async () => {
         const result = await webMaster.getPornstars()
 
-        expect(result[0]).to.equal('033120 mpp2')
+        expect(result[0].toLowerCase()).to.equal('033120 mpp2')
         expect(result.length).to.be.at.least(10000)
     })
 })