[imdb] fix reviews

The "_ajax" URL stopped working, use the query. Save the JSON even if it wasn't parsed. -thanks @adoxa
OpenPLi · Dec 8, 2024 · db2ecd1 · db2ecd1
1 parent 6691832
commit db2ecd1
Show file tree

Hide file tree

Showing 2 changed files with 55 additions and 26 deletions.
diff --git a/imdb/po/ru.po b/imdb/po/ru.po
@@ -289,3 +289,12 @@ msgstr "Импорт из EPG"
 
 msgid "Channel Selection"
 msgstr "Селектор каналов"
+
+msgid "Spoiler"
+msgstr "Спойлер"
+
+msgid "%d out of %d found this helpful."
+msgstr "%d из %d нашли это полезным."
+
+msgid "IMDb Reviews failed"
+msgstr "IMDb - ошибка получения отзывов"
diff --git a/imdb/src/plugin.py b/imdb/src/plugin.py
@@ -327,6 +327,7 @@ def resetLabels(self):
 		self["extralabel"].setText("")
 		self.ratingstars = -1
 		self.reviews = []
+		self.reviewsJSON = None
 		self.spoilers = False
 		self.originalName = ""
 
@@ -514,35 +515,49 @@ def downloadTitle(self, title, titleId):
 
 	def gotReviews(self, response):
 		self["statusbar"].setText(_("Parsing reviews..."))
-		self.reviewsHTML = response.content
-		self.reviewsHTML = self.reviewsHTML.decode("utf8")
-
-		reviewsmask = re.compile(
-			r'(?:<span>(?P<rating>\d+)</span>.*?)?'
-			r'class="title" > (?P<title>.*?)\n.*?'
-			r'\n>(?P<author>.*?)</a></span><span class="review-date">(?P<date>.*?)</span>.*?'
-			r'(?:spoiler-warning">(?P<spoiler>.*?)</.*?)?'
-			r'"text show-more__control">(?P<review>.*?)</div>.*?'
-			r'text-muted">\s+(?P<helpful>.*?)\s+<span', re.DOTALL)
-
-		for review in reviewsmask.finditer(self.reviewsHTML):
+		self.reviewsJSON = response.content.decode("utf8")
+
+		try:
+			reviews = json.loads(self.reviewsJSON)['data']['title']['reviews']['edges']
+		except Exception as e:
+			self["statusbar"].setText(_("IMDb Reviews failed"))
+			print("[IMDB] reviews failed:", str(e))
+			self.reviewsJSON = None
+			return
+
+		for review in reviews:
+			if 'node' not in review:
+				continue
+			review = review['node']
+			try:
+				helpful = review['helpfulness']['upVotes']
+				total = helpful + review['helpfulness']['downVotes']
+				if total:
+					helpful = _("%d out of %d found this helpful.") % (helpful, total)
+				else:
+					helpful = ""
+			except:
+				helpful = ""
 			self.reviews.append({
-				'rating': review.group('rating'),
-				'title': html2text(review.group('title')),
-				'author': html2text(review.group('author')),
-				'date': html2text(review.group('date')),
-				'spoiler': html2text(review.group('spoiler') or ""),
-				'review': html2text(review.group('review')),
-				'helpful': html2text(review.group('helpful'))
+				'rating': str(get(review, 'authorRating')),
+				'title': html2text(get(review, ('summary', 'originalText'))),
+				'author': html2text(get(review, ('author', 'nickName'))),
+				'date': get(review, 'submissionDate'),
+				'spoiler': get(review, 'spoiler') and self.spoiler_i18n,
+				'review': html2text(get(review, ('text', 'originalText', 'plaidHtml'))),
+				'helpful': helpful
 			})
 		self["statusbar"].setText(_("IMDb Reviews parsed"))
 		self.showExtras(reviews=True)
 
 	def downloadReviews(self):
 		self["statusbar"].setText(_("Downloading reviews..."))
-		fetchurl = "https://www.imdb.com/title/" + self.titleId + "/reviews/_ajax"
-#       print("[IMDB] downloadReviews()", fetchurl)
-		download = getPage(fetchurl, cookies=self.cookie)
+		params = {
+			"operationName": 'TitleReviewsRefine',
+			"variables": '{"const":"%s","first":25}' % self.titleId,
+			"extensions": '{"persistedQuery":{"sha256Hash":"89aff4cd7503e060ff1dd5aba91885d8bac0f7a21aa1e1f781848a786a5bdc19","version":1}}'
+		}
+		download = getPage("https://caching.graphql.imdb.com/", params=params, headers={"content-type": "application/json"}, cookies=self.cookie)
 		download.addCallback(self.gotReviews).addErrback(self.http_failed)
 
 	def showDetails(self):
@@ -606,8 +621,9 @@ def showExtras(self, synopsis=False, reviews=False):
 				if self.spoilers or not review['spoiler']:
 					reviews.append(review['review'])
 					reviews.append("")
-				reviews.append(review['helpful'])
-				reviews.append("")
+				if review['helpful']:
+					reviews.append(review['helpful'])
+					reviews.append("")
 				reviews.append("-" * 72)
 				reviews.append("")
 			self.reviewsTxt = "\n".join(reviews[:-3])
@@ -696,8 +712,11 @@ def saveHtmlDetails(self):
 				open(isave + ".html", 'w').write(self.html)
 				if self.json:
 					open(isave + ".json", 'w').write(self.json)
-				if self.reviews:
-					open(isave + "-reviews.html", 'w').write(self.reviewsHTML)
+				try:
+					if self.reviewsJSON:
+						open(isave + "-reviews.json", 'w').write(self.reviewsJSON)
+				except:
+					pass
 				try:
 					copy("/tmp/poster.jpg", isave + ".jpg")
 				except:
@@ -1082,6 +1101,7 @@ def makedate(date):
 				'sound': get(i18n, 'title_main_techspec_soundmix'),
 				'aspect': get(i18n, 'title_main_techspec_aspectratio'),
 			}
+			self.spoiler_i18n = get(i18n, 'common_label_spoiler', _("Spoiler"))
 
 			self.generalinfos = {
 				'director': ", ".join(get(name, ('name', 'nameText', 'text')) for name in get(main, ('directors', 'credits'))),