From 0c268c304714186b7ead5bb9cf9c89a151050631 Mon Sep 17 00:00:00 2001 From: syeopite Date: Tue, 12 Dec 2023 17:23:51 -0800 Subject: [PATCH 01/12] Update npf-renderer to 0.12.0 --- assets/css/blog.css | 3 +++ requirements.txt | 4 +++- src/helpers/helpers.py | 2 ++ src/routes/media.py | 17 +++++++++++++++-- src/server.py | 6 +++++- 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/assets/css/blog.css b/assets/css/blog.css index a1ef651..535557a 100644 --- a/assets/css/blog.css +++ b/assets/css/blog.css @@ -18,6 +18,9 @@ } #banner { + width: 100%; + height: 100%; + object-fit: cover; border-radius: 10px 10px 0px 0px; } diff --git a/requirements.txt b/requirements.txt index 75b28e7..9569c82 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,10 +10,11 @@ frozenlist==1.4.1 html5tagger==1.3.0 httptools==0.6.1 idna==3.6 +intervaltree==3.1.0 Jinja2==3.1.2 MarkupSafe==2.1.4 multidict==6.0.5 -npf-renderer==0.11.1 +npf-renderer==0.12.0 orjson==3.8.0 pycares==4.4.0 pycparser==2.21 @@ -21,6 +22,7 @@ PyYAML==6.0.1 sanic==23.3.0 sanic-ext==23.6.0 sanic-routing==23.6.0 +sortedcontainers==2.4.0 tracerite==1.1.1 ujson==5.9.0 uvloop==0.19.0 diff --git a/src/helpers/helpers.py b/src/helpers/helpers.py index 2ef2464..dd88bb3 100644 --- a/src/helpers/helpers.py +++ b/src/helpers/helpers.py @@ -34,6 +34,8 @@ def url_handler(raw_url): return f"/tblr/media/44{url.path}" elif hostname.endswith("static.tumblr.com"): return f"/tblr/static{url.path}" + elif hostname.startswith("va.media"): + return f"/tblr/media/va{url.path}" else: # Check for subdomain blog sub_domains = hostname.split(".") diff --git a/src/routes/media.py b/src/routes/media.py index 03a2d55..70d7135 100644 --- a/src/routes/media.py +++ b/src/routes/media.py @@ -1,12 +1,13 @@ import sanic +import aiohttp from ..helpers import exceptions media = sanic.Blueprint("TumblrMedia", url_prefix="/tblr") -async def get_media(request, client, path_to_request): - async with client.get(f"/{path_to_request}") as tumblr_response: +async def get_media(request, client : aiohttp.ClientSession, path_to_request, additional_headers = None): + async with client.get(f"/{path_to_request}", headers=additional_headers) as tumblr_response: # Sanitize the headers given by Tumblr priviblur_response_headers = {} for header_key, header_value in tumblr_response.headers.items(): @@ -28,6 +29,8 @@ async def get_media(request, client, path_to_request): async for chunk in tumblr_response.content.iter_any(): await priviblur_response.send(chunk) + await priviblur_response.eof() + @media.get(r"/media/64/") async def _64_media(request: sanic.Request, path: str): @@ -47,6 +50,16 @@ async def _44_media(request: sanic.Request, path: str): return await get_media(request, request.app.ctx.Media44Client, path) +@media.get(r"/media/va/") +async def _va_media(request: sanic.Request, path: str): + """Proxies the requested media from va.media.tumblr.com""" + additional_headers={ + "accept": "video/webm,video/ogg,video/*;q=0.9," \ + "application/ogg;q=0.7,audio/*;q=0.6,*/*;q=0.5" + } + return await get_media(request, request.app.ctx.MediaVaClient, path, additional_headers=additional_headers) + + @media.get(r"/assets/") async def _tb_assets(request: sanic.Request, path: str): """Proxies the requested media from assets.tumblr.com""" diff --git a/src/server.py b/src/server.py index 6ba8922..c1289f0 100644 --- a/src/server.py +++ b/src/server.py @@ -105,6 +105,10 @@ def create_image_client(url, timeout): "https://44.media.tumblr.com", priviblur_backend.image_response_timeout ) + app.ctx.MediaVaClient = create_image_client( + "https://va.media.tumblr.com", priviblur_backend.image_response_timeout + ) + app.ctx.TumblrAssetClient = create_image_client( "https://assets.tumblr.com", priviblur_backend.image_response_timeout ) @@ -137,7 +141,7 @@ def create_image_client(url, timeout): app.ext.environment.globals["format_npf"] = functools.partial( format_npf, url_handler=helpers.url_handler, - skip_cropped_images=True + forbid_external_iframes=True, ) From 88e791df4a660327ebc6a040a5dec01fe3f2933e Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 28 Jan 2024 20:40:30 -0800 Subject: [PATCH 02/12] Add route to fetch media from a.tumblr.com --- src/helpers/helpers.py | 2 ++ src/routes/media.py | 9 +++++++++ src/server.py | 4 ++++ 3 files changed, 15 insertions(+) diff --git a/src/helpers/helpers.py b/src/helpers/helpers.py index dd88bb3..307f5ec 100644 --- a/src/helpers/helpers.py +++ b/src/helpers/helpers.py @@ -36,6 +36,8 @@ def url_handler(raw_url): return f"/tblr/static{url.path}" elif hostname.startswith("va.media"): return f"/tblr/media/va{url.path}" + elif hostname.startswith("a."): + return f"/tblr/a{url.path}" else: # Check for subdomain blog sub_domains = hostname.split(".") diff --git a/src/routes/media.py b/src/routes/media.py index 70d7135..5992c80 100644 --- a/src/routes/media.py +++ b/src/routes/media.py @@ -60,6 +60,15 @@ async def _va_media(request: sanic.Request, path: str): return await get_media(request, request.app.ctx.MediaVaClient, path, additional_headers=additional_headers) +@media.get(r"/a/") +async def _a_media(request: sanic.Request, path: str): + """Proxies the requested media from va.media.tumblr.com""" + additional_headers={ + "accept": "audio/webm,audio/ogg,audio/wav,audio/*;q=0.9,application/ogg;q=0.7,video/*;q=0.6,*/*;q=0.5" + } + return await get_media(request, request.app.ctx.AudioClient, path, additional_headers=additional_headers) + + @media.get(r"/assets/") async def _tb_assets(request: sanic.Request, path: str): """Proxies the requested media from assets.tumblr.com""" diff --git a/src/server.py b/src/server.py index c1289f0..cd0c107 100644 --- a/src/server.py +++ b/src/server.py @@ -109,6 +109,10 @@ def create_image_client(url, timeout): "https://va.media.tumblr.com", priviblur_backend.image_response_timeout ) + app.ctx.AudioClient = create_image_client( + "https://a.tumblr.com", priviblur_backend.image_response_timeout + ) + app.ctx.TumblrAssetClient = create_image_client( "https://assets.tumblr.com", priviblur_backend.image_response_timeout ) From 808d53b541a323818fe163f269be19b8ece458c3 Mon Sep 17 00:00:00 2001 From: syeopite Date: Wed, 31 Jan 2024 22:47:32 -0800 Subject: [PATCH 03/12] Add support for polls --- src/helpers/ext_npf_renderer.py | 125 ++++++++++++++++++++++++++++ src/helpers/helpers.py | 9 +- src/priviblur_extractor/api/api.py | 17 +++- src/routes/__init__.py | 5 +- src/routes/api/__init__.py | 7 ++ src/routes/api/v1/__init__.py | 8 ++ src/routes/api/v1/misc.py | 13 +++ src/routes/explore.py | 2 - src/routes/search.py | 1 - src/routes/tagged.py | 2 - src/server.py | 11 +-- src/templates/components/post.jinja | 4 +- 12 files changed, 185 insertions(+), 19 deletions(-) create mode 100644 src/helpers/ext_npf_renderer.py create mode 100644 src/routes/api/__init__.py create mode 100644 src/routes/api/v1/__init__.py create mode 100644 src/routes/api/v1/misc.py diff --git a/src/helpers/ext_npf_renderer.py b/src/helpers/ext_npf_renderer.py new file mode 100644 index 0000000..d841ea8 --- /dev/null +++ b/src/helpers/ext_npf_renderer.py @@ -0,0 +1,125 @@ +"""Extensions to npf-renderer to allow asynchronous code and some other custom styling""" + +import dominate +import npf_renderer + +from .helpers import url_handler + + +class NPFParser(npf_renderer.parse.Parser): + def __init__(self, content, poll_callback=None): + super().__init__(content, poll_callback) + + async def _parse_poll_block(self): + poll_id = self.current.get("clientId") or self.current.get("client_id") + if poll_id is None: + raise ValueError("Invalid poll ID") + + question = self.current["question"] + + answers = {} + for raw_ans in self.current["answers"]: + answer_id = raw_ans.get("clientId") or raw_ans.get("client_id") + answer_text = raw_ans.get("answerText") or raw_ans.get("answer_text") + + if answer_id is None or answer_text is None: + raise ValueError("Invalid poll answer") + + answers[answer_id] = answer_text + + votes = None + total_votes = None + + if self.poll_result_callback: + callback_response = await self.poll_result_callback(poll_id) + + #{answer_id: vote_count} + raw_results = callback_response["results"].items() + processed_results = sorted(raw_results, key=lambda item: -item[1]) + + votes_dict = {} + total_votes = 0 + + for index, results in enumerate(processed_results): + vote_count = results[1] + total_votes += vote_count + + if index == 0: + votes_dict[results[0]] = npf_renderer.objects.poll_block.PollResult(is_winner=True, vote_count=vote_count) + else: + votes_dict[results[0]] = npf_renderer.objects.poll_block.PollResult(is_winner=False, vote_count=vote_count) + + votes = npf_renderer.objects.poll_block.PollResults( + timestamp=callback_response["timestamp"], + results=votes_dict + ) + + creation_timestamp = self.current["timestamp"] + expires_after = self.current["settings"]["expireAfter"] + + return npf_renderer.objects.poll_block.PollBlock( + poll_id=poll_id, + question=question, + answers=answers, + + creation_timestamp=int(creation_timestamp), + expires_after=int(expires_after), + + votes=votes, + total_votes=total_votes, + ) + + async def __parse_block(self): + """Parses a content block and appends the result to self.parsed_result + + Works by routing specific content types to corresponding parse methods + """ + + match self.current["type"]: + case "text": + block = self._parse_text() + case "image": + block = self._parse_image_block() + case "link": + block = self._parse_link_block() + case "audio": + block = self._parse_audio_block() + case "video": + block = self._parse_video_block() + case "poll": + block = await self._parse_poll_block() + case _: + block = unsupported.Unsupported(self.current["type"]) + + self.parsed_result.append(block) + + async def parse(self): + """Begins the parsing chain and returns the final list of parsed objects""" + while self.next(): + await self.__parse_block() + + return self.parsed_result + + +async def format_npf(contents, layouts=None, *, poll_callback=None): + try: + contents = await NPFParser(contents, poll_callback=poll_callback).parse() + if layouts: + layouts = npf_renderer.parse.LayoutParser(layouts).parse() + + contains_render_errors = False + + formatted = npf_renderer.format.Formatter(contents, layouts, url_handler=url_handler, + forbid_external_iframes=True, + ).format() + + except npf_renderer.exceptions.RenderErrorDisclaimerError as e: + contains_render_errors = True + formatted = e.rendered_result + assert formatted is not None + except Exception as e: + + formatted = dominate.tags.div(cls="post-body has-error") + contains_render_errors = True + + return contains_render_errors, formatted.render(pretty=False) diff --git a/src/helpers/helpers.py b/src/helpers/helpers.py index 307f5ec..6110118 100644 --- a/src/helpers/helpers.py +++ b/src/helpers/helpers.py @@ -99,4 +99,11 @@ def translate(language, id, number=None, substitution=None): translated = translated.format(substitution) return translated - \ No newline at end of file + + +async def create_poll_callback(tumblr_api, blog, post_id): + async def poll_callable(poll_id): + initial_results = await tumblr_api.poll_results(blog, post_id, poll_id) + return initial_results["response"] + + return poll_callable \ No newline at end of file diff --git a/src/priviblur_extractor/api/api.py b/src/priviblur_extractor/api/api.py index dd574b3..cc42fa0 100644 --- a/src/priviblur_extractor/api/api.py +++ b/src/priviblur_extractor/api/api.py @@ -171,7 +171,7 @@ async def explore_today(self, *, continuation: Optional[str] = None, fields: str url_parameters["cursor"] = continuation return await self._get_json("explore/home/today", url_parameters) - + async def explore_post(self, post_type: rconf.ExplorePostTypeFilters, *, continuation: Optional[str] = None, reblog_info: bool = True, fields: str = rconf.EXPLORE_BLOG_INFO_FIELDS,): @@ -312,4 +312,17 @@ async def blog_post(self, blog_name, post_id): return await self._get_json( f"blog/{urllib.parse.quote(blog_name, safe='')}/posts/{post_id}/permalink", url_params={"fields[blogs]": rconf.POST_BLOG_INFO_FIELDS, "reblog_info": True} - ) \ No newline at end of file + ) + + async def poll_results(self, blog_name, post_id, poll_id): + """Requests the /polls////results endpoint + + Parameters: + blog_name: the blog the post is from + post_id: the id of the post + poll_id: the id of the poll + """ + + return await self._get_json( + f"polls/{urllib.parse.quote(blog_name, safe='')}/{post_id}/{poll_id}/results", + ) diff --git a/src/routes/__init__.py b/src/routes/__init__.py index cef4d0c..18eb02e 100644 --- a/src/routes/__init__.py +++ b/src/routes/__init__.py @@ -1,4 +1,4 @@ -from . import explore, media, search, tagged, blogs, assets, priviblur, miscellaneous +from . import explore, media, search, tagged, blogs, assets, priviblur, miscellaneous, api BLUEPRINTS = ( explore.explore, @@ -8,5 +8,6 @@ blogs.blogs, assets.assets, priviblur.priviblur, - miscellaneous.miscellaneous + miscellaneous.miscellaneous, + api.api ) diff --git a/src/routes/api/__init__.py b/src/routes/api/__init__.py new file mode 100644 index 0000000..1344120 --- /dev/null +++ b/src/routes/api/__init__.py @@ -0,0 +1,7 @@ +from sanic import Blueprint +from .v1 import v1 + +api = Blueprint.group( + v1, + url_prefix="/api" +) \ No newline at end of file diff --git a/src/routes/api/v1/__init__.py b/src/routes/api/v1/__init__.py new file mode 100644 index 0000000..7ec8fea --- /dev/null +++ b/src/routes/api/v1/__init__.py @@ -0,0 +1,8 @@ +from sanic import Blueprint + +from .misc import misc + +v1 = Blueprint.group( + misc, + url_prefix="/v1" +) \ No newline at end of file diff --git a/src/routes/api/v1/misc.py b/src/routes/api/v1/misc.py new file mode 100644 index 0000000..7553110 --- /dev/null +++ b/src/routes/api/v1/misc.py @@ -0,0 +1,13 @@ +import urllib.parse + +import sanic + +misc = sanic.Blueprint("api_misc", url_prefix="/") + +@misc.get("/poll////results") +async def poll_results(request, blog : str, post_id : int, poll_id : int): + blog = urllib.parse.unquote(blog) + poll_id = urllib.parse.unquote(poll_id) + + initial_results = await request.app.ctx.TumblrAPI.poll_results(blog, post_id, poll_id) + return sanic.response.json(initial_results) \ No newline at end of file diff --git a/src/routes/explore.py b/src/routes/explore.py index 93ea3a7..a7acb49 100644 --- a/src/routes/explore.py +++ b/src/routes/explore.py @@ -4,8 +4,6 @@ import sanic import sanic_ext -import npf_renderer - from .. import priviblur_extractor explore = sanic.Blueprint("explore", url_prefix="/explore") diff --git a/src/routes/search.py b/src/routes/search.py index 1c07f55..731b06f 100644 --- a/src/routes/search.py +++ b/src/routes/search.py @@ -4,7 +4,6 @@ import sanic import sanic_ext -import npf_renderer from .. import priviblur_extractor search = sanic.Blueprint("search", url_prefix="/search") diff --git a/src/routes/tagged.py b/src/routes/tagged.py index 3d03525..f9a05c9 100644 --- a/src/routes/tagged.py +++ b/src/routes/tagged.py @@ -4,8 +4,6 @@ import sanic import sanic_ext -import npf_renderer - from .. import priviblur_extractor tagged = sanic.Blueprint("tagged", url_prefix="/tagged") diff --git a/src/server.py b/src/server.py index cd0c107..0eb5b9b 100644 --- a/src/server.py +++ b/src/server.py @@ -14,12 +14,12 @@ from sanic import Sanic import babel.numbers import babel.dates -from npf_renderer import VERSION as NPF_RENDERER_VERSION, format_npf +from npf_renderer import VERSION as NPF_RENDERER_VERSION from . import routes, priviblur_extractor from . import priviblur_extractor from .config import load_config -from .helpers import setup_logging, helpers, error_handlers, exceptions +from .helpers import setup_logging, helpers, error_handlers, exceptions, ext_npf_renderer from .version import VERSION, CURRENT_COMMIT @@ -142,11 +142,8 @@ def create_image_client(url, timeout): app.ext.environment.globals["translate"] = helpers.translate app.ext.environment.globals["url_handler"] = helpers.url_handler - app.ext.environment.globals["format_npf"] = functools.partial( - format_npf, - url_handler=helpers.url_handler, - forbid_external_iframes=True, - ) + app.ext.environment.globals["format_npf"] = ext_npf_renderer.format_npf + app.ext.environment.globals["create_poll_callback"] = helpers.create_poll_callback @app.listener("main_process_start") diff --git a/src/templates/components/post.jinja b/src/templates/components/post.jinja index 3ff97e5..89d7b55 100644 --- a/src/templates/components/post.jinja +++ b/src/templates/components/post.jinja @@ -1,7 +1,7 @@ {% from 'macros/post_header.jinja' import create_post_header %}
{%- if element.content -%} - {% set contains_errors, content_tag = format_npf(element.content, element.layout) -%} + {% set contains_errors, content_tag = format_npf(element.content, element.layout, poll_callback=create_poll_callback(request.app.ctx.TumblrAPI, element.blog.name, element.id)) -%} {%- endif -%} {{ create_post_header(element) }} @@ -14,7 +14,7 @@ {%- for trail_element in element.trail -%}
{{create_post_header(trail_element)}} - {% set trail_contains_errors, trail_content_tag = format_npf(trail_element.content, trail_element.layout) -%} + {% set trail_contains_errors, trail_content_tag = format_npf(trail_element.content, trail_element.layout, poll_callback=create_poll_callback(request.app.ctx.TumblrAPI, element.blog.name, element.id)) -%} {{- trail_content_tag -}}
{%- endfor -%} From cd8f538d5c7f25a1cd8b7a34c48dc724ef583943 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sat, 3 Feb 2024 23:45:31 -0800 Subject: [PATCH 04/12] Use JavaScript (with fallback) to populate polls In server-side rendering, as posts are rendered one by one sequentially the results for polls must also be fetched sequentially. This results in a lengthy load time as Priviblur has to send dozens of requests one-by-one before the page can be returned. By populating the polls client side, this lengthy fetch process can be avoided entirely. For JS-disabled users, a fetch_polls URL parameter is provided as a fallback to allow for server-side fetching of poll results. However, due to the aforementioned reasons above, this is only available on individual posts. --- assets/css/post-layout.css | 5 ++ assets/js/post.js | 82 +++++++++++++++++++++++++++++ src/helpers/ext_npf_renderer.py | 53 +++++++++++++++++-- src/routes/api/v1/misc.py | 2 +- src/routes/blogs.py | 23 +++++--- src/templates/base.jinja | 1 + src/templates/components/post.jinja | 41 +++++++++++++-- 7 files changed, 194 insertions(+), 13 deletions(-) create mode 100644 assets/js/post.js diff --git a/assets/css/post-layout.css b/assets/css/post-layout.css index 9dba053..8303aec 100644 --- a/assets/css/post-layout.css +++ b/assets/css/post-layout.css @@ -84,4 +84,9 @@ a.inline-link, a.inline-mention { color: #6b7280; text-transform: uppercase; font-size: 12px; +} + +a.toggle-poll-results { + color: unset; + text-decoration: underline; } \ No newline at end of file diff --git a/assets/js/post.js b/assets/js/post.js new file mode 100644 index 0000000..eff2302 --- /dev/null +++ b/assets/js/post.js @@ -0,0 +1,82 @@ +"use strict"; + +function requestPollResults(poll_element, pollId) { + return new Promise(function (resolve, reject) { + let post = poll_element.closest(".post"); + const blogName = post.getElementsByClassName("blog-name")[0].innerHTML; + const postId = post.dataset.postId; + + const pollResultsFetch = fetch(`/api/v1/poll/${blogName}/${postId}/${pollId}/results`); + + pollResultsFetch.then((results) => { + return results.json(); + }).then((parsed_results) => { + return resolve(parsed_results); + }); + }) +} + +function fill_poll_results(poll_element, results) { + const sorted_poll_results = Object.entries(results.response.results).sort((a,b) => (a[1]-b[1])).reverse(); + + // Find total votes first + let total_votes = 0; + for (let votes of sorted_poll_results) { + total_votes += votes[1]; + }; + + for (let i = 0; i < sorted_poll_results.length; ++i) { + const [answer_id, answer_votes] = sorted_poll_results[i]; + const choiceElement = document.getElementById(answer_id); + + const numericalVoteProportion = answer_votes/total_votes + + const voteProportionElement = document.createElement("span"); + voteProportionElement.classList.add("vote-proportion"); + voteProportionElement["style"] = `width: ${((numericalVoteProportion) * 100).toFixed(3)}%;`; + + const voteCountElement = document.createElement("span"); + voteCountElement.classList.add("vote-count"); + + // A greater rounding precision is needed here + if ((Math.round((numericalVoteProportion) * 10000)/10000) > 0.001) { + voteCountElement.innerHTML = new Intl.NumberFormat("en-US", {style: "percent", maximumSignificantDigits: 3}).format(Math.round((numericalVoteProportion) * 1000)/1000); + } else { + voteCountElement.innerHTML = "< 0.1%"; + } + + if (i == 0) { + choiceElement.classList.add("poll-winner"); + } + + choiceElement.appendChild(voteProportionElement); + choiceElement.appendChild(voteCountElement); + } + + const totalVotesElement = document.createElement("p") + + if (poll_element.classList.contains("expired")) { + totalVotesElement.innerHTML = `Final result from ${total_votes} votes` + } else { + totalVotesElement.innerHTML = `${total_votes} votes` + } + + const poll_footer = poll_element.getElementsByTagName("footer") + poll_footer[0].insertBefore(totalVotesElement, poll_footer[0].firstChild) +} + +const pollBlocks = document.getElementsByClassName("poll-block"); + + +function populate_polls() { + for (let poll of pollBlocks) { + if (poll.classList.contains("populated")) { + continue; + } + + requestPollResults(poll, poll.dataset.pollId).then((answers) => {fill_poll_results(poll, answers)}) + } +}; + +// TODO lazy load polls +populate_polls(); diff --git a/src/helpers/ext_npf_renderer.py b/src/helpers/ext_npf_renderer.py index d841ea8..229571d 100644 --- a/src/helpers/ext_npf_renderer.py +++ b/src/helpers/ext_npf_renderer.py @@ -101,7 +101,51 @@ async def parse(self): return self.parsed_result -async def format_npf(contents, layouts=None, *, poll_callback=None): +class NPFFormatter(npf_renderer.format.Formatter): + def __init__(self, content, layout=None, blog_name=None, post_id=None, *, url_handler=None, forbid_external_iframes=False): + super().__init__(content, layout, url_handler=url_handler, forbid_external_iframes=forbid_external_iframes) + + # We store the blog and post ID as to be able to render a link to + # fetch poll results for JS disabled users + self.blog_name = blog_name + self.post_id = post_id + + def _format_poll(self, block): + poll_html = super()._format_poll(block) + poll_html["data-poll-id"] = block.poll_id + + poll_body = poll_html[1] + for index, answer_id in enumerate(block.answers.keys()): + poll_body[index]["id"] = answer_id + + if (self.blog_name and self.post_id) and not block.votes: + poll_footer = poll_html[2] + poll_footer.add( + dominate.tags.noscript( + dominate.tags.a( + "See Results", + href=f"/{self.blog_name}/{self.post_id}?fetch_polls=true", + cls="toggle-poll-results" + ) + ) + ) + + return poll_html + + +async def format_npf(contents, layouts=None, blog_name=None, post_id=None,*, poll_callback=None): + """Wrapper around npf_renderer.format_npf for extra functionalities + + - Replaces internal Parser and Formatter with the modified variants above + - Accepts extra arguments to add additional details to formatted results + - Automatically sets Priviblur-specific rendering arguments + + Arguments (new): + blog_name: + Name of the blog the post comes from. This is used to render links to the parent post + post_id: + Unique ID of the post. This is used to render links to the parent post + """ try: contents = await NPFParser(contents, poll_callback=poll_callback).parse() if layouts: @@ -109,7 +153,11 @@ async def format_npf(contents, layouts=None, *, poll_callback=None): contains_render_errors = False - formatted = npf_renderer.format.Formatter(contents, layouts, url_handler=url_handler, + formatted = NPFFormatter( + contents, layouts, + blog_name=blog_name, + post_id=post_id, + url_handler=url_handler, forbid_external_iframes=True, ).format() @@ -118,7 +166,6 @@ async def format_npf(contents, layouts=None, *, poll_callback=None): formatted = e.rendered_result assert formatted is not None except Exception as e: - formatted = dominate.tags.div(cls="post-body has-error") contains_render_errors = True diff --git a/src/routes/api/v1/misc.py b/src/routes/api/v1/misc.py index 7553110..faf346e 100644 --- a/src/routes/api/v1/misc.py +++ b/src/routes/api/v1/misc.py @@ -10,4 +10,4 @@ async def poll_results(request, blog : str, post_id : int, poll_id : int): poll_id = urllib.parse.unquote(poll_id) initial_results = await request.app.ctx.TumblrAPI.poll_results(blog, post_id, poll_id) - return sanic.response.json(initial_results) \ No newline at end of file + return sanic.response.json(initial_results, headers={"Cache-Control": "max-age=600, immutable"}) \ No newline at end of file diff --git a/src/routes/blogs.py b/src/routes/blogs.py index edab9c4..5f4b428 100644 --- a/src/routes/blogs.py +++ b/src/routes/blogs.py @@ -9,13 +9,14 @@ blogs = sanic.Blueprint("blogs", url_prefix="/") -async def render_blog_post(app, blog, post): +async def render_blog_post(app, blog, post, request_poll_data = False): return await sanic_ext.render( "blog_post.jinja", context={ "app": app, "blog": blog, "element": post, + "request_poll_data" : request_poll_data } ) @@ -75,13 +76,18 @@ async def _blog_post_no_slug(request: sanic.Request, blog: str, post_id: str): post = timeline.elements[0] if post.slug: - return sanic.redirect(request.app.url_for("blogs._blog_post", blog=blog, post_id=post_id, slug=post.slug)) + return sanic.redirect(request.app.url_for("blogs._blog_post", blog=blog, post_id=post_id, slug=post.slug, **request.args)) else: # Fetch blog info and some posts from before this post initial_blog_results = await request.app.ctx.TumblrAPI.blog_posts(blog, before_id=post.id) blog_info = priviblur_extractor.parse_container(initial_blog_results) - return await render_blog_post(request.app, blog_info, post) + if request.args.get("fetch_polls") in {1, "true"}: + fetch_poll_results = True + else: + fetch_poll_results = False + + return await render_blog_post(request.app, blog_info, post, fetch_poll_results) @blogs.get("//") @@ -97,15 +103,20 @@ async def _blog_post(request: sanic.Request, blog: str, post_id: str, slug: str) if post.slug != slug: # Unless of course the slug is empty. In that case we'll remove the slug. if post.slug: - return sanic.redirect(request.app.url_for("blogs._blog_post", blog=blog, post_id=post_id, slug=post.slug)) + return sanic.redirect(request.app.url_for("blogs._blog_post", blog=blog, post_id=post_id, slug=post.slug, **request.args)) else: - return sanic.redirect(request.app.url_for("blogs._blog_post_no_slug", blog=blog, post_id=post_id)) + return sanic.redirect(request.app.url_for("blogs._blog_post_no_slug", blog=blog, post_id=post_id, **request.args)) else: # Fetch blog info and some posts from before this post initial_blog_results = await request.app.ctx.TumblrAPI.blog_posts(blog, before_id=post.id) blog_info = priviblur_extractor.parse_container(initial_blog_results) - return await render_blog_post(request.app, blog_info, post) + if request.args.get("fetch_polls") in ("1", "true"): + fetch_poll_results = True + else: + fetch_poll_results = False + + return await render_blog_post(request.app, blog_info, post, fetch_poll_results) # Redirects for /post/... diff --git a/src/templates/base.jinja b/src/templates/base.jinja index bade2d5..aceb6a5 100644 --- a/src/templates/base.jinja +++ b/src/templates/base.jinja @@ -10,6 +10,7 @@ + {%- block head -%} {%- endblock -%} diff --git a/src/templates/components/post.jinja b/src/templates/components/post.jinja index 89d7b55..502bbd7 100644 --- a/src/templates/components/post.jinja +++ b/src/templates/components/post.jinja @@ -1,7 +1,24 @@ {% from 'macros/post_header.jinja' import create_post_header %} -
+
{%- if element.content -%} - {% set contains_errors, content_tag = format_npf(element.content, element.layout, poll_callback=create_poll_callback(request.app.ctx.TumblrAPI, element.blog.name, element.id)) -%} + {%- if request_poll_data -%} + {%- set contains_errors, content_tag = format_npf( + element.content, + element.layout, + poll_callback=create_poll_callback( + request.app.ctx.TumblrAPI, element.blog.name, element.id + ), + ) + -%} + {%- else -%} + {%- set contains_errors, content_tag = format_npf( + element.content, + element.layout, + element.blog.name, + element.id, + ) + -%} + {% endif %} {%- endif -%} {{ create_post_header(element) }} @@ -14,7 +31,25 @@ {%- for trail_element in element.trail -%}
{{create_post_header(trail_element)}} - {% set trail_contains_errors, trail_content_tag = format_npf(trail_element.content, trail_element.layout, poll_callback=create_poll_callback(request.app.ctx.TumblrAPI, element.blog.name, element.id)) -%} + {%- if request_poll_data -%} + {%- set trail_contains_errors, trail_content_tag = format_npf( + trail_element.content, + trail_element.layout, + poll_callback=create_poll_callback( + request.app.ctx.TumblrAPI, trail_element.blog.name, trail_element.id + ), + ) + -%} + + {%- else -%} + {%- set trail_contains_errors, trail_content_tag = format_npf( + trail_element.content, + trail_element.layout, + trail_element.blog.name, + trail_element.id, + ) + -%} + {% endif %} {{- trail_content_tag -}}
{%- endfor -%} From e34d280df3a7d8c4746d148c10497a1863e3d8ce Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 4 Feb 2024 00:05:46 -0800 Subject: [PATCH 05/12] Fix JS poll rendering error on duplicate polls Prior to this commit, it was naively assumed that a poll can only appear once on a page and as such each poll choice was assigned an HTML id of their given answer ID. However, due to the presence of reblogs, polls can in fact appear more than once on a single page and as such the logic in the poll filling JS fails due to the presence of duplicate IDs. --- assets/js/post.js | 10 +++++++++- src/helpers/ext_npf_renderer.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/assets/js/post.js b/assets/js/post.js index eff2302..3707cd9 100644 --- a/assets/js/post.js +++ b/assets/js/post.js @@ -25,9 +25,17 @@ function fill_poll_results(poll_element, results) { total_votes += votes[1]; }; + // Create mapping of answer-id to answer choice element + const answerIdToChoiceElement = {} + const pollBody = poll_element.getElementsByClassName("poll-body")[0] + + for (let choiceElement of pollBody.children) { + answerIdToChoiceElement[choiceElement.dataset.answerId] = choiceElement + } + for (let i = 0; i < sorted_poll_results.length; ++i) { const [answer_id, answer_votes] = sorted_poll_results[i]; - const choiceElement = document.getElementById(answer_id); + const choiceElement = answerIdToChoiceElement[answer_id] const numericalVoteProportion = answer_votes/total_votes diff --git a/src/helpers/ext_npf_renderer.py b/src/helpers/ext_npf_renderer.py index 229571d..8e4ff4e 100644 --- a/src/helpers/ext_npf_renderer.py +++ b/src/helpers/ext_npf_renderer.py @@ -116,7 +116,7 @@ def _format_poll(self, block): poll_body = poll_html[1] for index, answer_id in enumerate(block.answers.keys()): - poll_body[index]["id"] = answer_id + poll_body[index]["data-answer-id"] = answer_id if (self.blog_name and self.post_id) and not block.votes: poll_footer = poll_html[2] From 13b3480372b75b071bbbea7137f42c460da33315 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 4 Feb 2024 00:18:55 -0800 Subject: [PATCH 06/12] Fix representation of 0% entries on polls --- assets/js/post.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/assets/js/post.js b/assets/js/post.js index 3707cd9..d8417a2 100644 --- a/assets/js/post.js +++ b/assets/js/post.js @@ -37,7 +37,13 @@ function fill_poll_results(poll_element, results) { const [answer_id, answer_votes] = sorted_poll_results[i]; const choiceElement = answerIdToChoiceElement[answer_id] - const numericalVoteProportion = answer_votes/total_votes + let numericalVoteProportion + + if (answer_votes == 0 || total_votes == 0) { + numericalVoteProportion = 0; + } else { + numericalVoteProportion = answer_votes/total_votes; + } const voteProportionElement = document.createElement("span"); voteProportionElement.classList.add("vote-proportion"); @@ -47,7 +53,8 @@ function fill_poll_results(poll_element, results) { voteCountElement.classList.add("vote-count"); // A greater rounding precision is needed here - if ((Math.round((numericalVoteProportion) * 10000)/10000) > 0.001) { + const comparison = Math.round((numericalVoteProportion) * 10000)/10000 + if ((comparison > 0.001) || comparison == 0) { voteCountElement.innerHTML = new Intl.NumberFormat("en-US", {style: "percent", maximumSignificantDigits: 3}).format(Math.round((numericalVoteProportion) * 1000)/1000); } else { voteCountElement.innerHTML = "< 0.1%"; From 89fe062d4143ec69381f5dd2a217e4d4230e7172 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 4 Feb 2024 00:44:20 -0800 Subject: [PATCH 07/12] Add populated class to filled polls --- assets/js/post.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/assets/js/post.js b/assets/js/post.js index d8417a2..678a8eb 100644 --- a/assets/js/post.js +++ b/assets/js/post.js @@ -76,8 +76,10 @@ function fill_poll_results(poll_element, results) { totalVotesElement.innerHTML = `${total_votes} votes` } - const poll_footer = poll_element.getElementsByTagName("footer") - poll_footer[0].insertBefore(totalVotesElement, poll_footer[0].firstChild) + const pollFooter = poll_element.getElementsByTagName("footer") + pollFooter[0].insertBefore(totalVotesElement, pollFooter[0].firstChild) + + poll_element.classList.add("populated") } const pollBlocks = document.getElementsByClassName("poll-block"); From de1b67ff4c6a12ae5587c51c37a7569d06dfa5a7 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 4 Feb 2024 17:32:43 -0800 Subject: [PATCH 08/12] Fix server-side poll results fetch on trail posts Trail polls should be fetched with the current post's blog name and ID --- src/templates/components/post.jinja | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/templates/components/post.jinja b/src/templates/components/post.jinja index 502bbd7..9fcb21f 100644 --- a/src/templates/components/post.jinja +++ b/src/templates/components/post.jinja @@ -36,7 +36,7 @@ trail_element.content, trail_element.layout, poll_callback=create_poll_callback( - request.app.ctx.TumblrAPI, trail_element.blog.name, trail_element.id + request.app.ctx.TumblrAPI, element.blog.name, element.id ), ) -%} @@ -45,8 +45,8 @@ {%- set trail_contains_errors, trail_content_tag = format_npf( trail_element.content, trail_element.layout, - trail_element.blog.name, - trail_element.id, + element.blog.name, + element.id, ) -%} {% endif %} From 87d2f560c19844df8c9df91ab385b97eeeecce1f Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 4 Feb 2024 20:01:19 -0800 Subject: [PATCH 09/12] Add support for polls with multiple winners --- assets/js/post.js | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/assets/js/post.js b/assets/js/post.js index 678a8eb..3a54e6b 100644 --- a/assets/js/post.js +++ b/assets/js/post.js @@ -19,23 +19,29 @@ function requestPollResults(poll_element, pollId) { function fill_poll_results(poll_element, results) { const sorted_poll_results = Object.entries(results.response.results).sort((a,b) => (a[1]-b[1])).reverse(); - // Find total votes first + // First we must find the total number of votes and the winner(s) of the poll let total_votes = 0; - for (let votes of sorted_poll_results) { - total_votes += votes[1]; - }; - // Create mapping of answer-id to answer choice element - const answerIdToChoiceElement = {} - const pollBody = poll_element.getElementsByClassName("poll-body")[0] + // Answer ID to winner status and amount of votes + const processed_poll_results = {}; + const winner_vote_count = sorted_poll_results[0][1]; + + for (let [answer_id, votes] of sorted_poll_results) { + processed_poll_results[answer_id] = {"is_winner": (winner_vote_count == votes), "votes": votes}; + total_votes += votes + } + + const answerIdChoiceElementArray = []; + const pollBody = poll_element.getElementsByClassName("poll-body")[0]; for (let choiceElement of pollBody.children) { - answerIdToChoiceElement[choiceElement.dataset.answerId] = choiceElement + answerIdChoiceElementArray.push([choiceElement.dataset.answerId, choiceElement]); } - for (let i = 0; i < sorted_poll_results.length; ++i) { - const [answer_id, answer_votes] = sorted_poll_results[i]; - const choiceElement = answerIdToChoiceElement[answer_id] + for (let [answer_id, choiceElement] of answerIdChoiceElementArray) { + const answer_results = processed_poll_results[answer_id]; + const is_winner = answer_results.is_winner; + const answer_votes = answer_results.votes; let numericalVoteProportion @@ -60,7 +66,7 @@ function fill_poll_results(poll_element, results) { voteCountElement.innerHTML = "< 0.1%"; } - if (i == 0) { + if (is_winner) { choiceElement.classList.add("poll-winner"); } From 6abc39d4c0bef776e261f787ad9720c737a916f8 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 4 Feb 2024 20:48:39 -0800 Subject: [PATCH 10/12] Poll: Fix bug on answer count != result count --- assets/js/post.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/js/post.js b/assets/js/post.js index 3a54e6b..c6d19d8 100644 --- a/assets/js/post.js +++ b/assets/js/post.js @@ -39,7 +39,7 @@ function fill_poll_results(poll_element, results) { } for (let [answer_id, choiceElement] of answerIdChoiceElementArray) { - const answer_results = processed_poll_results[answer_id]; + const answer_results = processed_poll_results[answer_id] || {"is_winner": false, "votes": 0}; const is_winner = answer_results.is_winner; const answer_votes = answer_results.votes; From 6bffe64c458b25f0fcb221cc4b432bac98cc1f81 Mon Sep 17 00:00:00 2001 From: syeopite Date: Mon, 5 Feb 2024 16:37:57 -0800 Subject: [PATCH 11/12] Increment z-index of timeline dropdown menus Fixes issue where polls would cover up the dropdown menus --- assets/css/base.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/css/base.css b/assets/css/base.css index bceffe7..2067b73 100644 --- a/assets/css/base.css +++ b/assets/css/base.css @@ -243,7 +243,7 @@ li.control-bar-action:hover { transform: translateX(-50%); overflow: hidden; width: fit-content; - z-index: 1; + z-index: 2; } .control-bar-dropdown-menu li { From 59646de73dee5a453114e99169a849843f79e81d Mon Sep 17 00:00:00 2001 From: syeopite Date: Mon, 5 Feb 2024 22:40:50 -0800 Subject: [PATCH 12/12] Add styling to polls --- assets/css/post-layout.css | 57 ++++++++++++++++++++++++++++++++- assets/js/post.js | 21 +++++++----- src/helpers/ext_npf_renderer.py | 6 ++-- 3 files changed, 72 insertions(+), 12 deletions(-) diff --git a/assets/css/post-layout.css b/assets/css/post-layout.css index 8303aec..6022d65 100644 --- a/assets/css/post-layout.css +++ b/assets/css/post-layout.css @@ -89,4 +89,59 @@ a.inline-link, a.inline-mention { a.toggle-poll-results { color: unset; text-decoration: underline; -} \ No newline at end of file +} + +.poll-block { + gap: 15px; + border: unset; + + padding: 0 10px; + margin: 10px 0; +} + +.poll-block > header > h3 { + font-weight: normal; + margin: 0; + font-size: 24px; +} + +.poll-choice { + border: 0; + border-radius: 10px; + background: #edeeee; + padding: 8px 18px; + overflow: hidden; +} + +.poll-choice > .answer { + font-weight: 550; +} + +.vote-proportion { + background: #dcdcdc; + border-radius: 10px 0px 0px 10px; +} + +.poll-winner >.vote-proportion { + background: #BABABA; +} + +.poll-block footer { + font-size: 15px; + margin-top: 0; + color: #6b7280; +} + +.toggle-poll-results { + display: block; + margin-top: 10px; + margin-bottom: 20px; +} + +.toggle-poll-results:hover { + font-weight: 700; +} + +.separator { + font-size: 12px; +} diff --git a/assets/js/post.js b/assets/js/post.js index c6d19d8..a8f4649 100644 --- a/assets/js/post.js +++ b/assets/js/post.js @@ -74,16 +74,21 @@ function fill_poll_results(poll_element, results) { choiceElement.appendChild(voteCountElement); } - const totalVotesElement = document.createElement("p") + const pollMetadataElement = poll_element.getElementsByClassName("poll-metadata")[0] - if (poll_element.classList.contains("expired")) { - totalVotesElement.innerHTML = `Final result from ${total_votes} votes` - } else { - totalVotesElement.innerHTML = `${total_votes} votes` - } + const totalVotesElement = document.createElement("span") + totalVotesElement.innerHTML = `${total_votes} votes` + + const separatorElement = document.createElement("span") + separatorElement.innerHTML = '•' + separatorElement.classList.add("separator") + + const pollTimestamp = pollMetadataElement.children[0] + + const newMetadataContents = document.createDocumentFragment() + newMetadataContents.append(totalVotesElement, separatorElement, pollTimestamp.cloneNode(true)) - const pollFooter = poll_element.getElementsByTagName("footer") - pollFooter[0].insertBefore(totalVotesElement, pollFooter[0].firstChild) + pollTimestamp.replaceWith(newMetadataContents) poll_element.classList.add("populated") } diff --git a/src/helpers/ext_npf_renderer.py b/src/helpers/ext_npf_renderer.py index 8e4ff4e..34d0f29 100644 --- a/src/helpers/ext_npf_renderer.py +++ b/src/helpers/ext_npf_renderer.py @@ -120,15 +120,15 @@ def _format_poll(self, block): if (self.blog_name and self.post_id) and not block.votes: poll_footer = poll_html[2] - poll_footer.add( - dominate.tags.noscript( + no_script_fallback = dominate.tags.noscript( dominate.tags.a( "See Results", href=f"/{self.blog_name}/{self.post_id}?fetch_polls=true", cls="toggle-poll-results" ) ) - ) + + poll_footer.children.insert(0, no_script_fallback) return poll_html