From ee69c4026ae035bd0a488cbe718eaae5d49fc0e1 Mon Sep 17 00:00:00 2001 From: Igor Date: Sat, 5 Aug 2017 17:14:43 +0300 Subject: [PATCH] add append_version feature into StaticResource url resolver (#2158) * add StaticResource.url append_version arg * add 2157.feature * 2157.feature contributor * sort contributors.txt * add documentation for StaticResource.url_for append_version arg #2157 * update read file code to be compatible with windows python 3.4 * fix flake8 line len issues * add append_version param to StaticResource constructor and add_static method docs and tests is updated to reflect changes * add more tests and docs for StaticRoute append_version * removed outdated comments * add ValueError exception catching for follow_symlinks and more tests --- CONTRIBUTORS.txt | 1 + aiohttp/web_urldispatcher.py | 56 ++++++++++++++--- changes/2157.feature | 2 + docs/web.rst | 11 ++++ docs/web_reference.rst | 19 +++++- tests/test_urldispatch.py | 119 +++++++++++++++++++++++++++++++++++ 6 files changed, 197 insertions(+), 11 deletions(-) create mode 100644 changes/2157.feature diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index f2deee8a68a..fe67c5e800d 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -79,6 +79,7 @@ Günther Jena Hu Bo Hugo Herter Hynek Schlawack +Igor Alexandrov Igor Davydenko Igor Pavlov Ingmar Steen diff --git a/aiohttp/web_urldispatcher.py b/aiohttp/web_urldispatcher.py index d4e6845118d..3793ec946a0 100644 --- a/aiohttp/web_urldispatcher.py +++ b/aiohttp/web_urldispatcher.py @@ -1,6 +1,8 @@ import abc import asyncio +import base64 import collections +import hashlib import inspect import keyword import os @@ -442,11 +444,13 @@ def add_prefix(self, prefix): class StaticResource(PrefixResource): + VERSION_KEY = 'v' def __init__(self, prefix, directory, *, name=None, - expect_handler=None, chunk_size=256*1024, + expect_handler=None, chunk_size=256 * 1024, response_factory=StreamResponse, - show_index=False, follow_symlinks=False): + show_index=False, follow_symlinks=False, + append_version=False): super().__init__(prefix, name=name) try: directory = Path(directory) @@ -463,6 +467,7 @@ def __init__(self, prefix, directory, *, name=None, self._chunk_size = chunk_size self._follow_symlinks = follow_symlinks self._expect_handler = expect_handler + self._append_version = append_version self._routes = {'GET': ResourceRoute('GET', self._handle, self, expect_handler=expect_handler), @@ -470,17 +475,48 @@ def __init__(self, prefix, directory, *, name=None, 'HEAD': ResourceRoute('HEAD', self._handle, self, expect_handler=expect_handler)} - def url(self, *, filename, query=None): - return str(self.url_for(filename=filename).with_query(query)) + def url(self, *, filename, append_version=None, query=None): + url = self.url_for(filename=filename, append_version=append_version) + if query is not None: + return str(url.update_query(query)) + return str(url) - def url_for(self, *, filename): + def url_for(self, *, filename, append_version=None): + if append_version is None: + append_version = self._append_version if isinstance(filename, Path): filename = str(filename) while filename.startswith('/'): filename = filename[1:] filename = '/' + filename url = self._prefix + URL(filename).raw_path - return URL(url) + url = URL(url) + if append_version is True: + try: + if filename.startswith('/'): + filename = filename[1:] + filepath = self._directory.joinpath(filename).resolve() + if not self._follow_symlinks: + filepath.relative_to(self._directory) + except (ValueError, FileNotFoundError): + # ValueError for case when path point to symlink + # with follow_symlinks is False + return url # relatively safe + if filepath.is_file(): + # TODO cache file content + # with file watcher for cache invalidation + with open(str(filepath), mode='rb') as f: + file_bytes = f.read() + h = self._get_file_hash(file_bytes) + url = url.with_query({self.VERSION_KEY: h}) + return url + return url + + def _get_file_hash(self, byte_array): + m = hashlib.sha256() # todo sha256 can be configurable param + m.update(byte_array) + b64 = base64.urlsafe_b64encode(m.digest()) + return b64.decode('ascii') def get_info(self): return {'directory': self._directory, @@ -848,8 +884,9 @@ def add_route(self, method, path, handler, expect_handler=expect_handler) def add_static(self, prefix, path, *, name=None, expect_handler=None, - chunk_size=256*1024, response_factory=StreamResponse, - show_index=False, follow_symlinks=False): + chunk_size=256 * 1024, response_factory=StreamResponse, + show_index=False, follow_symlinks=False, + append_version=False): """Add static files view. prefix - url prefix @@ -865,7 +902,8 @@ def add_static(self, prefix, path, *, name=None, expect_handler=None, chunk_size=chunk_size, response_factory=response_factory, show_index=show_index, - follow_symlinks=follow_symlinks) + follow_symlinks=follow_symlinks, + append_version=append_version) self.register_resource(resource) return resource diff --git a/changes/2157.feature b/changes/2157.feature new file mode 100644 index 00000000000..ce3e362e43f --- /dev/null +++ b/changes/2157.feature @@ -0,0 +1,2 @@ +add `append_version` arg into `StaticResource.url` and `StaticResource.url_for` methods +for getting an url with hash (version) of the file. \ No newline at end of file diff --git a/docs/web.rst b/docs/web.rst index 5a5170485b7..76300f29a49 100644 --- a/docs/web.rst +++ b/docs/web.rst @@ -455,6 +455,17 @@ symlinks, parameter ``follow_symlinks`` should be set to ``True``:: app.router.add_static('/prefix', path_to_static_folder, follow_symlinks=True) +When you want to enable cache busting, +parameter ``append_version`` can be set to ``True`` + +Cache busting is the process of appending some form of file version hash +to the filename of resources like JavaScript and CSS files. +The performance advantage of doing this is that we can tell the browser +to cache these files indefinitely without worrying about the client not getting +the latest version when the file changes:: + + app.router.add_static('/prefix', path_to_static_folder, append_version=True) + Template Rendering ------------------ diff --git a/docs/web_reference.rst b/docs/web_reference.rst index 84db0e1b226..6fb18997e29 100644 --- a/docs/web_reference.rst +++ b/docs/web_reference.rst @@ -1580,7 +1580,8 @@ Router is any object that implements :class:`AbstractRouter` interface. chunk_size=256*1024, \ response_factory=StreamResponse, \ show_index=False, \ - follow_symlinks=False) + follow_symlinks=False, \ + append_version=False) Adds a router and a handler for returning static files. @@ -1646,6 +1647,12 @@ Router is any object that implements :class:`AbstractRouter` interface. a directory, by default it's not allowed and HTTP/404 will be returned on access. + :param bool append_version: flag for adding file version (hash) + to the url query string, this value will be used + as default when you call to :meth:`StaticRoute.url` + and :meth:`StaticRoute.url_for` methods. + + :returns: new :class:`StaticRoute` instance. .. method:: add_subapp(prefix, subapp) @@ -1933,7 +1940,7 @@ Resource classes hierarchy:: The class corresponds to resources for :ref:`static file serving `. - .. method:: url_for(filename) + .. method:: url_for(filename, append_version=None) Returns a :class:`~yarl.URL` for file path under resource prefix. @@ -1944,6 +1951,14 @@ Resource classes hierarchy:: E.g. an URL for ``'/prefix/dir/file.txt'`` should be generated as ``resource.url_for(filename='dir/file.txt')`` + :param bool append_version: -- a flag for adding file version (hash) to the url query string for cache boosting + + By default has value from an constructor (``False`` by default) + When set to ``True`` - ``v=FILE_HASH`` query string param will be added + When set to ``False`` has no impact + + if file not found has no impact + .. versionadded:: 1.1 .. class:: PrefixedSubAppResource diff --git a/tests/test_urldispatch.py b/tests/test_urldispatch.py index 8f5eae5af43..00a317fd7cb 100644 --- a/tests/test_urldispatch.py +++ b/tests/test_urldispatch.py @@ -377,6 +377,125 @@ def test_add_static(router): assert len(resource) == 2 +def test_add_static_append_version(router): + resource = router.add_static('/st', + os.path.dirname(__file__), + name='static') + url = resource.url(filename='/data.unknown_mime_type', append_version=True) + expect_url = '/st/data.unknown_mime_type?' \ + 'v=aUsn8CHEhhszc81d28QmlcBW0KQpfS2F4trgQKhOYd8%3D' + assert expect_url == url + + +def test_add_static_append_version_set_from_constructor(router): + resource = router.add_static('/st', + os.path.dirname(__file__), + append_version=True, + name='static') + url = resource.url(filename='/data.unknown_mime_type') + expect_url = '/st/data.unknown_mime_type?' \ + 'v=aUsn8CHEhhszc81d28QmlcBW0KQpfS2F4trgQKhOYd8%3D' + assert expect_url == url + + +def test_add_static_append_version_override_constructor(router): + resource = router.add_static('/st', + os.path.dirname(__file__), + append_version=True, + name='static') + url = resource.url(filename='/data.unknown_mime_type', + append_version=False) + expect_url = '/st/data.unknown_mime_type' + assert expect_url == url + + +def test_add_static_append_version_filename_without_slash(router): + resource = router.add_static('/st', + os.path.dirname(__file__), + name='static') + url = resource.url(filename='data.unknown_mime_type', append_version=True) + expect_url = '/st/data.unknown_mime_type?' \ + 'v=aUsn8CHEhhszc81d28QmlcBW0KQpfS2F4trgQKhOYd8%3D' + assert expect_url == url + + +def test_add_static_append_version_non_exists_file(router): + resource = router.add_static('/st', + os.path.dirname(__file__), + name='static') + url = resource.url(filename='/non_exists_file', append_version=True) + assert '/st/non_exists_file' == url + + +def test_add_static_append_version_non_exists_file_without_slash(router): + resource = router.add_static('/st', + os.path.dirname(__file__), + name='static') + url = resource.url(filename='non_exists_file', append_version=True) + assert '/st/non_exists_file' == url + + +def test_add_static_append_version_follow_symlink(router, tmpdir): + """ + Tests the access to a symlink, in static folder with apeend_version + """ + tmp_dir_path = str(tmpdir) + symlink_path = os.path.join(tmp_dir_path, 'append_version_symlink') + symlink_target_path = os.path.dirname(__file__) + os.symlink(symlink_target_path, symlink_path, True) + + # Register global static route: + resource = router.add_static('/st', tmp_dir_path, follow_symlinks=True, + append_version=True) + + url = resource.url( + filename='/append_version_symlink/data.unknown_mime_type') + + expect_url = '/st/append_version_symlink/data.unknown_mime_type?' \ + 'v=aUsn8CHEhhszc81d28QmlcBW0KQpfS2F4trgQKhOYd8%3D' + assert expect_url == url + + +def test_add_static_append_version_not_follow_symlink(router, tmpdir): + """ + Tests the access to a symlink, in static folder with apeend_version + """ + tmp_dir_path = str(tmpdir) + symlink_path = os.path.join(tmp_dir_path, 'append_version_symlink') + symlink_target_path = os.path.dirname(__file__) + os.symlink(symlink_target_path, symlink_path, True) + + # Register global static route: + resource = router.add_static('/st', tmp_dir_path, follow_symlinks=False, + append_version=True) + + filename = '/append_version_symlink/data.unknown_mime_type' + url = resource.url(filename=filename) + assert '/st/append_version_symlink/data.unknown_mime_type' == url + + +def test_add_static_append_version_with_query(router): + resource = router.add_static('/st', + os.path.dirname(__file__), + name='static') + url = resource.url(filename='/data.unknown_mime_type', + append_version=True, + query={'key': 'val'}) + expect_url = '/st/data.unknown_mime_type?' \ + 'v=aUsn8CHEhhszc81d28QmlcBW0KQpfS2F4trgQKhOYd8%3D&key=val' + assert expect_url == url + + +def test_add_static_append_version_non_exists_file_with_query(router): + resource = router.add_static('/st', + os.path.dirname(__file__), + name='static') + url = resource.url(filename='/non_exists_file', + append_version=True, + query={'key': 'val'}) + assert '/st/non_exists_file?key=val' == url + + def test_plain_not_match(router): handler = make_handler() router.add_route('GET', '/get/path', handler, name='name')