add append_version feature into StaticResource url resolver (#2158)

* add StaticResource.url append_version arg

* add 2157.feature

* 2157.feature contributor

* sort contributors.txt

* add documentation for StaticResource.url_for append_version arg #2157

* update read file code to be compatible with windows python 3.4

* fix flake8 line len issues

* add append_version param to StaticResource constructor and add_static method
docs and tests is updated to reflect changes

* add more tests and docs for StaticRoute append_version

* removed outdated comments

* add ValueError exception catching for follow_symlinks and more tests

CONTRIBUTORS.txt

@@ -79,6 +79,7 @@ Günther Jena
 Hu Bo
 Hugo Herter
 Hynek Schlawack
+Igor Alexandrov
 Igor Davydenko
 Igor Pavlov
 Ingmar Steen

aiohttp/web_urldispatcher.py

@@ -1,6 +1,8 @@
 import abc
 import asyncio
+import base64
 import collections
+import hashlib
 import inspect
 import keyword
 import os
@@ -442,11 +444,13 @@ def add_prefix(self, prefix):
 
 
 class StaticResource(PrefixResource):
+    VERSION_KEY = 'v'
 
     def __init__(self, prefix, directory, *, name=None,
-                 expect_handler=None, chunk_size=256*1024,
+                 expect_handler=None, chunk_size=256 * 1024,
                  response_factory=StreamResponse,
-                 show_index=False, follow_symlinks=False):
+                 show_index=False, follow_symlinks=False,
+                 append_version=False):
         super().__init__(prefix, name=name)
         try:
             directory = Path(directory)
@@ -463,24 +467,56 @@ def __init__(self, prefix, directory, *, name=None,
         self._chunk_size = chunk_size
         self._follow_symlinks = follow_symlinks
         self._expect_handler = expect_handler
+        self._append_version = append_version
 
         self._routes = {'GET': ResourceRoute('GET', self._handle, self,
                                              expect_handler=expect_handler),
 
                         'HEAD': ResourceRoute('HEAD', self._handle, self,
                                               expect_handler=expect_handler)}
 
-    def url(self, *, filename, query=None):
-        return str(self.url_for(filename=filename).with_query(query))
+    def url(self, *, filename, append_version=None, query=None):
+        url = self.url_for(filename=filename, append_version=append_version)
+        if query is not None:
+            return str(url.update_query(query))
+        return str(url)
 
-    def url_for(self, *, filename):
+    def url_for(self, *, filename, append_version=None):
+        if append_version is None:
+            append_version = self._append_version
         if isinstance(filename, Path):
             filename = str(filename)
         while filename.startswith('/'):
             filename = filename[1:]
         filename = '/' + filename
         url = self._prefix + URL(filename).raw_path
-        return URL(url)
+        url = URL(url)
+        if append_version is True:
+            try:
+                if filename.startswith('/'):
+                    filename = filename[1:]
+                filepath = self._directory.joinpath(filename).resolve()
+                if not self._follow_symlinks:
+                    filepath.relative_to(self._directory)
+            except (ValueError, FileNotFoundError):
+                # ValueError for case when path point to symlink
+                # with follow_symlinks is False
+                return url  # relatively safe
+            if filepath.is_file():
+                # TODO cache file content
+                # with file watcher for cache invalidation
+                with open(str(filepath), mode='rb') as f:
+                    file_bytes = f.read()
+                h = self._get_file_hash(file_bytes)
+                url = url.with_query({self.VERSION_KEY: h})
+                return url
+        return url
+
+    def _get_file_hash(self, byte_array):
+        m = hashlib.sha256()  # todo sha256 can be configurable param
+        m.update(byte_array)
+        b64 = base64.urlsafe_b64encode(m.digest())
+        return b64.decode('ascii')
 
     def get_info(self):
         return {'directory': self._directory,
@@ -848,8 +884,9 @@ def add_route(self, method, path, handler,
                                   expect_handler=expect_handler)
 
     def add_static(self, prefix, path, *, name=None, expect_handler=None,
-                   chunk_size=256*1024, response_factory=StreamResponse,
-                   show_index=False, follow_symlinks=False):
+                   chunk_size=256 * 1024, response_factory=StreamResponse,
+                   show_index=False, follow_symlinks=False,
+                   append_version=False):
         """Add static files view.
 
         prefix - url prefix
@@ -865,7 +902,8 @@ def add_static(self, prefix, path, *, name=None, expect_handler=None,
                                   chunk_size=chunk_size,
                                   response_factory=response_factory,
                                   show_index=show_index,
-                                  follow_symlinks=follow_symlinks)
+                                  follow_symlinks=follow_symlinks,
+                                  append_version=append_version)
         self.register_resource(resource)
         return resource
 

changes/2157.feature

@@ -0,0 +1,2 @@
+add `append_version` arg into `StaticResource.url` and `StaticResource.url_for` methods
+for getting an url with hash (version) of the file.

docs/web.rst

@@ -455,6 +455,17 @@ symlinks, parameter ``follow_symlinks`` should be set to ``True``::
 
    app.router.add_static('/prefix', path_to_static_folder, follow_symlinks=True)
 
+When you want to enable cache busting,
+parameter ``append_version`` can be set to ``True``
+
+Cache busting is the process of appending some form of file version hash
+to the filename of resources like JavaScript and CSS files.
+The performance advantage of doing this is that we can tell the browser
+to cache these files indefinitely without worrying about the client not getting
+the latest version when the file changes::
+
+   app.router.add_static('/prefix', path_to_static_folder, append_version=True)
+
 Template Rendering
 ------------------
 

docs/web_reference.rst

@@ -1580,7 +1580,8 @@ Router is any object that implements :class:`AbstractRouter` interface.
                           chunk_size=256*1024, \
                           response_factory=StreamResponse, \
                           show_index=False, \
-                          follow_symlinks=False)
+                          follow_symlinks=False, \
+                          append_version=False)
 
       Adds a router and a handler for returning static files.
 
@@ -1646,6 +1647,12 @@ Router is any object that implements :class:`AbstractRouter` interface.
                               a directory, by default it's not allowed and
                               HTTP/404 will be returned on access.
 
+      :param bool append_version: flag for adding file version (hash)
+                              to the url query string, this value will be used
+                              as default when you call to :meth:`StaticRoute.url`
+                              and :meth:`StaticRoute.url_for` methods.
+
+
       :returns: new :class:`StaticRoute` instance.
 
    .. method:: add_subapp(prefix, subapp)
@@ -1933,7 +1940,7 @@ Resource classes hierarchy::
    The class corresponds to resources for :ref:`static file serving
    <aiohttp-web-static-file-handling>`.
 
-   .. method:: url_for(filename)
+   .. method:: url_for(filename, append_version=None)
 
       Returns a :class:`~yarl.URL` for file path under resource prefix.
 
@@ -1944,6 +1951,14 @@ Resource classes hierarchy::
          E.g. an URL for ``'/prefix/dir/file.txt'`` should
          be generated as ``resource.url_for(filename='dir/file.txt')``
 
+      :param bool append_version: -- a flag for adding file version (hash) to the url query string for cache boosting
+
+         By default has value from an constructor (``False`` by default)
+         When set to ``True`` - ``v=FILE_HASH`` query string param will be added
+         When set to ``False`` has no impact
+
+         if file not found has no impact
+
       .. versionadded:: 1.1
 
 .. class:: PrefixedSubAppResource

tests/test_urldispatch.py

@@ -377,6 +377,125 @@ def test_add_static(router):
     assert len(resource) == 2
 
 
+def test_add_static_append_version(router):
+    resource = router.add_static('/st',
+                                 os.path.dirname(__file__),
+                                 name='static')
+    url = resource.url(filename='/data.unknown_mime_type', append_version=True)
+    expect_url = '/st/data.unknown_mime_type?' \
+                 'v=aUsn8CHEhhszc81d28QmlcBW0KQpfS2F4trgQKhOYd8%3D'
+    assert expect_url == url
+
+
+def test_add_static_append_version_set_from_constructor(router):
+    resource = router.add_static('/st',
+                                 os.path.dirname(__file__),
+                                 append_version=True,
+                                 name='static')
+    url = resource.url(filename='/data.unknown_mime_type')
+    expect_url = '/st/data.unknown_mime_type?' \
+                 'v=aUsn8CHEhhszc81d28QmlcBW0KQpfS2F4trgQKhOYd8%3D'
+    assert expect_url == url
+
+
+def test_add_static_append_version_override_constructor(router):
+    resource = router.add_static('/st',
+                                 os.path.dirname(__file__),
+                                 append_version=True,
+                                 name='static')
+    url = resource.url(filename='/data.unknown_mime_type',
+                       append_version=False)
+    expect_url = '/st/data.unknown_mime_type'
+    assert expect_url == url
+
+
+def test_add_static_append_version_filename_without_slash(router):
+    resource = router.add_static('/st',
+                                 os.path.dirname(__file__),
+                                 name='static')
+    url = resource.url(filename='data.unknown_mime_type', append_version=True)
+    expect_url = '/st/data.unknown_mime_type?' \
+                 'v=aUsn8CHEhhszc81d28QmlcBW0KQpfS2F4trgQKhOYd8%3D'
+    assert expect_url == url
+
+
+def test_add_static_append_version_non_exists_file(router):
+    resource = router.add_static('/st',
+                                 os.path.dirname(__file__),
+                                 name='static')
+    url = resource.url(filename='/non_exists_file', append_version=True)
+    assert '/st/non_exists_file' == url
+
+
+def test_add_static_append_version_non_exists_file_without_slash(router):
+    resource = router.add_static('/st',
+                                 os.path.dirname(__file__),
+                                 name='static')
+    url = resource.url(filename='non_exists_file', append_version=True)
+    assert '/st/non_exists_file' == url
+
+
+def test_add_static_append_version_follow_symlink(router, tmpdir):
+    """
+    Tests the access to a symlink, in static folder with apeend_version
+    """
+    tmp_dir_path = str(tmpdir)
+    symlink_path = os.path.join(tmp_dir_path, 'append_version_symlink')
+    symlink_target_path = os.path.dirname(__file__)
+    os.symlink(symlink_target_path, symlink_path, True)
+
+    # Register global static route:
+    resource = router.add_static('/st', tmp_dir_path, follow_symlinks=True,
+                                 append_version=True)
+
+    url = resource.url(
+        filename='/append_version_symlink/data.unknown_mime_type')
+
+    expect_url = '/st/append_version_symlink/data.unknown_mime_type?' \
+                 'v=aUsn8CHEhhszc81d28QmlcBW0KQpfS2F4trgQKhOYd8%3D'
+    assert expect_url == url
+
+
+def test_add_static_append_version_not_follow_symlink(router, tmpdir):
+    """
+    Tests the access to a symlink, in static folder with apeend_version
+    """
+    tmp_dir_path = str(tmpdir)
+    symlink_path = os.path.join(tmp_dir_path, 'append_version_symlink')
+    symlink_target_path = os.path.dirname(__file__)
+    os.symlink(symlink_target_path, symlink_path, True)
+
+    # Register global static route:
+    resource = router.add_static('/st', tmp_dir_path, follow_symlinks=False,
+                                 append_version=True)
+
+    filename = '/append_version_symlink/data.unknown_mime_type'
+    url = resource.url(filename=filename)
+    assert '/st/append_version_symlink/data.unknown_mime_type' == url
+
+
+def test_add_static_append_version_with_query(router):
+    resource = router.add_static('/st',
+                                 os.path.dirname(__file__),
+                                 name='static')
+    url = resource.url(filename='/data.unknown_mime_type',
+                       append_version=True,
+                       query={'key': 'val'})
+    expect_url = '/st/data.unknown_mime_type?' \
+                 'v=aUsn8CHEhhszc81d28QmlcBW0KQpfS2F4trgQKhOYd8%3D&key=val'
+    assert expect_url == url
+
+
+def test_add_static_append_version_non_exists_file_with_query(router):
+    resource = router.add_static('/st',
+                                 os.path.dirname(__file__),
+                                 name='static')
+    url = resource.url(filename='/non_exists_file',
+                       append_version=True,
+                       query={'key': 'val'})
+    assert '/st/non_exists_file?key=val' == url
+
+
 def test_plain_not_match(router):
     handler = make_handler()
     router.add_route('GET', '/get/path', handler, name='name')