negotiate the best static asset using supported encodings

10 files changed, 350 insertions, 26 deletions

diff --git a/CHANGES.rst b/CHANGES.rst
index 537ca4671..8cf6eaff6 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -64,6 +64,12 @@ Features
   to predicate factories as their second argument.
   See https://github.com/Pylons/pyramid/pull/3514
 
+- Added support for serving pre-compressed static assets by using the
+  ``content_encodings`` argument of
+  ``pyramid.config.Configurator.add_static_view`` and
+  ``pyramid.static.static_view``.
+  See https://github.com/Pylons/pyramid/pull/3537
+
 Deprecations
 ------------
 
diff --git a/docs/narr/assets.rst b/docs/narr/assets.rst
index d1d64f0c3..f9d30563e 100644
--- a/docs/narr/assets.rst
+++ b/docs/narr/assets.rst
@@ -190,6 +190,37 @@ such a request.  The :meth:`~pyramid.request.Request.static_url` API is
 discussed in more detail later in this chapter.
 
 .. index::
+   single: pre-compressed assets
+
+.. _pre_compressed_assets:
+
+Serving Pre-compressed Assets
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 2.0
+
+It's possible to configure :app:`Pyramid` to serve pre-compressed static assets.
+This can greatly reduce the bandwidth required to serve assets - most modern browsers support ``gzip``, ``deflate``, and ``br`` (brotli) encoded responses.
+A client declares support for encoded responses using the ``Accept-Encoding`` HTTP header. For example, ``Accept-Encoding: gzip, default, br``.
+The response will then contain the pre-compressed content with the ``Content-Encoding`` header set to the matched encoding.
+This feature assumes that the static assets exist unencoded (``identity`` encoding) as well as in zero or more encoded formats.
+If the encoded version of a file is missing, or the client doesn't declare support for the encoded version, the unencoded version is returned instead.
+
+In order to configure this in your application, the first step is to compress your assets.
+For example, update your static asset pipeline to export ``.gz`` versions of every file.
+Second, add ``content_encodings=['gzip']`` when invoking :meth:`pyramid.config.Configurator.add_static_view`.
+
+The encoded file extensions are determined by :attr:`mimetypes.encodings_map`.
+So, if your desired encoding is missing, you'll need to add it there:
+
+.. code-block:: python
+
+    import mimetypes
+    mimetypes.encodings_map['.br'] = 'br'  # add brotli
+
+It is not necessary for every file to support every encoding, but :app:`Pyramid` will not serve an encoding that is not declared.
+
+.. index::
    single: generating static asset urls
    single: static asset urls
    pair:   assets; generating urls
diff --git a/src/pyramid/config/views.py b/src/pyramid/config/views.py
index afb685f93..509a64b2b 100644
--- a/src/pyramid/config/views.py
+++ b/src/pyramid/config/views.py
@@ -1943,6 +1943,16 @@ class ViewsConfiguratorMixin(object):
         prefix*.  By default, this argument is ``None``, meaning that no
         particular Expires or Cache-Control headers are set in the response.
 
+        The ``content_encodings`` keyword argument is a list of alternative
+        file encodings supported in the ``Accept-Encoding`` HTTP Header.
+        Alternative files are found using file extensions defined in
+        :attr:`mimetypes.encodings_map`. An encoded asset will be returned
+        with the ``Content-Encoding`` header set to the selected encoding.
+        If the asset contains alternative encodings then the
+        ``Accept-Encoding`` value will be added to the response's ``Vary``
+        header. By default, the list is empty and no alternatives will be
+        supported.
+
         The ``permission`` keyword argument is used to specify the
         :term:`permission` required by a user to execute the static view.  By
         default, it is the string
@@ -2019,6 +2029,11 @@ class ViewsConfiguratorMixin(object):
            static_url('mypackage:images/logo.png', request)
 
         See :ref:`static_assets_section` for more information.
+
+        .. versionchanged:: 2.0
+
+           Added the ``content_encodings`` argument.
+
         """
         spec = self._make_spec(path)
         info = self._get_static_info()
@@ -2191,10 +2206,15 @@ class StaticURLInfo(object):
             # it's a view name
             url = None
             cache_max_age = extra.pop('cache_max_age', None)
+            content_encodings = extra.pop('content_encodings', [])
 
             # create a view
             view = static_view(
-                spec, cache_max_age=cache_max_age, use_subpath=True
+                spec,
+                cache_max_age=cache_max_age,
+                use_subpath=True,
+                reload=config.registry.settings['pyramid.reload_assets'],
+                content_encodings=content_encodings,
             )
 
             # Mutate extra to allow factory, etc to be passed through here.
diff --git a/src/pyramid/static.py b/src/pyramid/static.py
index e3561e93e..7870b803e 100644
--- a/src/pyramid/static.py
+++ b/src/pyramid/static.py
@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
 from functools import lru_cache
 import json
+import mimetypes
 import os
 
-from os.path import getmtime, normcase, normpath, join, isdir, exists
+from os.path import getmtime, getsize, normcase, normpath, join, isdir, exists
 
 from pkg_resources import resource_exists, resource_filename, resource_isdir
 
@@ -53,6 +54,19 @@ class static_view(object):
     the static application will consider request.environ[``PATH_INFO``] as
     ``PATH_INFO`` input. By default, this is ``False``.
 
+    ``reload`` controls whether a cache of files is maintained or the asset
+    subsystem is queried per-request to determine what files are available.
+    By default, this is ``False`` and new files added while the process is
+    running are not recognized.
+
+    ``content_encodings`` is a list of alternative file encodings supported
+    in the ``Accept-Encoding`` HTTP Header. Alternative files are found using
+    file extensions defined in :attr:`mimetypes.encodings_map`. An encoded
+    asset will be returned with the ``Content-Encoding`` header set to the
+    selected encoding. If the asset contains alternative encodings then the
+    ``Accept-Encoding`` value will be added to the response's ``Vary`` header.
+    By default, the list is empty and no alternatives will be supported.
+
     .. note::
 
        If the ``root_dir`` is relative to a :term:`package`, or is a
@@ -61,6 +75,11 @@ class static_view(object):
        assets within the named ``root_dir`` package-relative directory.
        However, if the ``root_dir`` is absolute, configuration will not be able
        to override the assets it contains.
+
+    .. versionchanged:: 2.0
+
+       Added ``reload`` and ``content_encodings`` options.
+
     """
 
     def __init__(
@@ -70,6 +89,8 @@ class static_view(object):
         package_name=None,
         use_subpath=False,
         index='index.html',
+        reload=False,
+        content_encodings=(),
     ):
         # package_name is for bw compat; it is preferred to pass in a
         # package-relative path as root_dir
@@ -83,8 +104,36 @@ class static_view(object):
         self.docroot = docroot
         self.norm_docroot = normcase(normpath(docroot))
         self.index = index
+        self.reload = reload
+        self.content_encodings = _compile_content_encodings(content_encodings)
+        self.filemap = {}
 
     def __call__(self, context, request):
+        resource_name = self.get_resource_name(request)
+        files = self.get_possible_files(resource_name)
+        filepath, content_encoding = self.find_best_match(request, files)
+        if filepath is None:
+            raise HTTPNotFound(request.url)
+
+        content_type, _ = _guess_type(resource_name)
+        response = FileResponse(
+            filepath,
+            request,
+            self.cache_max_age,
+            content_type,
+            content_encoding,
+        )
+        if len(files) > 1:
+            _add_vary(response, 'Accept-Encoding')
+        return response
+
+    def get_resource_name(self, request):
+        """
+        Return the computed name of the requested resource.
+
+        The returned file is not guaranteed to exist.
+
+        """
         if self.use_subpath:
             path_tuple = request.subpath
         else:
@@ -94,46 +143,126 @@ class static_view(object):
         if path is None:
             raise HTTPNotFound('Out of bounds: %s' % request.url)
 
+        # normalize asset spec or fs path into resource_path
         if self.package_name:  # package resource
             resource_path = '%s/%s' % (self.docroot.rstrip('/'), path)
             if resource_isdir(self.package_name, resource_path):
                 if not request.path_url.endswith('/'):
-                    self.add_slash_redirect(request)
+                    raise self.add_slash_redirect(request)
                 resource_path = '%s/%s' % (
                     resource_path.rstrip('/'),
                     self.index,
                 )
 
-            if not resource_exists(self.package_name, resource_path):
-                raise HTTPNotFound(request.url)
-            filepath = resource_filename(self.package_name, resource_path)
-
         else:  # filesystem file
-
             # os.path.normpath converts / to \ on windows
-            filepath = normcase(normpath(join(self.norm_docroot, path)))
-            if isdir(filepath):
+            resource_path = normcase(normpath(join(self.norm_docroot, path)))
+            if isdir(resource_path):
                 if not request.path_url.endswith('/'):
-                    self.add_slash_redirect(request)
-                filepath = join(filepath, self.index)
-            if not exists(filepath):
-                raise HTTPNotFound(request.url)
+                    raise self.add_slash_redirect(request)
+                resource_path = join(resource_path, self.index)
 
-        content_type, content_encoding = _guess_type(filepath)
-        return FileResponse(
-            filepath,
-            request,
-            self.cache_max_age,
-            content_type,
-            content_encoding=None,
-        )
+        return resource_path
+
+    def find_resource_path(self, name):
+        """
+        Return the absolute path to the resource or ``None`` if it doesn't
+        exist.
+
+        """
+        if self.package_name:
+            if resource_exists(self.package_name, name):
+                return resource_filename(self.package_name, name)
+
+        elif exists(name):
+            return name
+
+    def get_possible_files(self, resource_name):
+        """ Return a sorted list of ``(size, encoding, path)`` entries."""
+        result = self.filemap.get(resource_name)
+        if result is not None:
+            return result
+
+        # XXX we could put a lock around this work but worst case scenario a
+        # couple requests scan the disk for files at the same time and then
+        # the cache is set going forward so do not bother
+        result = []
+
+        # add the identity
+        path = self.find_resource_path(resource_name)
+        if path:
+            result.append((path, None))
+
+        # add each file we find for the supported encodings
+        # we don't mind adding multiple files for the same encoding if there
+        # are copies with different extensions because we sort by size so the
+        # smallest is always found first and the rest ignored
+        for encoding, extensions in self.content_encodings.items():
+            for ext in extensions:
+                encoded_name = resource_name + ext
+                path = self.find_resource_path(encoded_name)
+                if path:
+                    result.append((path, encoding))
+
+        # sort the files by size, smallest first
+        result.sort(key=lambda x: getsize(x[0]))
+
+        # only cache the results if reload is disabled
+        if not self.reload:
+            self.filemap[resource_name] = result
+        return result
+
+    def find_best_match(self, request, files):
+        """ Return ``(path | None, encoding)``."""
+        # if the client did not specify encodings then assume only the
+        # identity is acceptable
+        if not request.accept_encoding:
+            identity_path = next(
+                (path for path, encoding in files if encoding is None), None,
+            )
+            return identity_path, None
+
+        # find encodings the client will accept
+        acceptable_encodings = {
+            x[0]
+            for x in request.accept_encoding.acceptable_offers(
+                [encoding for path, encoding in files if encoding is not None]
+            )
+        }
+        acceptable_encodings.add(None)
+
+        # return the smallest file from the acceptable encodings
+        # we know that files is sorted by size, smallest first
+        for path, encoding in files:
+            if encoding in acceptable_encodings:
+                return path, encoding
 
     def add_slash_redirect(self, request):
         url = request.path_url + '/'
         qs = request.query_string
         if qs:
             url = url + '?' + qs
-        raise HTTPMovedPermanently(url)
+        return HTTPMovedPermanently(url)
+
+
+def _compile_content_encodings(encodings):
+    """
+    Convert mimetypes.encodings_map into a dict of
+    ``(encoding) -> [file extensions]``.
+
+    """
+    result = {}
+    for ext, encoding in mimetypes.encodings_map.items():
+        if encoding in encodings:
+            result.setdefault(encoding, []).append(ext)
+    return result
+
+
+def _add_vary(response, option):
+    vary = response.vary or []
+    if not any(x.lower() == option.lower() for x in vary):
+        vary.append(option)
+    response.vary = vary
 
 
 _seps = set(['/', os.sep])
diff --git a/tests/fixtures/static/encoded.html b/tests/fixtures/static/encoded.html
new file mode 100644
index 000000000..0999b4f1b
--- /dev/null
+++ b/tests/fixtures/static/encoded.html
@@ -0,0 +1,15 @@
+<!--
+  when modified, re-run:
+    gzip -k encoded.html
+-->
+<html>
+<head>
+<title>
+A Simple HTML Document
+</title>
+</head>
+<body>
+<p>This is a very simple HTML document</p>
+<p>It only has two paragraphs</p>
+</body>
+</html>
diff --git a/tests/fixtures/static/encoded.html.gz b/tests/fixtures/static/encoded.html.gz
new file mode 100644
index 000000000..afcc25768
--- /dev/null
+++ b/tests/fixtures/static/encoded.html.gz
diff --git a/tests/pkgs/static_encodings/__init__.py b/tests/pkgs/static_encodings/__init__.py
new file mode 100644
index 000000000..3b86a1a15
--- /dev/null
+++ b/tests/pkgs/static_encodings/__init__.py
@@ -0,0 +1,2 @@
+def includeme(config):
+    config.add_static_view('/', 'tests:fixtures', content_encodings=['gzip'])
diff --git a/tests/test_config/test_views.py b/tests/test_config/test_views.py
index a1e975756..baa87dd6b 100644
--- a/tests/test_config/test_views.py
+++ b/tests/test_config/test_views.py
@@ -4174,7 +4174,9 @@ class DummyRegistry:
     utility = None
 
     def __init__(self):
-        self.settings = {}
+        self.settings = {
+            'pyramid.reload_assets': False,
+        }
 
     def queryUtility(self, type_or_iface, name=None, default=None):
         return self.utility or default
diff --git a/tests/test_integration.py b/tests/test_integration.py
index 331542d7d..8a4575d7b 100644
--- a/tests/test_integration.py
+++ b/tests/test_integration.py
@@ -73,8 +73,8 @@ class IntegrationBase(object):
             root_factory=self.root_factory, package=self.package
         )
         config.include(self.package)
-        app = config.make_wsgi_app()
-        self.testapp = TestApp(app)
+        self.app = config.make_wsgi_app()
+        self.testapp = TestApp(self.app)
         self.config = config
 
     def tearDown(self):
@@ -227,6 +227,57 @@ class TestStaticAppUsingAssetSpec(StaticAppBase, unittest.TestCase):
     package = 'tests.pkgs.static_assetspec'
 
 
+class TestStaticAppWithEncodings(IntegrationBase, unittest.TestCase):
+    package = 'tests.pkgs.static_encodings'
+
+    # XXX webtest actually runs response.decode_content() and so we can't
+    # use it to test gzip- or deflate-encoded responses to see if they
+    # were transferred correctly
+    def _getResponse(self, *args, **kwargs):
+        from pyramid.request import Request
+
+        req = Request.blank(*args, **kwargs)
+        return req.get_response(self.app)
+
+    def test_no_accept(self):
+        res = self._getResponse('/static/encoded.html')
+        self.assertEqual(res.headers['Vary'], 'Accept-Encoding')
+        self.assertNotIn('Content-Encoding', res.headers)
+        _assertBody(
+            res.body, os.path.join(here, 'fixtures/static/encoded.html')
+        )
+
+    def test_unsupported_accept(self):
+        res = self._getResponse(
+            '/static/encoded.html',
+            headers={'Accept-Encoding': 'br, foo, bar'},
+        )
+        self.assertEqual(res.headers['Vary'], 'Accept-Encoding')
+        self.assertNotIn('Content-Encoding', res.headers)
+        _assertBody(
+            res.body, os.path.join(here, 'fixtures/static/encoded.html')
+        )
+
+    def test_accept_gzip(self):
+        res = self._getResponse(
+            '/static/encoded.html',
+            headers={'Accept-Encoding': 'br, foo, gzip'},
+        )
+        self.assertEqual(res.headers['Vary'], 'Accept-Encoding')
+        self.assertEqual(res.headers['Content-Encoding'], 'gzip')
+        _assertBody(
+            res.body, os.path.join(here, 'fixtures/static/encoded.html.gz')
+        )
+
+    def test_accept_gzip_returns_identity(self):
+        res = self._getResponse(
+            '/static/index.html', headers={'Accept-Encoding': 'gzip'}
+        )
+        self.assertNotIn('Vary', res.headers)
+        self.assertNotIn('Content-Encoding', res.headers)
+        _assertBody(res.body, os.path.join(here, 'fixtures/static/index.html'))
+
+
 class TestStaticAppNoSubpath(unittest.TestCase):
     staticapp = static_view(os.path.join(here, 'fixtures'), use_subpath=False)
 
diff --git a/tests/test_static.py b/tests/test_static.py
index a323b1d89..3d0deda3f 100644
--- a/tests/test_static.py
+++ b/tests/test_static.py
@@ -39,6 +39,8 @@ class Test_static_view_use_subpath_False(unittest.TestCase):
         self.assertEqual(inst.docroot, 'resource_name')
         self.assertEqual(inst.cache_max_age, 3600)
         self.assertEqual(inst.index, 'index.html')
+        self.assertEqual(inst.reload, False)
+        self.assertEqual(inst.content_encodings, {})
 
     def test_call_adds_slash_path_info_empty(self):
         inst = self._makeOne('tests:fixtures/static')
@@ -252,6 +254,8 @@ class Test_static_view_use_subpath_True(unittest.TestCase):
         self.assertEqual(inst.docroot, 'resource_name')
         self.assertEqual(inst.cache_max_age, 3600)
         self.assertEqual(inst.index, 'index.html')
+        self.assertEqual(inst.reload, False)
+        self.assertEqual(inst.content_encodings, {})
 
     def test_call_adds_slash_path_info_empty(self):
         inst = self._makeOne('tests:fixtures/static')
@@ -403,6 +407,70 @@ class Test_static_view_use_subpath_True(unittest.TestCase):
         self.assertRaises(HTTPNotFound, inst, context, request)
 
 
+class Test_static_view_content_encodings(unittest.TestCase):
+    def _getTargetClass(self):
+        from pyramid.static import static_view
+
+        return static_view
+
+    def _makeOne(self, *arg, **kw):
+        return self._getTargetClass()(*arg, **kw)
+
+    def _makeRequest(self, kw=None):
+        from pyramid.request import Request
+
+        environ = {
+            'wsgi.url_scheme': 'http',
+            'wsgi.version': (1, 0),
+            'SERVER_NAME': 'example.com',
+            'SERVER_PORT': '6543',
+            'PATH_INFO': '/',
+            'SCRIPT_NAME': '',
+            'REQUEST_METHOD': 'GET',
+        }
+        if kw is not None:
+            environ.update(kw)
+        return Request(environ=environ)
+
+    def test_call_without_accept(self):
+        inst = self._makeOne(
+            'tests:fixtures/static', content_encodings=['gzip']
+        )
+        request = self._makeRequest({'PATH_INFO': '/encoded.html'})
+        context = DummyContext()
+
+        res = inst(context, request)
+        self.assertEqual(res.headers['Vary'], 'Accept-Encoding')
+        self.assertNotIn('Content-Encoding', res.headers)
+        self.assertEqual(len(res.body), 221)
+
+    def test_call_with_accept_gzip(self):
+        inst = self._makeOne(
+            'tests:fixtures/static', content_encodings=['gzip']
+        )
+        request = self._makeRequest(
+            {'PATH_INFO': '/encoded.html', 'HTTP_ACCEPT_ENCODING': 'gzip'}
+        )
+        context = DummyContext()
+
+        res = inst(context, request)
+        self.assertEqual(res.headers['Vary'], 'Accept-Encoding')
+        self.assertEqual(res.headers['Content-Encoding'], 'gzip')
+        self.assertEqual(len(res.body), 187)
+
+    def test_call_get_possible_files_is_cached(self):
+        inst = self._makeOne('tests:fixtures/static')
+        result1 = inst.get_possible_files('tests:fixtures/static/encoded.html')
+        result2 = inst.get_possible_files('tests:fixtures/static/encoded.html')
+        self.assertIs(result1, result2)
+
+    def test_call_get_possible_files_is_not_cached(self):
+        inst = self._makeOne('tests:fixtures/static', reload=True)
+        result1 = inst.get_possible_files('tests:fixtures/static/encoded.html')
+        result2 = inst.get_possible_files('tests:fixtures/static/encoded.html')
+        self.assertIsNot(result1, result2)
+
+
 class TestQueryStringConstantCacheBuster(unittest.TestCase):
     def _makeOne(self, param=None):
         from pyramid.static import QueryStringConstantCacheBuster as cls