From f6cb1efa8fba683bdc5c9b4a645f9357fe2e6208 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Wed, 4 Dec 2019 01:13:52 -0600 Subject: negotiate the best static asset using supported encodings --- CHANGES.rst | 6 ++ docs/narr/assets.rst | 31 ++++++ src/pyramid/config/views.py | 22 +++- src/pyramid/static.py | 173 ++++++++++++++++++++++++++++---- tests/fixtures/static/encoded.html | 15 +++ tests/fixtures/static/encoded.html.gz | Bin 0 -> 187 bytes tests/pkgs/static_encodings/__init__.py | 2 + tests/test_config/test_views.py | 4 +- tests/test_integration.py | 55 +++++++++- tests/test_static.py | 68 +++++++++++++ 10 files changed, 350 insertions(+), 26 deletions(-) create mode 100644 tests/fixtures/static/encoded.html create mode 100644 tests/fixtures/static/encoded.html.gz create mode 100644 tests/pkgs/static_encodings/__init__.py diff --git a/CHANGES.rst b/CHANGES.rst index 537ca4671..8cf6eaff6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -64,6 +64,12 @@ Features to predicate factories as their second argument. See https://github.com/Pylons/pyramid/pull/3514 +- Added support for serving pre-compressed static assets by using the + ``content_encodings`` argument of + ``pyramid.config.Configurator.add_static_view`` and + ``pyramid.static.static_view``. + See https://github.com/Pylons/pyramid/pull/3537 + Deprecations ------------ diff --git a/docs/narr/assets.rst b/docs/narr/assets.rst index d1d64f0c3..f9d30563e 100644 --- a/docs/narr/assets.rst +++ b/docs/narr/assets.rst @@ -189,6 +189,37 @@ listening on ``example.com`` must be itself configured to respond properly to such a request. The :meth:`~pyramid.request.Request.static_url` API is discussed in more detail later in this chapter. +.. index:: + single: pre-compressed assets + +.. _pre_compressed_assets: + +Serving Pre-compressed Assets +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 2.0 + +It's possible to configure :app:`Pyramid` to serve pre-compressed static assets. +This can greatly reduce the bandwidth required to serve assets - most modern browsers support ``gzip``, ``deflate``, and ``br`` (brotli) encoded responses. +A client declares support for encoded responses using the ``Accept-Encoding`` HTTP header. For example, ``Accept-Encoding: gzip, default, br``. +The response will then contain the pre-compressed content with the ``Content-Encoding`` header set to the matched encoding. +This feature assumes that the static assets exist unencoded (``identity`` encoding) as well as in zero or more encoded formats. +If the encoded version of a file is missing, or the client doesn't declare support for the encoded version, the unencoded version is returned instead. + +In order to configure this in your application, the first step is to compress your assets. +For example, update your static asset pipeline to export ``.gz`` versions of every file. +Second, add ``content_encodings=['gzip']`` when invoking :meth:`pyramid.config.Configurator.add_static_view`. + +The encoded file extensions are determined by :attr:`mimetypes.encodings_map`. +So, if your desired encoding is missing, you'll need to add it there: + +.. code-block:: python + + import mimetypes + mimetypes.encodings_map['.br'] = 'br' # add brotli + +It is not necessary for every file to support every encoding, but :app:`Pyramid` will not serve an encoding that is not declared. + .. index:: single: generating static asset urls single: static asset urls diff --git a/src/pyramid/config/views.py b/src/pyramid/config/views.py index afb685f93..509a64b2b 100644 --- a/src/pyramid/config/views.py +++ b/src/pyramid/config/views.py @@ -1943,6 +1943,16 @@ class ViewsConfiguratorMixin(object): prefix*. By default, this argument is ``None``, meaning that no particular Expires or Cache-Control headers are set in the response. + The ``content_encodings`` keyword argument is a list of alternative + file encodings supported in the ``Accept-Encoding`` HTTP Header. + Alternative files are found using file extensions defined in + :attr:`mimetypes.encodings_map`. An encoded asset will be returned + with the ``Content-Encoding`` header set to the selected encoding. + If the asset contains alternative encodings then the + ``Accept-Encoding`` value will be added to the response's ``Vary`` + header. By default, the list is empty and no alternatives will be + supported. + The ``permission`` keyword argument is used to specify the :term:`permission` required by a user to execute the static view. By default, it is the string @@ -2019,6 +2029,11 @@ class ViewsConfiguratorMixin(object): static_url('mypackage:images/logo.png', request) See :ref:`static_assets_section` for more information. + + .. versionchanged:: 2.0 + + Added the ``content_encodings`` argument. + """ spec = self._make_spec(path) info = self._get_static_info() @@ -2191,10 +2206,15 @@ class StaticURLInfo(object): # it's a view name url = None cache_max_age = extra.pop('cache_max_age', None) + content_encodings = extra.pop('content_encodings', []) # create a view view = static_view( - spec, cache_max_age=cache_max_age, use_subpath=True + spec, + cache_max_age=cache_max_age, + use_subpath=True, + reload=config.registry.settings['pyramid.reload_assets'], + content_encodings=content_encodings, ) # Mutate extra to allow factory, etc to be passed through here. diff --git a/src/pyramid/static.py b/src/pyramid/static.py index e3561e93e..7870b803e 100644 --- a/src/pyramid/static.py +++ b/src/pyramid/static.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- from functools import lru_cache import json +import mimetypes import os -from os.path import getmtime, normcase, normpath, join, isdir, exists +from os.path import getmtime, getsize, normcase, normpath, join, isdir, exists from pkg_resources import resource_exists, resource_filename, resource_isdir @@ -53,6 +54,19 @@ class static_view(object): the static application will consider request.environ[``PATH_INFO``] as ``PATH_INFO`` input. By default, this is ``False``. + ``reload`` controls whether a cache of files is maintained or the asset + subsystem is queried per-request to determine what files are available. + By default, this is ``False`` and new files added while the process is + running are not recognized. + + ``content_encodings`` is a list of alternative file encodings supported + in the ``Accept-Encoding`` HTTP Header. Alternative files are found using + file extensions defined in :attr:`mimetypes.encodings_map`. An encoded + asset will be returned with the ``Content-Encoding`` header set to the + selected encoding. If the asset contains alternative encodings then the + ``Accept-Encoding`` value will be added to the response's ``Vary`` header. + By default, the list is empty and no alternatives will be supported. + .. note:: If the ``root_dir`` is relative to a :term:`package`, or is a @@ -61,6 +75,11 @@ class static_view(object): assets within the named ``root_dir`` package-relative directory. However, if the ``root_dir`` is absolute, configuration will not be able to override the assets it contains. + + .. versionchanged:: 2.0 + + Added ``reload`` and ``content_encodings`` options. + """ def __init__( @@ -70,6 +89,8 @@ class static_view(object): package_name=None, use_subpath=False, index='index.html', + reload=False, + content_encodings=(), ): # package_name is for bw compat; it is preferred to pass in a # package-relative path as root_dir @@ -83,8 +104,36 @@ class static_view(object): self.docroot = docroot self.norm_docroot = normcase(normpath(docroot)) self.index = index + self.reload = reload + self.content_encodings = _compile_content_encodings(content_encodings) + self.filemap = {} def __call__(self, context, request): + resource_name = self.get_resource_name(request) + files = self.get_possible_files(resource_name) + filepath, content_encoding = self.find_best_match(request, files) + if filepath is None: + raise HTTPNotFound(request.url) + + content_type, _ = _guess_type(resource_name) + response = FileResponse( + filepath, + request, + self.cache_max_age, + content_type, + content_encoding, + ) + if len(files) > 1: + _add_vary(response, 'Accept-Encoding') + return response + + def get_resource_name(self, request): + """ + Return the computed name of the requested resource. + + The returned file is not guaranteed to exist. + + """ if self.use_subpath: path_tuple = request.subpath else: @@ -94,46 +143,126 @@ class static_view(object): if path is None: raise HTTPNotFound('Out of bounds: %s' % request.url) + # normalize asset spec or fs path into resource_path if self.package_name: # package resource resource_path = '%s/%s' % (self.docroot.rstrip('/'), path) if resource_isdir(self.package_name, resource_path): if not request.path_url.endswith('/'): - self.add_slash_redirect(request) + raise self.add_slash_redirect(request) resource_path = '%s/%s' % ( resource_path.rstrip('/'), self.index, ) - if not resource_exists(self.package_name, resource_path): - raise HTTPNotFound(request.url) - filepath = resource_filename(self.package_name, resource_path) - else: # filesystem file - # os.path.normpath converts / to \ on windows - filepath = normcase(normpath(join(self.norm_docroot, path))) - if isdir(filepath): + resource_path = normcase(normpath(join(self.norm_docroot, path))) + if isdir(resource_path): if not request.path_url.endswith('/'): - self.add_slash_redirect(request) - filepath = join(filepath, self.index) - if not exists(filepath): - raise HTTPNotFound(request.url) + raise self.add_slash_redirect(request) + resource_path = join(resource_path, self.index) - content_type, content_encoding = _guess_type(filepath) - return FileResponse( - filepath, - request, - self.cache_max_age, - content_type, - content_encoding=None, - ) + return resource_path + + def find_resource_path(self, name): + """ + Return the absolute path to the resource or ``None`` if it doesn't + exist. + + """ + if self.package_name: + if resource_exists(self.package_name, name): + return resource_filename(self.package_name, name) + + elif exists(name): + return name + + def get_possible_files(self, resource_name): + """ Return a sorted list of ``(size, encoding, path)`` entries.""" + result = self.filemap.get(resource_name) + if result is not None: + return result + + # XXX we could put a lock around this work but worst case scenario a + # couple requests scan the disk for files at the same time and then + # the cache is set going forward so do not bother + result = [] + + # add the identity + path = self.find_resource_path(resource_name) + if path: + result.append((path, None)) + + # add each file we find for the supported encodings + # we don't mind adding multiple files for the same encoding if there + # are copies with different extensions because we sort by size so the + # smallest is always found first and the rest ignored + for encoding, extensions in self.content_encodings.items(): + for ext in extensions: + encoded_name = resource_name + ext + path = self.find_resource_path(encoded_name) + if path: + result.append((path, encoding)) + + # sort the files by size, smallest first + result.sort(key=lambda x: getsize(x[0])) + + # only cache the results if reload is disabled + if not self.reload: + self.filemap[resource_name] = result + return result + + def find_best_match(self, request, files): + """ Return ``(path | None, encoding)``.""" + # if the client did not specify encodings then assume only the + # identity is acceptable + if not request.accept_encoding: + identity_path = next( + (path for path, encoding in files if encoding is None), None, + ) + return identity_path, None + + # find encodings the client will accept + acceptable_encodings = { + x[0] + for x in request.accept_encoding.acceptable_offers( + [encoding for path, encoding in files if encoding is not None] + ) + } + acceptable_encodings.add(None) + + # return the smallest file from the acceptable encodings + # we know that files is sorted by size, smallest first + for path, encoding in files: + if encoding in acceptable_encodings: + return path, encoding def add_slash_redirect(self, request): url = request.path_url + '/' qs = request.query_string if qs: url = url + '?' + qs - raise HTTPMovedPermanently(url) + return HTTPMovedPermanently(url) + + +def _compile_content_encodings(encodings): + """ + Convert mimetypes.encodings_map into a dict of + ``(encoding) -> [file extensions]``. + + """ + result = {} + for ext, encoding in mimetypes.encodings_map.items(): + if encoding in encodings: + result.setdefault(encoding, []).append(ext) + return result + + +def _add_vary(response, option): + vary = response.vary or [] + if not any(x.lower() == option.lower() for x in vary): + vary.append(option) + response.vary = vary _seps = set(['/', os.sep]) diff --git a/tests/fixtures/static/encoded.html b/tests/fixtures/static/encoded.html new file mode 100644 index 000000000..0999b4f1b --- /dev/null +++ b/tests/fixtures/static/encoded.html @@ -0,0 +1,15 @@ + + + + +A Simple HTML Document + + + +

This is a very simple HTML document

+

It only has two paragraphs

+ + diff --git a/tests/fixtures/static/encoded.html.gz b/tests/fixtures/static/encoded.html.gz new file mode 100644 index 000000000..afcc25768 Binary files /dev/null and b/tests/fixtures/static/encoded.html.gz differ diff --git a/tests/pkgs/static_encodings/__init__.py b/tests/pkgs/static_encodings/__init__.py new file mode 100644 index 000000000..3b86a1a15 --- /dev/null +++ b/tests/pkgs/static_encodings/__init__.py @@ -0,0 +1,2 @@ +def includeme(config): + config.add_static_view('/', 'tests:fixtures', content_encodings=['gzip']) diff --git a/tests/test_config/test_views.py b/tests/test_config/test_views.py index a1e975756..baa87dd6b 100644 --- a/tests/test_config/test_views.py +++ b/tests/test_config/test_views.py @@ -4174,7 +4174,9 @@ class DummyRegistry: utility = None def __init__(self): - self.settings = {} + self.settings = { + 'pyramid.reload_assets': False, + } def queryUtility(self, type_or_iface, name=None, default=None): return self.utility or default diff --git a/tests/test_integration.py b/tests/test_integration.py index 331542d7d..8a4575d7b 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -73,8 +73,8 @@ class IntegrationBase(object): root_factory=self.root_factory, package=self.package ) config.include(self.package) - app = config.make_wsgi_app() - self.testapp = TestApp(app) + self.app = config.make_wsgi_app() + self.testapp = TestApp(self.app) self.config = config def tearDown(self): @@ -227,6 +227,57 @@ class TestStaticAppUsingAssetSpec(StaticAppBase, unittest.TestCase): package = 'tests.pkgs.static_assetspec' +class TestStaticAppWithEncodings(IntegrationBase, unittest.TestCase): + package = 'tests.pkgs.static_encodings' + + # XXX webtest actually runs response.decode_content() and so we can't + # use it to test gzip- or deflate-encoded responses to see if they + # were transferred correctly + def _getResponse(self, *args, **kwargs): + from pyramid.request import Request + + req = Request.blank(*args, **kwargs) + return req.get_response(self.app) + + def test_no_accept(self): + res = self._getResponse('/static/encoded.html') + self.assertEqual(res.headers['Vary'], 'Accept-Encoding') + self.assertNotIn('Content-Encoding', res.headers) + _assertBody( + res.body, os.path.join(here, 'fixtures/static/encoded.html') + ) + + def test_unsupported_accept(self): + res = self._getResponse( + '/static/encoded.html', + headers={'Accept-Encoding': 'br, foo, bar'}, + ) + self.assertEqual(res.headers['Vary'], 'Accept-Encoding') + self.assertNotIn('Content-Encoding', res.headers) + _assertBody( + res.body, os.path.join(here, 'fixtures/static/encoded.html') + ) + + def test_accept_gzip(self): + res = self._getResponse( + '/static/encoded.html', + headers={'Accept-Encoding': 'br, foo, gzip'}, + ) + self.assertEqual(res.headers['Vary'], 'Accept-Encoding') + self.assertEqual(res.headers['Content-Encoding'], 'gzip') + _assertBody( + res.body, os.path.join(here, 'fixtures/static/encoded.html.gz') + ) + + def test_accept_gzip_returns_identity(self): + res = self._getResponse( + '/static/index.html', headers={'Accept-Encoding': 'gzip'} + ) + self.assertNotIn('Vary', res.headers) + self.assertNotIn('Content-Encoding', res.headers) + _assertBody(res.body, os.path.join(here, 'fixtures/static/index.html')) + + class TestStaticAppNoSubpath(unittest.TestCase): staticapp = static_view(os.path.join(here, 'fixtures'), use_subpath=False) diff --git a/tests/test_static.py b/tests/test_static.py index a323b1d89..3d0deda3f 100644 --- a/tests/test_static.py +++ b/tests/test_static.py @@ -39,6 +39,8 @@ class Test_static_view_use_subpath_False(unittest.TestCase): self.assertEqual(inst.docroot, 'resource_name') self.assertEqual(inst.cache_max_age, 3600) self.assertEqual(inst.index, 'index.html') + self.assertEqual(inst.reload, False) + self.assertEqual(inst.content_encodings, {}) def test_call_adds_slash_path_info_empty(self): inst = self._makeOne('tests:fixtures/static') @@ -252,6 +254,8 @@ class Test_static_view_use_subpath_True(unittest.TestCase): self.assertEqual(inst.docroot, 'resource_name') self.assertEqual(inst.cache_max_age, 3600) self.assertEqual(inst.index, 'index.html') + self.assertEqual(inst.reload, False) + self.assertEqual(inst.content_encodings, {}) def test_call_adds_slash_path_info_empty(self): inst = self._makeOne('tests:fixtures/static') @@ -403,6 +407,70 @@ class Test_static_view_use_subpath_True(unittest.TestCase): self.assertRaises(HTTPNotFound, inst, context, request) +class Test_static_view_content_encodings(unittest.TestCase): + def _getTargetClass(self): + from pyramid.static import static_view + + return static_view + + def _makeOne(self, *arg, **kw): + return self._getTargetClass()(*arg, **kw) + + def _makeRequest(self, kw=None): + from pyramid.request import Request + + environ = { + 'wsgi.url_scheme': 'http', + 'wsgi.version': (1, 0), + 'SERVER_NAME': 'example.com', + 'SERVER_PORT': '6543', + 'PATH_INFO': '/', + 'SCRIPT_NAME': '', + 'REQUEST_METHOD': 'GET', + } + if kw is not None: + environ.update(kw) + return Request(environ=environ) + + def test_call_without_accept(self): + inst = self._makeOne( + 'tests:fixtures/static', content_encodings=['gzip'] + ) + request = self._makeRequest({'PATH_INFO': '/encoded.html'}) + context = DummyContext() + + res = inst(context, request) + self.assertEqual(res.headers['Vary'], 'Accept-Encoding') + self.assertNotIn('Content-Encoding', res.headers) + self.assertEqual(len(res.body), 221) + + def test_call_with_accept_gzip(self): + inst = self._makeOne( + 'tests:fixtures/static', content_encodings=['gzip'] + ) + request = self._makeRequest( + {'PATH_INFO': '/encoded.html', 'HTTP_ACCEPT_ENCODING': 'gzip'} + ) + context = DummyContext() + + res = inst(context, request) + self.assertEqual(res.headers['Vary'], 'Accept-Encoding') + self.assertEqual(res.headers['Content-Encoding'], 'gzip') + self.assertEqual(len(res.body), 187) + + def test_call_get_possible_files_is_cached(self): + inst = self._makeOne('tests:fixtures/static') + result1 = inst.get_possible_files('tests:fixtures/static/encoded.html') + result2 = inst.get_possible_files('tests:fixtures/static/encoded.html') + self.assertIs(result1, result2) + + def test_call_get_possible_files_is_not_cached(self): + inst = self._makeOne('tests:fixtures/static', reload=True) + result1 = inst.get_possible_files('tests:fixtures/static/encoded.html') + result2 = inst.get_possible_files('tests:fixtures/static/encoded.html') + self.assertIsNot(result1, result2) + + class TestQueryStringConstantCacheBuster(unittest.TestCase): def _makeOne(self, param=None): from pyramid.static import QueryStringConstantCacheBuster as cls -- cgit v1.2.3 From 8fe8725d4d20b18715291d3b45899d3389fe1d8b Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Sun, 15 Dec 2019 20:46:12 -0600 Subject: handle a missing content negotiation case where the unencoded option is not available and the client requests an encoded variant that doesn't exist --- src/pyramid/static.py | 1 + tests/fixtures/static/only_encoded.html.gz | Bin 0 -> 187 bytes tests/test_static.py | 81 +++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 tests/fixtures/static/only_encoded.html.gz diff --git a/src/pyramid/static.py b/src/pyramid/static.py index 7870b803e..499706554 100644 --- a/src/pyramid/static.py +++ b/src/pyramid/static.py @@ -236,6 +236,7 @@ class static_view(object): for path, encoding in files: if encoding in acceptable_encodings: return path, encoding + return None, None def add_slash_redirect(self, request): url = request.path_url + '/' diff --git a/tests/fixtures/static/only_encoded.html.gz b/tests/fixtures/static/only_encoded.html.gz new file mode 100644 index 000000000..afcc25768 Binary files /dev/null and b/tests/fixtures/static/only_encoded.html.gz differ diff --git a/tests/test_static.py b/tests/test_static.py index 3d0deda3f..73814e222 100644 --- a/tests/test_static.py +++ b/tests/test_static.py @@ -458,6 +458,87 @@ class Test_static_view_content_encodings(unittest.TestCase): self.assertEqual(res.headers['Content-Encoding'], 'gzip') self.assertEqual(len(res.body), 187) + def test_call_for_encoded_variant_without_unencoded_variant_no_accept( + self, + ): + inst = self._makeOne( + 'tests:fixtures/static', content_encodings=['gzip'] + ) + request = self._makeRequest({'PATH_INFO': '/only_encoded.html.gz'}) + context = DummyContext() + + res = inst(context, request) + self.assertNotIn('Vary', res.headers) + self.assertNotIn('Content-Encoding', res.headers) + self.assertEqual(len(res.body), 187) + + def test_call_for_encoded_variant_without_unencoded_variant_with_accept( + self, + ): + inst = self._makeOne( + 'tests:fixtures/static', content_encodings=['gzip'] + ) + request = self._makeRequest( + { + 'PATH_INFO': '/only_encoded.html.gz', + 'HTTP_ACCEPT_ENCODING': 'gzip', + } + ) + context = DummyContext() + + res = inst(context, request) + self.assertNotIn('Vary', res.headers) + self.assertNotIn('Content-Encoding', res.headers) + self.assertEqual(len(res.body), 187) + + def test_call_for_unencoded_variant_with_only_encoded_variant_no_accept( + self, + ): + inst = self._makeOne( + 'tests:fixtures/static', content_encodings=['gzip'] + ) + request = self._makeRequest({'PATH_INFO': '/only_encoded.html'}) + context = DummyContext() + + res = inst(context, request) + self.assertNotIn('Vary', res.headers) + self.assertNotIn('Content-Encoding', res.headers) + self.assertEqual(len(res.body), 187) + + def test_call_for_unencoded_variant_with_only_encoded_variant_with_accept( + self, + ): + inst = self._makeOne( + 'tests:fixtures/static', content_encodings=['gzip'] + ) + request = self._makeRequest( + { + 'PATH_INFO': '/only_encoded.html', + 'HTTP_ACCEPT_ENCODING': 'gzip', + } + ) + context = DummyContext() + + res = inst(context, request) + self.assertNotIn('Vary', res.headers) + self.assertEqual(res.headers['Content-Encoding'], 'gzip') + self.assertEqual(len(res.body), 187) + + def test_call_for_unencoded_variant_with_only_encoded_variant_bad_accept( + self, + ): + from pyramid.httpexceptions import HTTPNotFound + + inst = self._makeOne( + 'tests:fixtures/static', content_encodings=['gzip'] + ) + request = self._makeRequest( + {'PATH_INFO': '/only_encoded.html', 'HTTP_ACCEPT_ENCODING': 'br',} + ) + context = DummyContext() + + self.assertRaises(HTTPNotFound, lambda: inst(context, request)) + def test_call_get_possible_files_is_cached(self): inst = self._makeOne('tests:fixtures/static') result1 = inst.get_possible_files('tests:fixtures/static/encoded.html') -- cgit v1.2.3 From 08933ad17a230792a886020e06725e44522974d4 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Sun, 15 Dec 2019 20:52:01 -0600 Subject: fix lint --- tests/test_static.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_static.py b/tests/test_static.py index 73814e222..2933e4310 100644 --- a/tests/test_static.py +++ b/tests/test_static.py @@ -533,7 +533,7 @@ class Test_static_view_content_encodings(unittest.TestCase): 'tests:fixtures/static', content_encodings=['gzip'] ) request = self._makeRequest( - {'PATH_INFO': '/only_encoded.html', 'HTTP_ACCEPT_ENCODING': 'br',} + {'PATH_INFO': '/only_encoded.html', 'HTTP_ACCEPT_ENCODING': 'br'} ) context = DummyContext() -- cgit v1.2.3 From 497e667e41b83193d7c5a4f74e9419320d755c46 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Sun, 15 Dec 2019 20:53:52 -0600 Subject: fix failing test --- tests/test_static.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_static.py b/tests/test_static.py index 2933e4310..7b6e74a64 100644 --- a/tests/test_static.py +++ b/tests/test_static.py @@ -494,16 +494,15 @@ class Test_static_view_content_encodings(unittest.TestCase): def test_call_for_unencoded_variant_with_only_encoded_variant_no_accept( self, ): + from pyramid.httpexceptions import HTTPNotFound + inst = self._makeOne( 'tests:fixtures/static', content_encodings=['gzip'] ) request = self._makeRequest({'PATH_INFO': '/only_encoded.html'}) context = DummyContext() - res = inst(context, request) - self.assertNotIn('Vary', res.headers) - self.assertNotIn('Content-Encoding', res.headers) - self.assertEqual(len(res.body), 187) + self.assertRaises(HTTPNotFound, lambda: inst(context, request)) def test_call_for_unencoded_variant_with_only_encoded_variant_with_accept( self, -- cgit v1.2.3 From 5e7f4e10f45e78afe04e18c9f904aebff2d9d6b4 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Sun, 15 Dec 2019 20:24:56 -0600 Subject: set the minimum supported version at py36 --- .travis.yml | 10 ++-------- HACKING.txt | 5 ++--- appveyor.yml | 10 ++++------ setup.py | 2 +- tox.ini | 2 +- 5 files changed, 10 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index c4860d2de..550c79611 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,17 +4,13 @@ sudo: false matrix: include: - - python: 3.4 - env: TOXENV=py34 - - python: 3.5 - env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 - python: pypy3 env: TOXENV=pypy3 - python: 3.6 env: TOXENV=py36-cover,coverage - - python: 3.5 + - python: 3.6 env: TOXENV=docs - python: 3.6 env: TOXENV=lint @@ -22,12 +18,10 @@ matrix: env: TOXENV=py37 dist: xenial sudo: true - - python: 3.8-dev + - python: 3.8 env: TOXENV=py38 dist: xenial sudo: true - allow_failures: - - env: TOXENV=py38 install: - travis_retry pip install tox diff --git a/HACKING.txt b/HACKING.txt index 5ccc318de..fbfd0ecd0 100644 --- a/HACKING.txt +++ b/HACKING.txt @@ -34,8 +34,7 @@ In order to add a feature to Pyramid: - The feature must be documented in both the API and narrative documentation (in `docs/`). -- The feature must work fully on the following CPython versions: 3.4, 3.5, 3.6, - and 3.7 on both UNIX and Windows. +- The feature must work fully on the following CPython versions: 3.6, 3.7, and 3.8 on both UNIX and Windows. - The feature must work on the latest version of PyPy3. @@ -67,7 +66,7 @@ Running Tests This command will run tests on the latest version of Python 3 with coverage. - $ tox -e py3-cover,coverage + $ tox -e py36-cover,coverage - To run individual tests (i.e., during development), you can use `nosetests` syntax as follows, where `$VENV` is an environment variable set to the path diff --git a/appveyor.yml b/appveyor.yml index a9bcd40f1..d0d22146d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,13 +1,11 @@ environment: matrix: - - PYTHON: "C:\\Python37" - TOXENV: "py37" - PYTHON: "C:\\Python36" TOXENV: "py36" - - PYTHON: "C:\\Python35" - TOXENV: "py35" - - PYTHON: "C:\\Python34" - TOXENV: "py34" + - PYTHON: "C:\\Python37" + TOXENV: "py37" + - PYTHON: "C:\\Python38" + TOXENV: "py38" cache: - '%LOCALAPPDATA%\pip\Cache' diff --git a/setup.py b/setup.py index 4db78d158..189f0700f 100644 --- a/setup.py +++ b/setup.py @@ -94,7 +94,7 @@ setup( package_dir={'': 'src'}, include_package_data=True, zip_safe=False, - python_requires='>=3.4', + python_requires='>=3.6', install_requires=install_requires, extras_require={'testing': testing_extras, 'docs': docs_extras}, tests_require=tests_require, diff --git a/tox.ini b/tox.ini index 441a118a8..34eb11317 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] envlist = lint, - py34,py35,py36,py37,pypy3, + py36,py37,pypy3, docs,py36-cover,coverage, [testenv] -- cgit v1.2.3 From 3f42e5f2c848e69f282337f6fbf56e69b295b58e Mon Sep 17 00:00:00 2001 From: Steve Piercy Date: Sun, 15 Dec 2019 23:14:19 -0800 Subject: Remove Python 3.4, Python 3.5, and add Python 3.8 --- docs/narr/install.rst | 6 +++--- docs/narr/logging.rst | 2 +- docs/narr/upgrading.rst | 6 +++--- docs/quick_tutorial/requirements.rst | 4 ++-- docs/tutorials/modwsgi/index.rst | 2 +- docs/tutorials/wiki/installation.rst | 2 -- docs/tutorials/wiki/tests.rst | 3 --- setup.py | 3 +-- 8 files changed, 11 insertions(+), 17 deletions(-) diff --git a/docs/narr/install.rst b/docs/narr/install.rst index 268ae5f8d..903769b59 100644 --- a/docs/narr/install.rst +++ b/docs/narr/install.rst @@ -5,7 +5,7 @@ Installing :app:`Pyramid` .. note:: - This installation guide emphasizes the use of Python 3.4 and greater for + This installation guide emphasizes the use of Python 3.6 and greater for simplicity. @@ -15,13 +15,13 @@ Installing :app:`Pyramid` Before You Install Pyramid -------------------------- -Install Python version 3.4 or greater for your operating system, and satisfy +Install Python version 3.6 or greater for your operating system, and satisfy the :ref:`requirements-for-installing-packages`, as described in the following sections. .. sidebar:: Python Versions - As of this writing, :app:`Pyramid` is tested against Python 3.4, Python 3.5, Python 3.6, Python 3.7, Python 3.8 (with allowed failures), and PyPy3. + As of this writing, :app:`Pyramid` is tested against Python 3.6, Python 3.7, Python 3.8, and PyPy3. :app:`Pyramid` is known to run on all popular Unix-like systems such as Linux, macOS, and FreeBSD, as well as on Windows platforms. It is also known to diff --git a/docs/narr/logging.rst b/docs/narr/logging.rst index 58bd2d4ec..844128758 100644 --- a/docs/narr/logging.rst +++ b/docs/narr/logging.rst @@ -32,7 +32,7 @@ you to send messages to :mod:`Python standard library logging package :term:`PasteDeploy` ``development.ini`` and ``production.ini`` files created when you use our cookiecutter include a basic configuration for the Python :mod:`logging` package. -These ``.ini`` file sections are passed to the `logging module's config file configuration engine `_. +These ``.ini`` file sections are passed to the `logging module's config file configuration engine `_. PasteDeploy ``.ini`` files use the Python standard library :mod:`ConfigParser format `. This is the same format used as the Python diff --git a/docs/narr/upgrading.rst b/docs/narr/upgrading.rst index af552741c..31ca6adfa 100644 --- a/docs/narr/upgrading.rst +++ b/docs/narr/upgrading.rst @@ -86,10 +86,10 @@ At the time of a Pyramid version release, each supports all versions of Python through the end of their lifespans. The end-of-life for a given version of Python is when security updates are no longer released. -- `Python 3.4 Lifespan `_ 2019-03-16 . -- `Python 3.5 Lifespan `_ 2020-09-13 . +- `Python 3.5 Lifespan `_ 2020-09-13. - `Python 3.6 Lifespan `_ 2021-12-23. -- `Python 3.7 Lifespan `_ 2023-06-27 . +- `Python 3.7 Lifespan `_ 2023-06-27. +- `Python 3.8 Lifespan `_ 2024-10-??. To determine the Python support for a specific release of Pyramid, view its ``tox.ini`` file at the root of the repository's version. diff --git a/docs/quick_tutorial/requirements.rst b/docs/quick_tutorial/requirements.rst index 2ed9b8b55..901f6134d 100644 --- a/docs/quick_tutorial/requirements.rst +++ b/docs/quick_tutorial/requirements.rst @@ -19,8 +19,8 @@ virtual environment.) This *Quick Tutorial* is based on: -* **Python 3.7**. Pyramid fully supports Python 3.4+. - This tutorial uses **Python 3.7**. +* **Python 3.8**. Pyramid fully supports Python 3.6+. + This tutorial uses **Python 3.8**. * **venv**. We believe in virtual environments. For this tutorial, we use Python 3's built-in solution :term:`venv`. diff --git a/docs/tutorials/modwsgi/index.rst b/docs/tutorials/modwsgi/index.rst index fa0d4f0cb..be72c014c 100644 --- a/docs/tutorials/modwsgi/index.rst +++ b/docs/tutorials/modwsgi/index.rst @@ -117,7 +117,7 @@ specific path information for commands and files. WSGIApplicationGroup %{GLOBAL} WSGIPassAuthorization On WSGIDaemonProcess pyramid user=chrism group=staff threads=4 \ - python-path=/Users/chrism/myproject/env/lib/python3.5/site-packages + python-path=/Users/chrism/myproject/env/lib/python3.8/site-packages WSGIScriptAlias /myapp /Users/chrism/myproject/pyramid.wsgi diff --git a/docs/tutorials/wiki/installation.rst b/docs/tutorials/wiki/installation.rst index 37e3498b2..cfa021540 100644 --- a/docs/tutorials/wiki/installation.rst +++ b/docs/tutorials/wiki/installation.rst @@ -127,8 +127,6 @@ On Unix On Windows ^^^^^^^^^^ -Python 3.7: - .. code-block:: doscon python -m venv %VENV% diff --git a/docs/tutorials/wiki/tests.rst b/docs/tutorials/wiki/tests.rst index a0872e605..9dacc5f96 100644 --- a/docs/tutorials/wiki/tests.rst +++ b/docs/tutorials/wiki/tests.rst @@ -73,6 +73,3 @@ The expected result should look like the following: ......................... 25 passed in 6.87 seconds - -If you use Python 3.7, you may see deprecation warnings from the docutils 0.14 package. -You can apply a [patch](https://sourceforge.net/p/docutils/patches/144/) to fix the issue, or ignore it and wait for the next release of docutils. diff --git a/setup.py b/setup.py index 189f0700f..02abf6471 100644 --- a/setup.py +++ b/setup.py @@ -69,10 +69,9 @@ setup( "Intended Audience :: Developers", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Framework :: Pyramid", -- cgit v1.2.3 From 4ec93ffe23cbf3acbbfc68236696ee474ddf73ba Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Mon, 23 Dec 2019 11:56:55 -0600 Subject: Update tox.ini Co-Authored-By: Steve Piercy --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 34eb11317..f2a2b7243 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] envlist = lint, - py36,py37,pypy3, + py36,py37,py38,pypy3, docs,py36-cover,coverage, [testenv] -- cgit v1.2.3 From e268aa622f57fd76b06fc6203448143f4fbe7652 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Mon, 23 Dec 2019 12:08:50 -0600 Subject: restore py35 --- .travis.yml | 2 ++ HACKING.txt | 2 +- appveyor.yml | 2 ++ docs/narr/install.rst | 6 +++--- docs/quick_tutorial/requirements.rst | 2 +- setup.py | 3 ++- tox.ini | 2 +- 7 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 550c79611..c762c085b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,8 @@ sudo: false matrix: include: + - python: 3.5 + env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 - python: pypy3 diff --git a/HACKING.txt b/HACKING.txt index fbfd0ecd0..492b8675b 100644 --- a/HACKING.txt +++ b/HACKING.txt @@ -34,7 +34,7 @@ In order to add a feature to Pyramid: - The feature must be documented in both the API and narrative documentation (in `docs/`). -- The feature must work fully on the following CPython versions: 3.6, 3.7, and 3.8 on both UNIX and Windows. +- The feature must work fully on the following CPython versions: 3.5, 3.6, 3.7, and 3.8 on both UNIX and Windows. - The feature must work on the latest version of PyPy3. diff --git a/appveyor.yml b/appveyor.yml index d0d22146d..ba07274f8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,5 +1,7 @@ environment: matrix: + - PYTHON: "C:\\Python35" + TOXENV: "py35" - PYTHON: "C:\\Python36" TOXENV: "py36" - PYTHON: "C:\\Python37" diff --git a/docs/narr/install.rst b/docs/narr/install.rst index 903769b59..8e2bfa866 100644 --- a/docs/narr/install.rst +++ b/docs/narr/install.rst @@ -5,7 +5,7 @@ Installing :app:`Pyramid` .. note:: - This installation guide emphasizes the use of Python 3.6 and greater for + This installation guide emphasizes the use of Python 3.5 and greater for simplicity. @@ -15,13 +15,13 @@ Installing :app:`Pyramid` Before You Install Pyramid -------------------------- -Install Python version 3.6 or greater for your operating system, and satisfy +Install Python version 3.5 or greater for your operating system, and satisfy the :ref:`requirements-for-installing-packages`, as described in the following sections. .. sidebar:: Python Versions - As of this writing, :app:`Pyramid` is tested against Python 3.6, Python 3.7, Python 3.8, and PyPy3. + As of this writing, :app:`Pyramid` is tested against Python 3.5, 3.6, Python 3.7, Python 3.8, and PyPy3. :app:`Pyramid` is known to run on all popular Unix-like systems such as Linux, macOS, and FreeBSD, as well as on Windows platforms. It is also known to diff --git a/docs/quick_tutorial/requirements.rst b/docs/quick_tutorial/requirements.rst index 901f6134d..fd1726dbd 100644 --- a/docs/quick_tutorial/requirements.rst +++ b/docs/quick_tutorial/requirements.rst @@ -19,7 +19,7 @@ virtual environment.) This *Quick Tutorial* is based on: -* **Python 3.8**. Pyramid fully supports Python 3.6+. +* **Python 3.8**. Pyramid fully supports Python 3.5+. This tutorial uses **Python 3.8**. * **venv**. We believe in virtual environments. diff --git a/setup.py b/setup.py index 02abf6471..2487d0952 100644 --- a/setup.py +++ b/setup.py @@ -69,6 +69,7 @@ setup( "Intended Audience :: Developers", "Programming Language :: Python", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", @@ -93,7 +94,7 @@ setup( package_dir={'': 'src'}, include_package_data=True, zip_safe=False, - python_requires='>=3.6', + python_requires='>=3.5', install_requires=install_requires, extras_require={'testing': testing_extras, 'docs': docs_extras}, tests_require=tests_require, diff --git a/tox.ini b/tox.ini index f2a2b7243..1d68122f4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] envlist = lint, - py36,py37,py38,pypy3, + py35,py36,py37,py38,pypy3, docs,py36-cover,coverage, [testenv] -- cgit v1.2.3 From 4f78c131ebc39dba81c20deb54d656c6ca4c6079 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Mon, 23 Dec 2019 12:32:21 -0600 Subject: update changelog for #3547 --- CHANGES.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 0e6d69252..794301578 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,9 @@ unreleased Features -------- +- Add support for Python 3.8. + See https://github.com/Pylons/pyramid/pull/3547 + - Changed the default ``serializer`` on ``pyramid.session.SignedCookieSessionFactory`` to use ``pyramid.session.JSONSerializer`` instead of @@ -76,6 +79,11 @@ Deprecations Backward Incompatibilities -------------------------- +- Drop support for Python 2.7. + +- Drop support for Python 3.4. + See https://github.com/Pylons/pyramid/pull/3547 + - ``pcreate`` and the builtin scaffolds have been removed in favor of using the ``cookiecutter`` tool and the ``pyramid-cookiecutter-starter`` cookiecutter. The script and scaffolds were deprecated in Pyramid 1.8. -- cgit v1.2.3 From 912dc539ca793959d7465995f906279dad21ccc9 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Mon, 23 Dec 2019 14:07:04 -0600 Subject: fix links in docs to cookiecutter --- docs/conf.py | 1 - docs/glossary.rst | 2 +- docs/narr/cookiecutters.rst | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 9f2b56225..365af5fdb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -64,7 +64,6 @@ extensions = [ intersphinx_mapping = { 'colander': ('https://docs.pylonsproject.org/projects/colander/en/latest/', None), 'cookbook': ('https://docs.pylonsproject.org/projects/pyramid-cookbook/en/latest/', None), - 'cookiecutter': ('https://cookiecutter.readthedocs.io/en/latest/', None), 'deform': ('https://docs.pylonsproject.org/projects/deform/en/latest/', None), 'jinja2': ('https://docs.pylonsproject.org/projects/pyramid-jinja2/en/latest/', None), 'pylonswebframework': ('https://docs.pylonsproject.org/projects/pylons-webframework/en/latest/', None), diff --git a/docs/glossary.rst b/docs/glossary.rst index ac60ebd24..81358e688 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -1157,7 +1157,7 @@ Glossary packaging. cookiecutter - A command-line utility that creates projects from :ref:`cookiecutters ` (project templates), e.g., creating a Python package project from a Python package project template. + A command-line utility that creates projects from `cookiecutters `__ (project templates), e.g., creating a Python package project from a Python package project template. .. versionadded:: 1.8 Added cookiecutter support. diff --git a/docs/narr/cookiecutters.rst b/docs/narr/cookiecutters.rst index c6829056c..066d6c2e5 100644 --- a/docs/narr/cookiecutters.rst +++ b/docs/narr/cookiecutters.rst @@ -3,7 +3,7 @@ Pyramid cookiecutters ===================== -A :term:`cookiecutter` is a command-line utility that creates projects from :ref:`cookiecutters ` (project templates). +A :term:`cookiecutter` is a command-line utility that creates projects from `cookiecutters `__ (project templates). `pyramid-cookiecutter-starter `_ -- cgit v1.2.3