From 6e29b425182ccc4abc87fcfb32e20b60b15d4bdf Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Tue, 1 Dec 2015 01:59:14 -0600 Subject: initial work on config.add_cache_buster --- pyramid/config/views.py | 108 ++++++++++++++++++-------------- pyramid/interfaces.py | 45 ++++--------- pyramid/static.py | 10 ++- pyramid/tests/test_config/test_views.py | 66 +++++++++---------- 4 files changed, 106 insertions(+), 123 deletions(-) diff --git a/pyramid/config/views.py b/pyramid/config/views.py index e386bc4e1..67a70145c 100644 --- a/pyramid/config/views.py +++ b/pyramid/config/views.py @@ -1,3 +1,4 @@ +import bisect import inspect import operator import os @@ -1855,18 +1856,7 @@ class ViewsConfiguratorMixin(object): ``Expires`` and ``Cache-Control`` headers for static assets served. Note that this argument has no effect when the ``name`` is a *url prefix*. By default, this argument is ``None``, meaning that no - particular Expires or Cache-Control headers are set in the response, - unless ``cachebust`` is specified. - - The ``cachebust`` keyword argument may be set to cause - :meth:`~pyramid.request.Request.static_url` to use cache busting when - generating URLs. See :ref:`cache_busting` for general information - about cache busting. The value of the ``cachebust`` argument must - be an object which implements - :class:`~pyramid.interfaces.ICacheBuster`. If the ``cachebust`` - argument is provided, the default for ``cache_max_age`` is modified - to be ten years. ``cache_max_age`` may still be explicitly provided - to override this default. + particular Expires or Cache-Control headers are set in the response. The ``permission`` keyword argument is used to specify the :term:`permission` required by a user to execute the static view. By @@ -1946,11 +1936,32 @@ class ViewsConfiguratorMixin(object): See :ref:`static_assets_section` for more information. """ spec = self._make_spec(path) + info = self._get_static_info() + info.add(self, name, spec, **kw) + + def add_cache_buster(self, path, cachebust): + """ + The ``cachebust`` keyword argument may be set to cause + :meth:`~pyramid.request.Request.static_url` to use cache busting when + generating URLs. See :ref:`cache_busting` for general information + about cache busting. The value of the ``cachebust`` argument must + be an object which implements + :class:`~pyramid.interfaces.ICacheBuster`. If the ``cachebust`` + argument is provided, the default for ``cache_max_age`` is modified + to be ten years. ``cache_max_age`` may still be explicitly provided + to override this default. + + """ + spec = self._make_spec(path) + info = self._get_static_info() + info.add_cache_buster(self, spec, cachebust) + + def _get_static_info(self): info = self.registry.queryUtility(IStaticURLInfo) if info is None: info = StaticURLInfo() self.registry.registerUtility(info, IStaticURLInfo) - info.add(self, name, spec, **kw) + return info def isexception(o): if IInterface.providedBy(o): @@ -1964,26 +1975,18 @@ def isexception(o): @implementer(IStaticURLInfo) class StaticURLInfo(object): - def _get_registrations(self, registry): - try: - reg = registry._static_url_registrations - except AttributeError: - reg = registry._static_url_registrations = [] - return reg + def __init__(self): + self.registrations = [] + self.cache_busters = [] def generate(self, path, request, **kw): - try: - registry = request.registry - except AttributeError: # bw compat (for tests) - registry = get_current_registry() - registrations = self._get_registrations(registry) - for (url, spec, route_name, cachebust) in registrations: + for (url, spec, route_name) in self.registrations: if path.startswith(spec): subpath = path[len(spec):] if WIN: # pragma: no cover subpath = subpath.replace('\\', '/') # windows - if cachebust: - subpath, kw = cachebust(subpath, kw) + # translate spec into overridden spec and lookup cache buster + # to modify subpath, kw if url is None: kw['subpath'] = subpath return request.route_url(route_name, **kw) @@ -2023,19 +2026,6 @@ class StaticURLInfo(object): # make sure it ends with a slash name = name + '/' - if config.registry.settings.get('pyramid.prevent_cachebust'): - cb = None - else: - cb = extra.pop('cachebust', None) - if cb: - def cachebust(subpath, kw): - subpath_tuple = tuple(subpath.split('/')) - subpath_tuple, kw = cb.pregenerate( - spec + subpath, subpath_tuple, kw) - return '/'.join(subpath_tuple), kw - else: - cachebust = None - if url_parse(name).netloc: # it's a URL # url, spec, route_name @@ -2044,14 +2034,11 @@ class StaticURLInfo(object): else: # it's a view name url = None - ten_years = 10 * 365 * 24 * 60 * 60 # more or less - default = ten_years if cb else None - cache_max_age = extra.pop('cache_max_age', default) + cache_max_age = extra.pop('cache_max_age', None) # create a view - cb_match = getattr(cb, 'match', None) view = static_view(spec, cache_max_age=cache_max_age, - use_subpath=True, cachebust_match=cb_match) + use_subpath=True) # Mutate extra to allow factory, etc to be passed through here. # Treat permission specially because we'd like to default to @@ -2083,7 +2070,7 @@ class StaticURLInfo(object): ) def register(): - registrations = self._get_registrations(config.registry) + registrations = self.registrations names = [t[0] for t in registrations] @@ -2092,7 +2079,7 @@ class StaticURLInfo(object): registrations.pop(idx) # url, spec, route_name - registrations.append((url, spec, route_name, cachebust)) + registrations.append((url, spec, route_name)) intr = config.introspectable('static views', name, @@ -2102,3 +2089,30 @@ class StaticURLInfo(object): intr['spec'] = spec config.action(None, callable=register, introspectables=(intr,)) + + def add_cache_buster(self, config, spec, cachebust): + def register(): + cache_busters = self.cache_busters + + specs = [t[0] for t in cache_busters] + if spec in specs: + idx = specs.index(spec) + cache_busters.pop(idx) + + lengths = [len(t[0]) for t in cache_busters] + new_idx = bisect.bisect_left(lengths, len(spec)) + cache_busters.insert(new_idx, (spec, cachebust)) + + intr = config.introspectable('cache busters', + spec, + 'cache buster for %r' % spec, + 'cache buster') + intr['cachebust'] = cachebust + intr['spec'] = spec + + config.action(None, callable=register, introspectables=(intr,)) + + def _find_cache_buster(self, registry, spec): + for base_spec, cachebust in self.cache_busters: + if base_spec.startswith(spec): + pass diff --git a/pyramid/interfaces.py b/pyramid/interfaces.py index 90534593c..bdf5bdfbe 100644 --- a/pyramid/interfaces.py +++ b/pyramid/interfaces.py @@ -584,6 +584,9 @@ class IStaticURLInfo(Interface): def generate(path, request, **kw): """ Generate a URL for the given path """ + def add_cache_buster(config, spec, cache_buster): + """ Add a new cache buster to a particular set of assets """ + class IResponseFactory(Interface): """ A utility which generates a response """ def __call__(request): @@ -1186,45 +1189,23 @@ class IPredicateList(Interface): class ICacheBuster(Interface): """ - Instances of ``ICacheBuster`` may be provided as arguments to - :meth:`~pyramid.config.Configurator.add_static_view`. Instances of - ``ICacheBuster`` provide mechanisms for generating a cache bust token for - a static asset, modifying a static asset URL to include a cache bust token, - and, optionally, unmodifying a static asset URL in order to look up an - asset. See :ref:`cache_busting`. + A cache buster modifies the URL generation machinery for + :meth:`~pyramid.request.Request.static_url`. See :ref:`cache_busting`. .. versionadded:: 1.6 """ - def pregenerate(pathspec, subpath, kw): + def __call__(pathspec, subpath, kw): """ Modifies a subpath and/or keyword arguments from which a static asset URL will be computed during URL generation. The ``pathspec`` argument is the path specification for the resource to be cache busted. - The ``subpath`` argument is a tuple of path elements that represent the - portion of the asset URL which is used to find the asset. The ``kw`` - argument is a dict of keywords that are to be passed eventually to - :meth:`~pyramid.request.Request.route_url` for URL generation. The - return value should be a two-tuple of ``(subpath, kw)`` which are - versions of the same arguments modified to include the cache bust token - in the generated URL. - """ - - def match(subpath): - """ - Performs the logical inverse of - :meth:`~pyramid.interfaces.ICacheBuster.pregenerate` by taking a - subpath from a cache busted URL and removing the cache bust token, so - that :app:`Pyramid` can find the underlying asset. - - ``subpath`` is the subpath portion of the URL for an incoming request - for a static asset. The return value should be the same tuple with the - cache busting token elided. - - If the cache busting scheme in use doesn't specifically modify the path - portion of the generated URL (e.g. it adds a query string), a method - which implements this interface may not be necessary. It is - permissible for an instance of - :class:`~pyramid.interfaces.ICacheBuster` to omit this method. + The ``subpath`` argument is a path of ``/``-delimited segments that + represent the portion of the asset URL which is used to find the asset. + The ``kw`` argument is a dict of keywords that are to be passed + eventually to :meth:`~pyramid.request.Request.static_url` for URL + generation. The return value should be a two-tuple of + ``(subpath, kw)`` which are versions of the same arguments modified + to include the cache bust token in the generated URL. """ # configuration phases: a lower phase number means the actions associated diff --git a/pyramid/static.py b/pyramid/static.py index c7a5c7ba5..cda98bea4 100644 --- a/pyramid/static.py +++ b/pyramid/static.py @@ -179,7 +179,7 @@ class QueryStringCacheBuster(object): def __init__(self, param='x'): self.param = param - def pregenerate(self, pathspec, subpath, kw): + def __call__(self, pathspec, subpath, kw): token = self.tokenize(pathspec) query = kw.setdefault('_query', {}) if isinstance(query, dict): @@ -289,8 +289,6 @@ class ManifestCacheBuster(object): self._mtime = mtime return self._manifest - def pregenerate(self, pathspec, subpath, kw): - path = '/'.join(subpath) - path = self.manifest.get(path, path) - new_subpath = path.split('/') - return (new_subpath, kw) + def __call__(self, pathspec, subpath, kw): + subpath = self.manifest.get(subpath, subpath) + return (subpath, kw) diff --git a/pyramid/tests/test_config/test_views.py b/pyramid/tests/test_config/test_views.py index acfb81962..020ed131d 100644 --- a/pyramid/tests/test_config/test_views.py +++ b/pyramid/tests/test_config/test_views.py @@ -3865,22 +3865,11 @@ class TestStaticURLInfo(unittest.TestCase): def _makeOne(self): return self._getTargetClass()() - def _makeConfig(self, registrations=None): - config = DummyConfig() - registry = DummyRegistry() - if registrations is not None: - registry._static_url_registrations = registrations - config.registry = registry - return config - def _makeRequest(self): request = DummyRequest() request.registry = DummyRegistry() return request - def _assertRegistrations(self, config, expected): - self.assertEqual(config.registry._static_url_registrations, expected) - def test_verifyClass(self): from pyramid.interfaces import IStaticURLInfo from zope.interface.verify import verifyClass @@ -4002,12 +3991,12 @@ class TestStaticURLInfo(unittest.TestCase): 'http://example.com/abc%20def#La%20Pe%C3%B1a') def test_generate_url_cachebust(self): - def cachebust(subpath, kw): + def cachebust(request, subpath, kw): kw['foo'] = 'bar' return 'foo' + '/' + subpath, kw inst = self._makeOne() - registrations = [(None, 'package:path/', '__viewname', cachebust)] - inst._get_registrations = lambda *x: registrations + inst.registrations = [(None, 'package:path/', '__viewname', cachebust)] + inst.cache_busters = [('package:path/', cachebust)] request = self._makeRequest() def route_url(n, **kw): self.assertEqual(n, '__viewname') @@ -4016,88 +4005,88 @@ class TestStaticURLInfo(unittest.TestCase): inst.generate('package:path/abc', request) def test_add_already_exists(self): + config = DummyConfig() inst = self._makeOne() - config = self._makeConfig( - [('http://example.com/', 'package:path/', None)]) + inst.registrations = [('http://example.com/', 'package:path/', None)] inst.add(config, 'http://example.com', 'anotherpackage:path') expected = [ - ('http://example.com/', 'anotherpackage:path/', None, None)] - self._assertRegistrations(config, expected) + ('http://example.com/', 'anotherpackage:path/', None, None)] + self.assertEqual(inst.registrations, expected) def test_add_package_root(self): + config = DummyConfig() inst = self._makeOne() - config = self._makeConfig() inst.add(config, 'http://example.com', 'package:') - expected = [('http://example.com/', 'package:', None, None)] - self._assertRegistrations(config, expected) + expected = [('http://example.com/', 'package:', None, None)] + self.assertEqual(inst.registrations, expected) def test_add_url_withendslash(self): + config = DummyConfig() inst = self._makeOne() - config = self._makeConfig() inst.add(config, 'http://example.com/', 'anotherpackage:path') expected = [ ('http://example.com/', 'anotherpackage:path/', None, None)] - self._assertRegistrations(config, expected) + self.assertEqual(inst.registrations, expected) def test_add_url_noendslash(self): + config = DummyConfig() inst = self._makeOne() - config = self._makeConfig() inst.add(config, 'http://example.com', 'anotherpackage:path') expected = [ ('http://example.com/', 'anotherpackage:path/', None, None)] - self._assertRegistrations(config, expected) + self.assertEqual(inst.registrations, expected) def test_add_url_noscheme(self): + config = DummyConfig() inst = self._makeOne() - config = self._makeConfig() inst.add(config, '//example.com', 'anotherpackage:path') expected = [('//example.com/', 'anotherpackage:path/', None, None)] - self._assertRegistrations(config, expected) + self.assertEqual(inst.registrations, expected) def test_add_viewname(self): from pyramid.security import NO_PERMISSION_REQUIRED from pyramid.static import static_view - config = self._makeConfig() + config = DummyConfig() inst = self._makeOne() inst.add(config, 'view', 'anotherpackage:path', cache_max_age=1) expected = [(None, 'anotherpackage:path/', '__view/', None)] - self._assertRegistrations(config, expected) + self.assertEqual(inst.registrations, expected) self.assertEqual(config.route_args, ('__view/', 'view/*subpath')) self.assertEqual(config.view_kw['permission'], NO_PERMISSION_REQUIRED) self.assertEqual(config.view_kw['view'].__class__, static_view) def test_add_viewname_with_route_prefix(self): - config = self._makeConfig() + config = DummyConfig() config.route_prefix = '/abc' inst = self._makeOne() inst.add(config, 'view', 'anotherpackage:path',) expected = [(None, 'anotherpackage:path/', '__/abc/view/', None)] - self._assertRegistrations(config, expected) + self.assertEqual(inst.registrations, expected) self.assertEqual(config.route_args, ('__/abc/view/', 'view/*subpath')) def test_add_viewname_with_permission(self): - config = self._makeConfig() + config = DummyConfig() inst = self._makeOne() inst.add(config, 'view', 'anotherpackage:path', cache_max_age=1, permission='abc') self.assertEqual(config.view_kw['permission'], 'abc') def test_add_viewname_with_context(self): - config = self._makeConfig() + config = DummyConfig() inst = self._makeOne() inst.add(config, 'view', 'anotherpackage:path', cache_max_age=1, context=DummyContext) self.assertEqual(config.view_kw['context'], DummyContext) def test_add_viewname_with_for_(self): - config = self._makeConfig() + config = DummyConfig() inst = self._makeOne() inst.add(config, 'view', 'anotherpackage:path', cache_max_age=1, for_=DummyContext) self.assertEqual(config.view_kw['context'], DummyContext) def test_add_viewname_with_renderer(self): - config = self._makeConfig() + config = DummyConfig() inst = self._makeOne() inst.add(config, 'view', 'anotherpackage:path', cache_max_age=1, renderer='mypackage:templates/index.pt') @@ -4105,7 +4094,7 @@ class TestStaticURLInfo(unittest.TestCase): 'mypackage:templates/index.pt') def test_add_cachebust_prevented(self): - config = self._makeConfig() + config = DummyConfig() config.registry.settings['pyramid.prevent_cachebust'] = True inst = self._makeOne() inst.add(config, 'view', 'mypackage:path', cachebust=True) @@ -4113,7 +4102,7 @@ class TestStaticURLInfo(unittest.TestCase): self.assertEqual(cachebust, None) def test_add_cachebust_custom(self): - config = self._makeConfig() + config = DummyConfig() inst = self._makeOne() inst.add(config, 'view', 'mypackage:path', cachebust=DummyCacheBuster('foo')) @@ -4236,7 +4225,8 @@ class DummyMultiView: class DummyCacheBuster(object): def __init__(self, token): self.token = token - def pregenerate(self, pathspec, subpath, kw): + + def __call__(self, pathspec, subpath, kw): kw['x'] = self.token return subpath, kw -- cgit v1.2.3 From 62222d69b7b6ef573d7f52529b15285af4111f20 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Mon, 7 Dec 2015 21:11:11 -0600 Subject: support getting the file path from a FSAssetSource even if it doesn't exist --- pyramid/config/assets.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyramid/config/assets.py b/pyramid/config/assets.py index bbdf18ced..d05314384 100644 --- a/pyramid/config/assets.py +++ b/pyramid/config/assets.py @@ -262,12 +262,15 @@ class FSAssetSource(object): def __init__(self, prefix): self.prefix = prefix - def get_filename(self, resource_name): + def get_path(self, resource_name): if resource_name: path = os.path.join(self.prefix, resource_name) else: path = self.prefix + return path + def get_filename(self, resource_name): + path = self.get_path(resource_name) if os.path.exists(path): return path -- cgit v1.2.3 From d0bd5fb326d8999e3a40e6e2d121aa69cfe05476 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Mon, 7 Dec 2015 22:58:06 -0600 Subject: add a first cut at an add_cache_buster api --- pyramid/config/views.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/pyramid/config/views.py b/pyramid/config/views.py index 67a70145c..f496dfb7d 100644 --- a/pyramid/config/views.py +++ b/pyramid/config/views.py @@ -1,5 +1,5 @@ -import bisect import inspect +import posixpath import operator import os import warnings @@ -21,6 +21,7 @@ from pyramid.interfaces import ( IException, IExceptionViewClassifier, IMultiView, + IPackageOverrides, IRendererFactory, IRequest, IResponse, @@ -1987,6 +1988,8 @@ class StaticURLInfo(object): subpath = subpath.replace('\\', '/') # windows # translate spec into overridden spec and lookup cache buster # to modify subpath, kw + subpath, kw = self._bust_asset_path( + request.registry, spec, subpath, kw) if url is None: kw['subpath'] = subpath return request.route_url(route_name, **kw) @@ -2099,9 +2102,7 @@ class StaticURLInfo(object): idx = specs.index(spec) cache_busters.pop(idx) - lengths = [len(t[0]) for t in cache_busters] - new_idx = bisect.bisect_left(lengths, len(spec)) - cache_busters.insert(new_idx, (spec, cachebust)) + cache_busters.insert(0, (spec, cachebust)) intr = config.introspectable('cache busters', spec, @@ -2112,7 +2113,24 @@ class StaticURLInfo(object): config.action(None, callable=register, introspectables=(intr,)) - def _find_cache_buster(self, registry, spec): + def _bust_asset_path(self, registry, spec, subpath, kw): + pkg_name, pkg_subpath = spec.split(':') + absspec = rawspec = '{0}:{1}{2}'.format(pkg_name, pkg_subpath, subpath) + overrides = registry.queryUtility(IPackageOverrides, name=pkg_name) + if overrides is not None: + resource_name = posixpath.join(pkg_subpath, subpath) + sources = overrides.filtered_sources(resource_name) + for source, filtered_path in sources: + rawspec = source.get_path(filtered_path) + if hasattr(source, 'pkg_name'): + rawspec = '{0}:{1}'.format(source.pkg_name, rawspec) + break + for base_spec, cachebust in self.cache_busters: - if base_spec.startswith(spec): - pass + if ( + base_spec == rawspec or + (base_spec.endswith('/') and rawspec.startswith(base_spec)) + ): + subpath, kw = cachebust(absspec, subpath, kw) + break + return subpath, kw -- cgit v1.2.3 From 73630eae045549b792c4e3ef77920357f89c6874 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Mon, 7 Dec 2015 23:16:26 -0600 Subject: sort by length such that longer paths are tested first --- pyramid/config/views.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pyramid/config/views.py b/pyramid/config/views.py index f496dfb7d..16b150a9e 100644 --- a/pyramid/config/views.py +++ b/pyramid/config/views.py @@ -1,3 +1,4 @@ +import bisect import inspect import posixpath import operator @@ -2102,7 +2103,9 @@ class StaticURLInfo(object): idx = specs.index(spec) cache_busters.pop(idx) - cache_busters.insert(0, (spec, cachebust)) + lengths = [len(t[0]) for t in cache_busters] + new_idx = bisect.bisect_left(lengths, len(spec)) + cache_busters.insert(new_idx, (spec, cachebust)) intr = config.introspectable('cache busters', spec, @@ -2126,7 +2129,7 @@ class StaticURLInfo(object): rawspec = '{0}:{1}'.format(source.pkg_name, rawspec) break - for base_spec, cachebust in self.cache_busters: + for base_spec, cachebust in reversed(self.cache_busters): if ( base_spec == rawspec or (base_spec.endswith('/') and rawspec.startswith(base_spec)) -- cgit v1.2.3 From 54d00fd7ff0fcfd19799cbffedb860d08604b83c Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Mon, 7 Dec 2015 23:21:22 -0600 Subject: support os.sep on windows --- pyramid/config/views.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyramid/config/views.py b/pyramid/config/views.py index 16b150a9e..304ce2d43 100644 --- a/pyramid/config/views.py +++ b/pyramid/config/views.py @@ -2132,7 +2132,11 @@ class StaticURLInfo(object): for base_spec, cachebust in reversed(self.cache_busters): if ( base_spec == rawspec or - (base_spec.endswith('/') and rawspec.startswith(base_spec)) + ( + base_spec.endswith(os.sep) + if os.path.isabs(base_spec) + else base_spec.endswith('/') + ) and rawspec.startswith(base_spec) ): subpath, kw = cachebust(absspec, subpath, kw) break -- cgit v1.2.3 From aecb4722640bc49a8e479f5eb5f332346535be8d Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Tue, 8 Dec 2015 09:22:35 -0600 Subject: allow disabling the cache buster --- pyramid/config/views.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pyramid/config/views.py b/pyramid/config/views.py index 304ce2d43..1115ccffc 100644 --- a/pyramid/config/views.py +++ b/pyramid/config/views.py @@ -1982,15 +1982,16 @@ class StaticURLInfo(object): self.cache_busters = [] def generate(self, path, request, **kw): + disable_cache_buster = ( + request.registry.settings['pyramid.prevent_cachebust']) for (url, spec, route_name) in self.registrations: if path.startswith(spec): subpath = path[len(spec):] if WIN: # pragma: no cover subpath = subpath.replace('\\', '/') # windows - # translate spec into overridden spec and lookup cache buster - # to modify subpath, kw - subpath, kw = self._bust_asset_path( - request.registry, spec, subpath, kw) + if not disable_cache_buster: + subpath, kw = self._bust_asset_path( + request.registry, spec, subpath, kw) if url is None: kw['subpath'] = subpath return request.route_url(route_name, **kw) -- cgit v1.2.3 From eedef93f0c4c52ea11320bcd49386262fa7293a1 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Tue, 8 Dec 2015 11:12:38 -0600 Subject: update the cache busting narrative to use add_cache_buster --- docs/narr/assets.rst | 103 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 31 deletions(-) diff --git a/docs/narr/assets.rst b/docs/narr/assets.rst index 0f819570c..8b41f9b8a 100644 --- a/docs/narr/assets.rst +++ b/docs/narr/assets.rst @@ -366,8 +366,7 @@ resource and requests the asset, regardless of any caching policy set for the resource's old URL. :app:`Pyramid` can be configured to produce cache busting URLs for static -assets by passing the optional argument, ``cachebust`` to -:meth:`~pyramid.config.Configurator.add_static_view`: +assets using :meth:`~pyramid.config.Configurator.add_cache_buster`: .. code-block:: python :linenos: @@ -376,14 +375,18 @@ assets by passing the optional argument, ``cachebust`` to from pyramid.static import QueryStringConstantCacheBuster # config is an instance of pyramid.config.Configurator - config.add_static_view( - name='static', path='mypackage:folder/static', - cachebust=QueryStringConstantCacheBuster(str(int(time.time()))), - ) + config.add_static_view(name='static', path='mypackage:folder/static/') + config.add_cache_buster( + 'mypackage:folder/static/', + QueryStringConstantCacheBuster(str(int(time.time())))) + +.. note:: + The trailing slash on the ``add_cache_buster`` call is important to + indicate that it is overriding every asset in the folder and not just a + file named ``static``. -Setting the ``cachebust`` argument instructs :app:`Pyramid` to use a cache -busting scheme which adds the curent time for a static asset to the query -string in the asset's URL: +Adding the cachebuster instructs :app:`Pyramid` to add the current time for +a static asset to the query string in the asset's URL: .. code-block:: python :linenos: @@ -392,10 +395,7 @@ string in the asset's URL: # Returns: 'http://www.example.com/static/js/myapp.js?x=1445318121' When the web server restarts, the time constant will change and therefore so -will its URL. Supplying the ``cachebust`` argument also causes the static -view to set headers instructing clients to cache the asset for ten years, -unless the ``cache_max_age`` argument is also passed, in which case that -value is used. +will its URL. .. note:: @@ -410,7 +410,7 @@ Disabling the Cache Buster It can be useful in some situations (e.g., development) to globally disable all configured cache busters without changing calls to -:meth:`~pyramid.config.Configurator.add_static_view`. To do this set the +:meth:`~pyramid.config.Configurator.add_cache_buster`. To do this set the ``PYRAMID_PREVENT_CACHEBUST`` environment variable or the ``pyramid.prevent_cachebust`` configuration value to a true value. @@ -419,9 +419,9 @@ configured cache busters without changing calls to Customizing the Cache Buster ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The ``cachebust`` option to -:meth:`~pyramid.config.Configurator.add_static_view` may be set to any object -that implements the interface :class:`~pyramid.interfaces.ICacheBuster`. +Calls to :meth:`~pyramid.config.Configurator.add_cache_buster` may use +any object that implements the interface +:class:`~pyramid.interfaces.ICacheBuster`. :app:`Pyramid` ships with a very simplistic :class:`~pyramid.static.QueryStringConstantCacheBuster`, which adds an @@ -461,8 +461,30 @@ the hash of the current commit: def tokenize(self, pathspec): return self.sha1 -Choosing a Cache Buster -~~~~~~~~~~~~~~~~~~~~~~~ +A simple cache buster that modifies the path segment can be constructed as +well: + +.. code-block:: python + :linenos: + + import posixpath + + def cache_buster(spec, subpath, kw): + base_subpath, ext = posixpath.splitext(subpath) + new_subpath = base_subpath + '-asdf' + ext + return new_subpath, kw + +The caveat with this approach is that modifying the path segment +changes the file name, and thus must match what is actually on the +filesystem in order for :meth:`~pyramid.config.Configurator.add_static_view` +to find the file. It's better to use the +:class:`~pyramid.static.ManifestCacheBuster` for these situations, as +described in the next section. + +.. _path_segment_cache_busters: + +Path Segments and Choosing a Cache Buster +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Many caching HTTP proxies will fail to cache a resource if the URL contains a query string. Therefore, in general, you should prefer a cache busting @@ -504,8 +526,11 @@ The following code would set up a cachebuster: config.add_static_view( name='http://mycdn.example.com/', - path='mypackage:static', - cachebust=ManifestCacheBuster('myapp:static/manifest.json')) + path='mypackage:static') + + config.add_cache_buster( + 'mypackage:static/', + ManifestCacheBuster('myapp:static/manifest.json')) A simpler approach is to use the :class:`~pyramid.static.QueryStringConstantCacheBuster` to generate a global @@ -524,8 +549,11 @@ start up as a cachebust token: config.add_static_view( name='http://mycdn.example.com/', - path='mypackage:static', - cachebust=QueryStringConstantCacheBuster(str(int(time.time())))) + path='mypackage:static') + + config.add_cache_buster( + 'mypackage:static/', + QueryStringConstantCacheBuster(str(int(time.time())))) .. index:: single: static assets view @@ -536,25 +564,38 @@ CSS and JavaScript source and cache busting Often one needs to refer to images and other static assets inside CSS and JavaScript files. If cache busting is active, the final static asset URL is not available until the static assets have been assembled. These URLs cannot be -handwritten. Thus, when having static asset references in CSS and JavaScript, -one needs to perform one of the following tasks: +handwritten. Below is an example of how to integrate the cache buster into +the entire stack. Remember, it is just an example and should be modified to +fit your specific tools. -* Process the files by using a precompiler which rewrites URLs to their final - cache busted form. Then, you can use the +* First, process the files by using a precompiler which rewrites URLs to their + final cache-busted form. Then, you can use the :class:`~pyramid.static.ManifestCacheBuster` to synchronize your asset pipeline with :app:`Pyramid`, allowing the pipeline to have full control over the final URLs of your assets. -* Templatize JS and CSS, and call ``request.static_url()`` inside their - template code. +Now that you are able to generate static URLs within :app:`Pyramid`, +you'll need to handle URLs that are out of our control. To do this you may +use some of the following options to get started: -* Pass static URL references to CSS and JavaScript via other means. +* Configure your asset pipeline to rewrite URL references inline in + CSS and JavaScript. This is the best approach because then the files + may be hosted by :app:`Pyramid` or an external CDN without haven't to + change anything. They really are static. + +* Templatize JS and CSS, and call ``request.static_url()`` inside their + template code. While this approach may work in certain scenarios, it is not + recommended because your static assets will not really be static and are now + dependent on :app:`Pyramid` to be served correctly. See + :ref:`advanced static` for more information on this approach. If your CSS and JavaScript assets use URLs to reference other assets it is recommended that you implement an external asset pipeline that can rewrite the generated static files with new URLs containing cache busting tokens. The machinery inside :app:`Pyramid` will not help with this step as it has very -little knowledge of the asset types your application may use. +little knowledge of the asset types your application may use. The integration +into :app:`Pyramid` is simply for linking those assets into your HTML and +other dynamic content. .. _advanced_static: -- cgit v1.2.3 From ffad12b0ac1ee24ad12d6d1a2f300da1ec004010 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Tue, 8 Dec 2015 11:24:35 -0600 Subject: pass the raw asset spec into the cache buster --- pyramid/config/views.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pyramid/config/views.py b/pyramid/config/views.py index 1115ccffc..44003127a 100644 --- a/pyramid/config/views.py +++ b/pyramid/config/views.py @@ -2119,7 +2119,7 @@ class StaticURLInfo(object): def _bust_asset_path(self, registry, spec, subpath, kw): pkg_name, pkg_subpath = spec.split(':') - absspec = rawspec = '{0}:{1}{2}'.format(pkg_name, pkg_subpath, subpath) + rawspec = None overrides = registry.queryUtility(IPackageOverrides, name=pkg_name) if overrides is not None: resource_name = posixpath.join(pkg_subpath, subpath) @@ -2130,6 +2130,9 @@ class StaticURLInfo(object): rawspec = '{0}:{1}'.format(source.pkg_name, rawspec) break + if rawspec is None: + rawspec = '{0}:{1}{2}'.format(pkg_name, pkg_subpath, subpath) + for base_spec, cachebust in reversed(self.cache_busters): if ( base_spec == rawspec or @@ -2139,6 +2142,6 @@ class StaticURLInfo(object): else base_spec.endswith('/') ) and rawspec.startswith(base_spec) ): - subpath, kw = cachebust(absspec, subpath, kw) + subpath, kw = cachebust(rawspec, subpath, kw) break return subpath, kw -- cgit v1.2.3 From edad4ceca802324194000a98f3b07da7cedee546 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Tue, 8 Dec 2015 15:56:30 -0600 Subject: tweak ManifestCacheBuster to allow overriding parse_manifest without touching the file loading logic --- pyramid/static.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/pyramid/static.py b/pyramid/static.py index cda98bea4..9559cd881 100644 --- a/pyramid/static.py +++ b/pyramid/static.py @@ -227,7 +227,7 @@ class ManifestCacheBuster(object): "images/background.png": "images/background-a8169106.png", } - Specifically, it is a JSON-serialized dictionary where the keys are the + By default, it is a JSON-serialized dictionary where the keys are the source asset paths used in calls to :meth:`~pyramid.request.Request.static_url`. For example:: @@ -236,6 +236,9 @@ class ManifestCacheBuster(object): >>> request.static_url('myapp:static/css/main.css') "http://www.example.com/static/css/main-678b7c80.css" + The file format and location can be changed by subclassing and overriding + :meth:`.parse_manifest`. + If a path is not found in the manifest it will pass through unchanged. If ``reload`` is ``True`` then the manifest file will be reloaded when @@ -244,11 +247,6 @@ class ManifestCacheBuster(object): If the manifest file cannot be found on disk it will be treated as an empty mapping unless ``reload`` is ``False``. - The default implementation assumes the requested (possibly cache-busted) - path is the actual filename on disk. Subclasses may override the ``match`` - method to alter this behavior. For example, to strip the cache busting - token from the path. - .. versionadded:: 1.6 """ exists = staticmethod(exists) # testing @@ -262,20 +260,23 @@ class ManifestCacheBuster(object): self._mtime = None if not reload: - self._manifest = self.parse_manifest() + self._manifest = self.get_manifest() - def parse_manifest(self): + def get_manifest(self): + with open(self.manifest_path, 'rb') as fp: + return self.parse_manifest(fp.read()) + + def parse_manifest(self, content): """ - Return a mapping parsed from the ``manifest_path``. + Parse the ``content`` read from the ``manifest_path`` into a + dictionary mapping. Subclasses may override this method to use something other than ``json.loads`` to load any type of file format and return a conforming dictionary. """ - with open(self.manifest_path, 'rb') as fp: - content = fp.read().decode('utf-8') - return json.loads(content) + return json.loads(content.decode('utf-8')) @property def manifest(self): @@ -285,7 +286,7 @@ class ManifestCacheBuster(object): return {} mtime = self.getmtime(self.manifest_path) if self._mtime is None or mtime > self._mtime: - self._manifest = self.parse_manifest() + self._manifest = self.get_manifest() self._mtime = mtime return self._manifest -- cgit v1.2.3 From b2fc4ace7fdb1dd2e90d6d3cc82f7b7b923ffa68 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Tue, 8 Dec 2015 13:19:56 -0600 Subject: update docs on pathspec arg to ICacheBuster --- pyramid/interfaces.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pyramid/interfaces.py b/pyramid/interfaces.py index bdf5bdfbe..153fdad03 100644 --- a/pyramid/interfaces.py +++ b/pyramid/interfaces.py @@ -1204,8 +1204,16 @@ class ICacheBuster(Interface): The ``kw`` argument is a dict of keywords that are to be passed eventually to :meth:`~pyramid.request.Request.static_url` for URL generation. The return value should be a two-tuple of - ``(subpath, kw)`` which are versions of the same arguments modified - to include the cache bust token in the generated URL. + ``(subpath, kw)`` where ``subpath`` is the relative URL from where the + file is served and ``kw`` is the same input argument. The return value + should be modified to include the cache bust token in the generated + URL. + + The ``pathspec`` refers to original location of the file, ignoring any + calls to :meth:`pyramid.config.Configurator.override_asset`. For + example, with a call ``request.static_url('myapp:static/foo.png'), the + ``pathspec`` may be ``themepkg:bar.png``, assuming a call to + ``config.override_asset('myapp:static/foo.png', 'themepkg:bar.png')``. """ # configuration phases: a lower phase number means the actions associated -- cgit v1.2.3 From 6923cae7f493c39b17367a3935a26065d4795ea6 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Tue, 8 Dec 2015 16:38:56 -0600 Subject: support cache busting only full folders --- pyramid/config/views.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/pyramid/config/views.py b/pyramid/config/views.py index 44003127a..ed7ae42ce 100644 --- a/pyramid/config/views.py +++ b/pyramid/config/views.py @@ -1943,15 +1943,18 @@ class ViewsConfiguratorMixin(object): def add_cache_buster(self, path, cachebust): """ - The ``cachebust`` keyword argument may be set to cause + Add a cache buster to a set of files on disk. + + The ``path`` should be the path on disk where the static files + reside. This can be an absolute path, a package-relative path, or a + :term:`asset specification`. + + The ``cachebust`` argument may be set to cause :meth:`~pyramid.request.Request.static_url` to use cache busting when generating URLs. See :ref:`cache_busting` for general information about cache busting. The value of the ``cachebust`` argument must be an object which implements - :class:`~pyramid.interfaces.ICacheBuster`. If the ``cachebust`` - argument is provided, the default for ``cache_max_age`` is modified - to be ten years. ``cache_max_age`` may still be explicitly provided - to override this default. + :class:`~pyramid.interfaces.ICacheBuster`. """ spec = self._make_spec(path) @@ -2096,6 +2099,15 @@ class StaticURLInfo(object): config.action(None, callable=register, introspectables=(intr,)) def add_cache_buster(self, config, spec, cachebust): + # ensure the spec always has a trailing slash as we only support + # adding cache busters to folders, not files + if os.path.isabs(spec): # FBO windows + sep = os.sep + else: + sep = '/' + if not spec.endswith(sep) and not spec.endswith(':'): + spec = spec + sep + def register(): cache_busters = self.cache_busters @@ -2134,14 +2146,7 @@ class StaticURLInfo(object): rawspec = '{0}:{1}{2}'.format(pkg_name, pkg_subpath, subpath) for base_spec, cachebust in reversed(self.cache_busters): - if ( - base_spec == rawspec or - ( - base_spec.endswith(os.sep) - if os.path.isabs(base_spec) - else base_spec.endswith('/') - ) and rawspec.startswith(base_spec) - ): + if rawspec.startswith(base_spec): subpath, kw = cachebust(rawspec, subpath, kw) break return subpath, kw -- cgit v1.2.3 From 5e3439059daa94543f9437a280fed8d804cc7596 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Wed, 9 Dec 2015 20:34:54 -0600 Subject: fix broken tests --- pyramid/config/views.py | 35 ++++--- pyramid/tests/test_config/test_views.py | 173 ++++++++++++++++++++++---------- pyramid/tests/test_static.py | 74 +++++++------- 3 files changed, 178 insertions(+), 104 deletions(-) diff --git a/pyramid/config/views.py b/pyramid/config/views.py index ed7ae42ce..e5bf1203e 100644 --- a/pyramid/config/views.py +++ b/pyramid/config/views.py @@ -38,6 +38,7 @@ from pyramid.interfaces import ( from pyramid import renderers +from pyramid.asset import resolve_asset_spec from pyramid.compat import ( string_types, urlparse, @@ -1985,14 +1986,12 @@ class StaticURLInfo(object): self.cache_busters = [] def generate(self, path, request, **kw): - disable_cache_buster = ( - request.registry.settings['pyramid.prevent_cachebust']) for (url, spec, route_name) in self.registrations: if path.startswith(spec): subpath = path[len(spec):] if WIN: # pragma: no cover subpath = subpath.replace('\\', '/') # windows - if not disable_cache_buster: + if self.cache_busters: subpath, kw = self._bust_asset_path( request.registry, spec, subpath, kw) if url is None: @@ -2099,6 +2098,9 @@ class StaticURLInfo(object): config.action(None, callable=register, introspectables=(intr,)) def add_cache_buster(self, config, spec, cachebust): + if config.registry.settings.get('pyramid.prevent_cachebust'): + return + # ensure the spec always has a trailing slash as we only support # adding cache busters to folders, not files if os.path.isabs(spec): # FBO windows @@ -2130,20 +2132,25 @@ class StaticURLInfo(object): config.action(None, callable=register, introspectables=(intr,)) def _bust_asset_path(self, registry, spec, subpath, kw): - pkg_name, pkg_subpath = spec.split(':') + pkg_name, pkg_subpath = resolve_asset_spec(spec) rawspec = None - overrides = registry.queryUtility(IPackageOverrides, name=pkg_name) - if overrides is not None: - resource_name = posixpath.join(pkg_subpath, subpath) - sources = overrides.filtered_sources(resource_name) - for source, filtered_path in sources: - rawspec = source.get_path(filtered_path) - if hasattr(source, 'pkg_name'): - rawspec = '{0}:{1}'.format(source.pkg_name, rawspec) - break + + if pkg_name is not None: + overrides = registry.queryUtility(IPackageOverrides, name=pkg_name) + if overrides is not None: + resource_name = posixpath.join(pkg_subpath, subpath) + sources = overrides.filtered_sources(resource_name) + for source, filtered_path in sources: + rawspec = source.get_path(filtered_path) + if hasattr(source, 'pkg_name'): + rawspec = '{0}:{1}'.format(source.pkg_name, rawspec) + break + + if rawspec is None: + rawspec = '{0}:{1}{2}'.format(pkg_name, pkg_subpath, subpath) if rawspec is None: - rawspec = '{0}:{1}{2}'.format(pkg_name, pkg_subpath, subpath) + rawspec = pkg_subpath + subpath for base_spec, cachebust in reversed(self.cache_busters): if rawspec.startswith(base_spec): diff --git a/pyramid/tests/test_config/test_views.py b/pyramid/tests/test_config/test_views.py index 020ed131d..eda8d8b05 100644 --- a/pyramid/tests/test_config/test_views.py +++ b/pyramid/tests/test_config/test_views.py @@ -1,3 +1,4 @@ +import os import unittest from pyramid import testing @@ -3887,49 +3888,35 @@ class TestStaticURLInfo(unittest.TestCase): def test_generate_registration_miss(self): inst = self._makeOne() - registrations = [ - (None, 'spec', 'route_name', None), - ('http://example.com/foo/', 'package:path/', None, None)] - inst._get_registrations = lambda *x: registrations + inst.registrations = [ + (None, 'spec', 'route_name'), + ('http://example.com/foo/', 'package:path/', None)] request = self._makeRequest() result = inst.generate('package:path/abc', request) self.assertEqual(result, 'http://example.com/foo/abc') - def test_generate_registration_no_registry_on_request(self): - inst = self._makeOne() - registrations = [ - ('http://example.com/foo/', 'package:path/', None, None)] - inst._get_registrations = lambda *x: registrations - request = self._makeRequest() - del request.registry - result = inst.generate('package:path/abc', request) - self.assertEqual(result, 'http://example.com/foo/abc') - def test_generate_slash_in_name1(self): inst = self._makeOne() - registrations = [ - ('http://example.com/foo/', 'package:path/', None, None)] - inst._get_registrations = lambda *x: registrations + inst.registrations = [('http://example.com/foo/', 'package:path/', None)] request = self._makeRequest() result = inst.generate('package:path/abc', request) self.assertEqual(result, 'http://example.com/foo/abc') def test_generate_slash_in_name2(self): inst = self._makeOne() - registrations = [ - ('http://example.com/foo/', 'package:path/', None, None)] - inst._get_registrations = lambda *x: registrations + inst.registrations = [('http://example.com/foo/', 'package:path/', None)] request = self._makeRequest() result = inst.generate('package:path/', request) self.assertEqual(result, 'http://example.com/foo/') def test_generate_quoting(self): + from pyramid.interfaces import IStaticURLInfo config = testing.setUp() try: config.add_static_view('images', path='mypkg:templates') - inst = self._makeOne() request = testing.DummyRequest() request.registry = config.registry + inst = config.registry.getUtility(IStaticURLInfo) result = inst.generate('mypkg:templates/foo%2Fbar', request) self.assertEqual(result, 'http://example.com/images/foo%252Fbar') finally: @@ -3937,8 +3924,7 @@ class TestStaticURLInfo(unittest.TestCase): def test_generate_route_url(self): inst = self._makeOne() - registrations = [(None, 'package:path/', '__viewname/', None)] - inst._get_registrations = lambda *x: registrations + inst.registrations = [(None, 'package:path/', '__viewname/')] def route_url(n, **kw): self.assertEqual(n, '__viewname/') self.assertEqual(kw, {'subpath':'abc', 'a':1}) @@ -3950,8 +3936,7 @@ class TestStaticURLInfo(unittest.TestCase): def test_generate_url_unquoted_local(self): inst = self._makeOne() - registrations = [(None, 'package:path/', '__viewname/', None)] - inst._get_registrations = lambda *x: registrations + inst.registrations = [(None, 'package:path/', '__viewname/')] def route_url(n, **kw): self.assertEqual(n, '__viewname/') self.assertEqual(kw, {'subpath':'abc def', 'a':1}) @@ -3963,16 +3948,15 @@ class TestStaticURLInfo(unittest.TestCase): def test_generate_url_quoted_remote(self): inst = self._makeOne() - registrations = [('http://example.com/', 'package:path/', None, None)] - inst._get_registrations = lambda *x: registrations + inst.registrations = [('http://example.com/', 'package:path/', None)] request = self._makeRequest() result = inst.generate('package:path/abc def', request, a=1) self.assertEqual(result, 'http://example.com/abc%20def') def test_generate_url_with_custom_query(self): inst = self._makeOne() - registrations = [('http://example.com/', 'package:path/', None, None)] - inst._get_registrations = lambda *x: registrations + registrations = [('http://example.com/', 'package:path/', None)] + inst.registrations = registrations request = self._makeRequest() result = inst.generate('package:path/abc def', request, a=1, _query='(openlayers)') @@ -3981,12 +3965,10 @@ class TestStaticURLInfo(unittest.TestCase): def test_generate_url_with_custom_anchor(self): inst = self._makeOne() - registrations = [('http://example.com/', 'package:path/', None, None)] - inst._get_registrations = lambda *x: registrations + inst.registrations = [('http://example.com/', 'package:path/', None)] request = self._makeRequest() uc = text_(b'La Pe\xc3\xb1a', 'utf-8') - result = inst.generate('package:path/abc def', request, a=1, - _anchor=uc) + result = inst.generate('package:path/abc def', request, a=1, _anchor=uc) self.assertEqual(result, 'http://example.com/abc%20def#La%20Pe%C3%B1a') @@ -3995,52 +3977,102 @@ class TestStaticURLInfo(unittest.TestCase): kw['foo'] = 'bar' return 'foo' + '/' + subpath, kw inst = self._makeOne() - inst.registrations = [(None, 'package:path/', '__viewname', cachebust)] + inst.registrations = [(None, 'package:path/', '__viewname')] inst.cache_busters = [('package:path/', cachebust)] request = self._makeRequest() + called = [False] def route_url(n, **kw): + called[0] = True self.assertEqual(n, '__viewname') - self.assertEqual(kw, {'subpath':'foo/abc', 'foo':'bar'}) + self.assertEqual(kw, {'subpath': 'foo/abc', 'foo': 'bar'}) request.route_url = route_url inst.generate('package:path/abc', request) + self.assertTrue(called[0]) + + def test_generate_url_cachebust_abspath(self): + here = os.path.dirname(__file__) + os.sep + def cachebust(pathspec, subpath, kw): + kw['foo'] = 'bar' + return 'foo' + '/' + subpath, kw + inst = self._makeOne() + inst.registrations = [(None, here, '__viewname')] + inst.cache_busters = [(here, cachebust)] + request = self._makeRequest() + called = [False] + def route_url(n, **kw): + called[0] = True + self.assertEqual(n, '__viewname') + self.assertEqual(kw, {'subpath': 'foo/abc', 'foo': 'bar'}) + request.route_url = route_url + inst.generate(here + 'abc', request) + self.assertTrue(called[0]) + + def test_generate_url_cachebust_nomatch(self): + def fake_cb(*a, **kw): raise AssertionError + inst = self._makeOne() + inst.registrations = [(None, 'package:path/', '__viewname')] + inst.cache_busters = [('package:path2/', fake_cb)] + request = self._makeRequest() + called = [False] + def route_url(n, **kw): + called[0] = True + self.assertEqual(n, '__viewname') + self.assertEqual(kw, {'subpath': 'abc'}) + request.route_url = route_url + inst.generate('package:path/abc', request) + self.assertTrue(called[0]) + + def test_generate_url_cachebust_with_overrides(self): + config = testing.setUp() + try: + config.add_static_view('static', 'path') + config.override_asset( + 'pyramid.tests.test_config:path/', + 'pyramid.tests.test_config:other_path/') + def cb(pathspec, subpath, kw): + kw['_query'] = {'x': 'foo'} + return subpath, kw + config.add_cache_buster('other_path', cb) + request = testing.DummyRequest() + result = request.static_url('path/foo.png') + self.assertEqual(result, 'http://example.com/static/foo.png?x=foo') + finally: + testing.tearDown() def test_add_already_exists(self): config = DummyConfig() inst = self._makeOne() inst.registrations = [('http://example.com/', 'package:path/', None)] inst.add(config, 'http://example.com', 'anotherpackage:path') - expected = [ - ('http://example.com/', 'anotherpackage:path/', None, None)] + expected = [('http://example.com/', 'anotherpackage:path/', None)] self.assertEqual(inst.registrations, expected) def test_add_package_root(self): config = DummyConfig() inst = self._makeOne() inst.add(config, 'http://example.com', 'package:') - expected = [('http://example.com/', 'package:', None, None)] + expected = [('http://example.com/', 'package:', None)] self.assertEqual(inst.registrations, expected) def test_add_url_withendslash(self): config = DummyConfig() inst = self._makeOne() inst.add(config, 'http://example.com/', 'anotherpackage:path') - expected = [ - ('http://example.com/', 'anotherpackage:path/', None, None)] + expected = [('http://example.com/', 'anotherpackage:path/', None)] self.assertEqual(inst.registrations, expected) def test_add_url_noendslash(self): config = DummyConfig() inst = self._makeOne() inst.add(config, 'http://example.com', 'anotherpackage:path') - expected = [ - ('http://example.com/', 'anotherpackage:path/', None, None)] + expected = [('http://example.com/', 'anotherpackage:path/', None)] self.assertEqual(inst.registrations, expected) def test_add_url_noscheme(self): config = DummyConfig() inst = self._makeOne() inst.add(config, '//example.com', 'anotherpackage:path') - expected = [('//example.com/', 'anotherpackage:path/', None, None)] + expected = [('//example.com/', 'anotherpackage:path/', None)] self.assertEqual(inst.registrations, expected) def test_add_viewname(self): @@ -4049,7 +4081,7 @@ class TestStaticURLInfo(unittest.TestCase): config = DummyConfig() inst = self._makeOne() inst.add(config, 'view', 'anotherpackage:path', cache_max_age=1) - expected = [(None, 'anotherpackage:path/', '__view/', None)] + expected = [(None, 'anotherpackage:path/', '__view/')] self.assertEqual(inst.registrations, expected) self.assertEqual(config.route_args, ('__view/', 'view/*subpath')) self.assertEqual(config.view_kw['permission'], NO_PERMISSION_REQUIRED) @@ -4060,7 +4092,7 @@ class TestStaticURLInfo(unittest.TestCase): config.route_prefix = '/abc' inst = self._makeOne() inst.add(config, 'view', 'anotherpackage:path',) - expected = [(None, 'anotherpackage:path/', '__/abc/view/', None)] + expected = [(None, 'anotherpackage:path/', '__/abc/view/')] self.assertEqual(inst.registrations, expected) self.assertEqual(config.route_args, ('__/abc/view/', 'view/*subpath')) @@ -4097,20 +4129,47 @@ class TestStaticURLInfo(unittest.TestCase): config = DummyConfig() config.registry.settings['pyramid.prevent_cachebust'] = True inst = self._makeOne() - inst.add(config, 'view', 'mypackage:path', cachebust=True) - cachebust = config.registry._static_url_registrations[0][3] - self.assertEqual(cachebust, None) + cachebust = DummyCacheBuster('foo') + inst.add_cache_buster(config, 'mypackage:path', cachebust) + self.assertEqual(inst.cache_busters, []) - def test_add_cachebust_custom(self): + def test_add_cachebuster(self): config = DummyConfig() inst = self._makeOne() - inst.add(config, 'view', 'mypackage:path', - cachebust=DummyCacheBuster('foo')) - cachebust = config.registry._static_url_registrations[0][3] - subpath, kw = cachebust('some/path', {}) + inst.add_cache_buster(config, 'mypackage:path', DummyCacheBuster('foo')) + cachebust = inst.cache_busters[-1][1] + subpath, kw = cachebust('mypackage:some/path', 'some/path', {}) self.assertEqual(subpath, 'some/path') self.assertEqual(kw['x'], 'foo') + def test_add_cachebuster_abspath(self): + here = os.path.dirname(__file__) + config = DummyConfig() + inst = self._makeOne() + cb = DummyCacheBuster('foo') + inst.add_cache_buster(config, here, cb) + self.assertEqual(inst.cache_busters, [(here + '/', cb)]) + + def test_add_cachebuster_overwrite(self): + config = DummyConfig() + inst = self._makeOne() + cb1 = DummyCacheBuster('foo') + cb2 = DummyCacheBuster('bar') + inst.add_cache_buster(config, 'mypackage:path/', cb1) + inst.add_cache_buster(config, 'mypackage:path', cb2) + self.assertEqual(inst.cache_busters, + [('mypackage:path/', cb2)]) + + def test_add_cachebuster_for_more_specific_path(self): + config = DummyConfig() + inst = self._makeOne() + cb1 = DummyCacheBuster('foo') + cb2 = DummyCacheBuster('bar') + inst.add_cache_buster(config, 'mypackage:path', cb1) + inst.add_cache_buster(config, 'mypackage:path/sub', cb2) + self.assertEqual(inst.cache_busters, + [('mypackage:path/', cb1), ('mypackage:path/sub/', cb2)]) + class Test_view_description(unittest.TestCase): def _callFUT(self, view): from pyramid.config.views import view_description @@ -4130,9 +4189,14 @@ class Test_view_description(unittest.TestCase): class DummyRegistry: + utility = None + def __init__(self): self.settings = {} + def queryUtility(self, type_or_iface, name=None, default=None): + return self.utility or default + from zope.interface import implementer from pyramid.interfaces import ( IResponse, @@ -4193,6 +4257,9 @@ class DummySecurityPolicy: return self.permitted class DummyConfig: + def __init__(self): + self.registry = DummyRegistry() + route_prefix = '' def add_route(self, *args, **kw): self.route_args = args diff --git a/pyramid/tests/test_static.py b/pyramid/tests/test_static.py index 4a07c2cb1..73f242add 100644 --- a/pyramid/tests/test_static.py +++ b/pyramid/tests/test_static.py @@ -385,29 +385,29 @@ class TestQueryStringConstantCacheBuster(unittest.TestCase): fut = self._makeOne().tokenize self.assertEqual(fut('whatever'), 'foo') - def test_pregenerate(self): - fut = self._makeOne().pregenerate + def test_it(self): + fut = self._makeOne() self.assertEqual( - fut('foo', ('bar',), {}), - (('bar',), {'_query': {'x': 'foo'}})) + fut('foo', 'bar', {}), + ('bar', {'_query': {'x': 'foo'}})) - def test_pregenerate_change_param(self): - fut = self._makeOne('y').pregenerate + def test_change_param(self): + fut = self._makeOne('y') self.assertEqual( - fut('foo', ('bar',), {}), - (('bar',), {'_query': {'y': 'foo'}})) + fut('foo', 'bar', {}), + ('bar', {'_query': {'y': 'foo'}})) - def test_pregenerate_query_is_already_tuples(self): - fut = self._makeOne().pregenerate + def test_query_is_already_tuples(self): + fut = self._makeOne() self.assertEqual( - fut('foo', ('bar',), {'_query': [('a', 'b')]}), - (('bar',), {'_query': (('a', 'b'), ('x', 'foo'))})) + fut('foo', 'bar', {'_query': [('a', 'b')]}), + ('bar', {'_query': (('a', 'b'), ('x', 'foo'))})) - def test_pregenerate_query_is_tuple_of_tuples(self): - fut = self._makeOne().pregenerate + def test_query_is_tuple_of_tuples(self): + fut = self._makeOne() self.assertEqual( - fut('foo', ('bar',), {'_query': (('a', 'b'),)}), - (('bar',), {'_query': (('a', 'b'), ('x', 'foo'))})) + fut('foo', 'bar', {'_query': (('a', 'b'),)}), + ('bar', {'_query': (('a', 'b'), ('x', 'foo'))})) class TestManifestCacheBuster(unittest.TestCase): @@ -417,55 +417,55 @@ class TestManifestCacheBuster(unittest.TestCase): def test_it(self): manifest_path = os.path.join(here, 'fixtures', 'manifest.json') - fut = self._makeOne(manifest_path).pregenerate - self.assertEqual(fut('foo', ('bar',), {}), (['bar'], {})) + fut = self._makeOne(manifest_path) + self.assertEqual(fut('foo', 'bar', {}), ('bar', {})) self.assertEqual( - fut('foo', ('css', 'main.css'), {}), - (['css', 'main-test.css'], {})) + fut('foo', 'css/main.css', {}), + ('css/main-test.css', {})) def test_it_with_relspec(self): - fut = self._makeOne('fixtures/manifest.json').pregenerate - self.assertEqual(fut('foo', ('bar',), {}), (['bar'], {})) + fut = self._makeOne('fixtures/manifest.json') + self.assertEqual(fut('foo', 'bar', {}), ('bar', {})) self.assertEqual( - fut('foo', ('css', 'main.css'), {}), - (['css', 'main-test.css'], {})) + fut('foo', 'css/main.css', {}), + ('css/main-test.css', {})) def test_it_with_absspec(self): - fut = self._makeOne('pyramid.tests:fixtures/manifest.json').pregenerate - self.assertEqual(fut('foo', ('bar',), {}), (['bar'], {})) + fut = self._makeOne('pyramid.tests:fixtures/manifest.json') + self.assertEqual(fut('foo', 'bar', {}), ('bar', {})) self.assertEqual( - fut('foo', ('css', 'main.css'), {}), - (['css', 'main-test.css'], {})) + fut('foo', 'css/main.css', {}), + ('css/main-test.css', {})) def test_reload(self): manifest_path = os.path.join(here, 'fixtures', 'manifest.json') new_manifest_path = os.path.join(here, 'fixtures', 'manifest2.json') inst = self._makeOne('foo', reload=True) inst.getmtime = lambda *args, **kwargs: 0 - fut = inst.pregenerate + fut = inst # test without a valid manifest self.assertEqual( - fut('foo', ('css', 'main.css'), {}), - (['css', 'main.css'], {})) + fut('foo', 'css/main.css', {}), + ('css/main.css', {})) # swap to a real manifest, setting mtime to 0 inst.manifest_path = manifest_path self.assertEqual( - fut('foo', ('css', 'main.css'), {}), - (['css', 'main-test.css'], {})) + fut('foo', 'css/main.css', {}), + ('css/main-test.css', {})) # ensure switching the path doesn't change the result inst.manifest_path = new_manifest_path self.assertEqual( - fut('foo', ('css', 'main.css'), {}), - (['css', 'main-test.css'], {})) + fut('foo', 'css/main.css', {}), + ('css/main-test.css', {})) # update mtime, should cause a reload inst.getmtime = lambda *args, **kwargs: 1 self.assertEqual( - fut('foo', ('css', 'main.css'), {}), - (['css', 'main-678b7c80.css'], {})) + fut('foo', 'css/main.css', {}), + ('css/main-678b7c80.css', {})) def test_invalid_manifest(self): self.assertRaises(IOError, lambda: self._makeOne('foo')) -- cgit v1.2.3 From 3175c990bc02805b729594996f6360b81f5f0ebc Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Wed, 9 Dec 2015 23:01:35 -0600 Subject: fix broken ref in docs --- docs/narr/assets.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/narr/assets.rst b/docs/narr/assets.rst index 8b41f9b8a..0e3f6af11 100644 --- a/docs/narr/assets.rst +++ b/docs/narr/assets.rst @@ -587,7 +587,7 @@ use some of the following options to get started: template code. While this approach may work in certain scenarios, it is not recommended because your static assets will not really be static and are now dependent on :app:`Pyramid` to be served correctly. See - :ref:`advanced static` for more information on this approach. + :ref:`advanced_static` for more information on this approach. If your CSS and JavaScript assets use URLs to reference other assets it is recommended that you implement an external asset pipeline that can rewrite the -- cgit v1.2.3 From 4350699a208dc9304ae8c8dd165251f227ff5189 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Wed, 9 Dec 2015 23:02:28 -0600 Subject: remove unused import --- pyramid/config/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyramid/config/views.py b/pyramid/config/views.py index e5bf1203e..1fcdcb136 100644 --- a/pyramid/config/views.py +++ b/pyramid/config/views.py @@ -69,7 +69,6 @@ from pyramid.response import Response from pyramid.security import NO_PERMISSION_REQUIRED from pyramid.static import static_view -from pyramid.threadlocal import get_current_registry from pyramid.url import parse_url_overrides -- cgit v1.2.3 From 4d19b84cb8134a0e7f030064e5d944defaa6970a Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Mon, 14 Dec 2015 00:17:39 -0600 Subject: new default behavior matches virtual specs, old behavior hidden behind explicit=True --- pyramid/config/views.py | 67 ++++++++++++++++++++++---------- pyramid/interfaces.py | 26 +++++++++---- pyramid/static.py | 10 ++--- pyramid/tests/test_config/test_views.py | 69 ++++++++++++++++++++++++--------- pyramid/tests/test_static.py | 2 +- 5 files changed, 123 insertions(+), 51 deletions(-) diff --git a/pyramid/config/views.py b/pyramid/config/views.py index 1fcdcb136..759276351 100644 --- a/pyramid/config/views.py +++ b/pyramid/config/views.py @@ -1,4 +1,3 @@ -import bisect import inspect import posixpath import operator @@ -1941,7 +1940,7 @@ class ViewsConfiguratorMixin(object): info = self._get_static_info() info.add(self, name, spec, **kw) - def add_cache_buster(self, path, cachebust): + def add_cache_buster(self, path, cachebust, explicit=False): """ Add a cache buster to a set of files on disk. @@ -1956,10 +1955,16 @@ class ViewsConfiguratorMixin(object): be an object which implements :class:`~pyramid.interfaces.ICacheBuster`. + If ``explicit`` is set to ``True`` then the ``path`` for the cache + buster will be matched based on the ``rawspec`` instead of the + ``pathspec`` as defined in the + :class:`~pyramid.interfaces.ICacheBuster` interface. + Default: ``False``. + """ spec = self._make_spec(path) info = self._get_static_info() - info.add_cache_buster(self, spec, cachebust) + info.add_cache_buster(self, spec, cachebust, explicit=explicit) def _get_static_info(self): info = self.registry.queryUtility(IStaticURLInfo) @@ -1992,7 +1997,7 @@ class StaticURLInfo(object): subpath = subpath.replace('\\', '/') # windows if self.cache_busters: subpath, kw = self._bust_asset_path( - request.registry, spec, subpath, kw) + request, spec, subpath, kw) if url is None: kw['subpath'] = subpath return request.route_url(route_name, **kw) @@ -2096,7 +2101,7 @@ class StaticURLInfo(object): config.action(None, callable=register, introspectables=(intr,)) - def add_cache_buster(self, config, spec, cachebust): + def add_cache_buster(self, config, spec, cachebust, explicit=False): if config.registry.settings.get('pyramid.prevent_cachebust'): return @@ -2112,29 +2117,46 @@ class StaticURLInfo(object): def register(): cache_busters = self.cache_busters - specs = [t[0] for t in cache_busters] - if spec in specs: - idx = specs.index(spec) - cache_busters.pop(idx) + # find duplicate cache buster (old_idx) + # and insertion location (new_idx) + new_idx, old_idx = len(cache_busters), None + for idx, (spec_, cb_, explicit_) in enumerate(cache_busters): + # if we find an identical (spec, explicit) then use it + if spec == spec_ and explicit == explicit_: + old_idx = new_idx = idx + break + + # past all explicit==False specs then add to the end + elif not explicit and explicit_: + new_idx = idx + break + + # explicit matches and spec is shorter + elif explicit == explicit_ and len(spec) < len(spec_): + new_idx = idx + break - lengths = [len(t[0]) for t in cache_busters] - new_idx = bisect.bisect_left(lengths, len(spec)) - cache_busters.insert(new_idx, (spec, cachebust)) + if old_idx is not None: + cache_busters.pop(old_idx) + cache_busters.insert(new_idx, (spec, cachebust, explicit)) intr = config.introspectable('cache busters', spec, 'cache buster for %r' % spec, 'cache buster') intr['cachebust'] = cachebust - intr['spec'] = spec + intr['path'] = spec + intr['explicit'] = explicit config.action(None, callable=register, introspectables=(intr,)) - def _bust_asset_path(self, registry, spec, subpath, kw): + def _bust_asset_path(self, request, spec, subpath, kw): + registry = request.registry pkg_name, pkg_subpath = resolve_asset_spec(spec) rawspec = None if pkg_name is not None: + pathspec = '{0}:{1}{2}'.format(pkg_name, pkg_subpath, subpath) overrides = registry.queryUtility(IPackageOverrides, name=pkg_name) if overrides is not None: resource_name = posixpath.join(pkg_subpath, subpath) @@ -2145,14 +2167,19 @@ class StaticURLInfo(object): rawspec = '{0}:{1}'.format(source.pkg_name, rawspec) break - if rawspec is None: - rawspec = '{0}:{1}{2}'.format(pkg_name, pkg_subpath, subpath) + else: + pathspec = pkg_subpath + subpath if rawspec is None: - rawspec = pkg_subpath + subpath + rawspec = pathspec - for base_spec, cachebust in reversed(self.cache_busters): - if rawspec.startswith(base_spec): - subpath, kw = cachebust(rawspec, subpath, kw) + kw['pathspec'] = pathspec + kw['rawspec'] = rawspec + for spec_, cachebust, explicit in reversed(self.cache_busters): + if ( + (explicit and rawspec.startswith(spec_)) or + (not explicit and pathspec.startswith(spec_)) + ): + subpath, kw = cachebust(request, subpath, kw) break return subpath, kw diff --git a/pyramid/interfaces.py b/pyramid/interfaces.py index 153fdad03..bbdc5121d 100644 --- a/pyramid/interfaces.py +++ b/pyramid/interfaces.py @@ -1194,11 +1194,11 @@ class ICacheBuster(Interface): .. versionadded:: 1.6 """ - def __call__(pathspec, subpath, kw): + def __call__(request, subpath, kw): """ Modifies a subpath and/or keyword arguments from which a static asset - URL will be computed during URL generation. The ``pathspec`` argument - is the path specification for the resource to be cache busted. + URL will be computed during URL generation. + The ``subpath`` argument is a path of ``/``-delimited segments that represent the portion of the asset URL which is used to find the asset. The ``kw`` argument is a dict of keywords that are to be passed @@ -1209,10 +1209,22 @@ class ICacheBuster(Interface): should be modified to include the cache bust token in the generated URL. - The ``pathspec`` refers to original location of the file, ignoring any - calls to :meth:`pyramid.config.Configurator.override_asset`. For - example, with a call ``request.static_url('myapp:static/foo.png'), the - ``pathspec`` may be ``themepkg:bar.png``, assuming a call to + The ``kw`` dictionary contains extra arguments passed to + :meth:`~pyramid.request.Request.static_url` as well as some extra + items that may be usful including: + + - ``pathspec`` is the path specification for the resource + to be cache busted. + + - ``rawspec`` is the original location of the file, ignoring + any calls to :meth:`pyramid.config.Configurator.override_asset`. + + The ``pathspec`` and ``rawspec`` values are only different in cases + where an asset has been mounted into a virtual location using + :meth:`pyramid.config.Configurator.override_asset`. For example, with + a call to ``request.static_url('myapp:static/foo.png'), the + ``pathspec`` is ``myapp:static/foo.png`` whereas the ``rawspec`` may + be ``themepkg:bar.png``, assuming a call to ``config.override_asset('myapp:static/foo.png', 'themepkg:bar.png')``. """ diff --git a/pyramid/static.py b/pyramid/static.py index 9559cd881..4054d5be0 100644 --- a/pyramid/static.py +++ b/pyramid/static.py @@ -172,15 +172,15 @@ class QueryStringCacheBuster(object): to the query string and defaults to ``'x'``. To use this class, subclass it and provide a ``tokenize`` method which - accepts a ``pathspec`` and returns a token. + accepts ``request, pathspec, kw`` and returns a token. .. versionadded:: 1.6 """ def __init__(self, param='x'): self.param = param - def __call__(self, pathspec, subpath, kw): - token = self.tokenize(pathspec) + def __call__(self, request, subpath, kw): + token = self.tokenize(request, subpath, kw) query = kw.setdefault('_query', {}) if isinstance(query, dict): query[self.param] = token @@ -205,7 +205,7 @@ class QueryStringConstantCacheBuster(QueryStringCacheBuster): super(QueryStringConstantCacheBuster, self).__init__(param=param) self._token = token - def tokenize(self, pathspec): + def tokenize(self, request, subpath, kw): return self._token class ManifestCacheBuster(object): @@ -290,6 +290,6 @@ class ManifestCacheBuster(object): self._mtime = mtime return self._manifest - def __call__(self, pathspec, subpath, kw): + def __call__(self, request, subpath, kw): subpath = self.manifest.get(subpath, subpath) return (subpath, kw) diff --git a/pyramid/tests/test_config/test_views.py b/pyramid/tests/test_config/test_views.py index eda8d8b05..e89d43c9a 100644 --- a/pyramid/tests/test_config/test_views.py +++ b/pyramid/tests/test_config/test_views.py @@ -3978,13 +3978,15 @@ class TestStaticURLInfo(unittest.TestCase): return 'foo' + '/' + subpath, kw inst = self._makeOne() inst.registrations = [(None, 'package:path/', '__viewname')] - inst.cache_busters = [('package:path/', cachebust)] + inst.cache_busters = [('package:path/', cachebust, False)] request = self._makeRequest() called = [False] def route_url(n, **kw): called[0] = True self.assertEqual(n, '__viewname') - self.assertEqual(kw, {'subpath': 'foo/abc', 'foo': 'bar'}) + self.assertEqual(kw, {'subpath': 'foo/abc', 'foo': 'bar', + 'pathspec': 'package:path/abc', + 'rawspec': 'package:path/abc'}) request.route_url = route_url inst.generate('package:path/abc', request) self.assertTrue(called[0]) @@ -3996,13 +3998,15 @@ class TestStaticURLInfo(unittest.TestCase): return 'foo' + '/' + subpath, kw inst = self._makeOne() inst.registrations = [(None, here, '__viewname')] - inst.cache_busters = [(here, cachebust)] + inst.cache_busters = [(here, cachebust, False)] request = self._makeRequest() called = [False] def route_url(n, **kw): called[0] = True self.assertEqual(n, '__viewname') - self.assertEqual(kw, {'subpath': 'foo/abc', 'foo': 'bar'}) + self.assertEqual(kw, {'subpath': 'foo/abc', 'foo': 'bar', + 'pathspec': here + 'abc', + 'rawspec': here + 'abc'}) request.route_url = route_url inst.generate(here + 'abc', request) self.assertTrue(called[0]) @@ -4011,13 +4015,15 @@ class TestStaticURLInfo(unittest.TestCase): def fake_cb(*a, **kw): raise AssertionError inst = self._makeOne() inst.registrations = [(None, 'package:path/', '__viewname')] - inst.cache_busters = [('package:path2/', fake_cb)] + inst.cache_busters = [('package:path2/', fake_cb, False)] request = self._makeRequest() called = [False] def route_url(n, **kw): called[0] = True self.assertEqual(n, '__viewname') - self.assertEqual(kw, {'subpath': 'abc'}) + self.assertEqual(kw, {'subpath': 'abc', + 'pathspec': 'package:path/abc', + 'rawspec': 'package:path/abc'}) request.route_url = route_url inst.generate('package:path/abc', request) self.assertTrue(called[0]) @@ -4025,17 +4031,22 @@ class TestStaticURLInfo(unittest.TestCase): def test_generate_url_cachebust_with_overrides(self): config = testing.setUp() try: + request = testing.DummyRequest() config.add_static_view('static', 'path') config.override_asset( 'pyramid.tests.test_config:path/', 'pyramid.tests.test_config:other_path/') - def cb(pathspec, subpath, kw): - kw['_query'] = {'x': 'foo'} - return subpath, kw - config.add_cache_buster('other_path', cb) - request = testing.DummyRequest() + def cb(val): + def cb_(request, subpath, kw): + kw['_query'] = {'x': val} + return subpath, kw + return cb_ + config.add_cache_buster('path', cb('foo')) result = request.static_url('path/foo.png') self.assertEqual(result, 'http://example.com/static/foo.png?x=foo') + config.add_cache_buster('other_path', cb('bar'), explicit=True) + result = request.static_url('path/foo.png') + self.assertEqual(result, 'http://example.com/static/foo.png?x=bar') finally: testing.tearDown() @@ -4138,7 +4149,7 @@ class TestStaticURLInfo(unittest.TestCase): inst = self._makeOne() inst.add_cache_buster(config, 'mypackage:path', DummyCacheBuster('foo')) cachebust = inst.cache_busters[-1][1] - subpath, kw = cachebust('mypackage:some/path', 'some/path', {}) + subpath, kw = cachebust(None, 'some/path', {}) self.assertEqual(subpath, 'some/path') self.assertEqual(kw['x'], 'foo') @@ -4148,7 +4159,7 @@ class TestStaticURLInfo(unittest.TestCase): inst = self._makeOne() cb = DummyCacheBuster('foo') inst.add_cache_buster(config, here, cb) - self.assertEqual(inst.cache_busters, [(here + '/', cb)]) + self.assertEqual(inst.cache_busters, [(here + '/', cb, False)]) def test_add_cachebuster_overwrite(self): config = DummyConfig() @@ -4158,17 +4169,39 @@ class TestStaticURLInfo(unittest.TestCase): inst.add_cache_buster(config, 'mypackage:path/', cb1) inst.add_cache_buster(config, 'mypackage:path', cb2) self.assertEqual(inst.cache_busters, - [('mypackage:path/', cb2)]) + [('mypackage:path/', cb2, False)]) + + def test_add_cachebuster_overwrite_explicit(self): + config = DummyConfig() + inst = self._makeOne() + cb1 = DummyCacheBuster('foo') + cb2 = DummyCacheBuster('bar') + inst.add_cache_buster(config, 'mypackage:path/', cb1) + inst.add_cache_buster(config, 'mypackage:path', cb2, True) + self.assertEqual(inst.cache_busters, + [('mypackage:path/', cb1, False), + ('mypackage:path/', cb2, True)]) def test_add_cachebuster_for_more_specific_path(self): config = DummyConfig() inst = self._makeOne() cb1 = DummyCacheBuster('foo') cb2 = DummyCacheBuster('bar') + cb3 = DummyCacheBuster('baz') + cb4 = DummyCacheBuster('xyz') + cb5 = DummyCacheBuster('w') inst.add_cache_buster(config, 'mypackage:path', cb1) - inst.add_cache_buster(config, 'mypackage:path/sub', cb2) - self.assertEqual(inst.cache_busters, - [('mypackage:path/', cb1), ('mypackage:path/sub/', cb2)]) + inst.add_cache_buster(config, 'mypackage:path/sub', cb2, True) + inst.add_cache_buster(config, 'mypackage:path/sub/other', cb3) + inst.add_cache_buster(config, 'mypackage:path/sub/other', cb4, True) + inst.add_cache_buster(config, 'mypackage:path/sub/less', cb5, True) + self.assertEqual( + inst.cache_busters, + [('mypackage:path/', cb1, False), + ('mypackage:path/sub/other/', cb3, False), + ('mypackage:path/sub/', cb2, True), + ('mypackage:path/sub/less/', cb5, True), + ('mypackage:path/sub/other/', cb4, True)]) class Test_view_description(unittest.TestCase): def _callFUT(self, view): @@ -4293,7 +4326,7 @@ class DummyCacheBuster(object): def __init__(self, token): self.token = token - def __call__(self, pathspec, subpath, kw): + def __call__(self, request, subpath, kw): kw['x'] = self.token return subpath, kw diff --git a/pyramid/tests/test_static.py b/pyramid/tests/test_static.py index 73f242add..2ca86bc44 100644 --- a/pyramid/tests/test_static.py +++ b/pyramid/tests/test_static.py @@ -383,7 +383,7 @@ class TestQueryStringConstantCacheBuster(unittest.TestCase): def test_token(self): fut = self._makeOne().tokenize - self.assertEqual(fut('whatever'), 'foo') + self.assertEqual(fut(None, 'whatever', None), 'foo') def test_it(self): fut = self._makeOne() -- cgit v1.2.3 From ca573ea73db05d7ea9bdbb51eb7db26ab602ccf2 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Wed, 16 Dec 2015 19:50:13 -0600 Subject: defer prevent check until register --- pyramid/config/views.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyramid/config/views.py b/pyramid/config/views.py index 759276351..acaf9462b 100644 --- a/pyramid/config/views.py +++ b/pyramid/config/views.py @@ -2102,9 +2102,6 @@ class StaticURLInfo(object): config.action(None, callable=register, introspectables=(intr,)) def add_cache_buster(self, config, spec, cachebust, explicit=False): - if config.registry.settings.get('pyramid.prevent_cachebust'): - return - # ensure the spec always has a trailing slash as we only support # adding cache busters to folders, not files if os.path.isabs(spec): # FBO windows @@ -2115,6 +2112,9 @@ class StaticURLInfo(object): spec = spec + sep def register(): + if config.registry.settings.get('pyramid.prevent_cachebust'): + return + cache_busters = self.cache_busters # find duplicate cache buster (old_idx) @@ -2138,6 +2138,7 @@ class StaticURLInfo(object): if old_idx is not None: cache_busters.pop(old_idx) + cache_busters.insert(new_idx, (spec, cachebust, explicit)) intr = config.introspectable('cache busters', -- cgit v1.2.3 From 313ff3c28fbd3b784e4c87daf8ae8a4cf713262b Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Wed, 16 Dec 2015 21:30:56 -0600 Subject: update docs to support explicit asset overrides --- docs/narr/assets.rst | 100 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 65 insertions(+), 35 deletions(-) diff --git a/docs/narr/assets.rst b/docs/narr/assets.rst index 0e3f6af11..b28e6b5b3 100644 --- a/docs/narr/assets.rst +++ b/docs/narr/assets.rst @@ -380,11 +380,6 @@ assets using :meth:`~pyramid.config.Configurator.add_cache_buster`: 'mypackage:folder/static/', QueryStringConstantCacheBuster(str(int(time.time())))) -.. note:: - The trailing slash on the ``add_cache_buster`` call is important to - indicate that it is overriding every asset in the folder and not just a - file named ``static``. - Adding the cachebuster instructs :app:`Pyramid` to add the current time for a static asset to the query string in the asset's URL: @@ -451,12 +446,13 @@ the hash of the current commit: from an egg repository like PYPI, you can use this cachebuster to get the current commit's SHA1 to use as the cache bust token. """ - def __init__(self, param='x'): + def __init__(self, param='x', repo_path=None): super(GitCacheBuster, self).__init__(param=param) - here = os.path.dirname(os.path.abspath(__file__)) + if repo_path is None: + repo_path = os.path.dirname(os.path.abspath(__file__)) self.sha1 = subprocess.check_output( ['git', 'rev-parse', 'HEAD'], - cwd=here).strip() + cwd=repo_path).strip() def tokenize(self, pathspec): return self.sha1 @@ -469,10 +465,14 @@ well: import posixpath - def cache_buster(spec, subpath, kw): - base_subpath, ext = posixpath.splitext(subpath) - new_subpath = base_subpath + '-asdf' + ext - return new_subpath, kw + class PathConstantCacheBuster(object): + def __init__(self, token): + self.token = token + + def __call__(self, request, subpath, kw): + base_subpath, ext = posixpath.splitext(subpath) + new_subpath = base_subpath + self.token + ext + return new_subpath, kw The caveat with this approach is that modifying the path segment changes the file name, and thus must match what is actually on the @@ -532,29 +532,6 @@ The following code would set up a cachebuster: 'mypackage:static/', ManifestCacheBuster('myapp:static/manifest.json')) -A simpler approach is to use the -:class:`~pyramid.static.QueryStringConstantCacheBuster` to generate a global -token that will bust all of the assets at once. The advantage of this strategy -is that it is simple and by using the query string there doesn't need to be -any shared information between your application and the static assets. - -The following code would set up a cachebuster that just uses the time at -start up as a cachebust token: - -.. code-block:: python - :linenos: - - import time - from pyramid.static import QueryStringConstantCacheBuster - - config.add_static_view( - name='http://mycdn.example.com/', - path='mypackage:static') - - config.add_cache_buster( - 'mypackage:static/', - QueryStringConstantCacheBuster(str(int(time.time())))) - .. index:: single: static assets view @@ -834,3 +811,56 @@ when an override is used. As of Pyramid 1.6, it is also possible to override an asset by supplying an absolute path to a file or directory. This may be useful if the assets are not distributed as part of a Python package. + +Cache Busting and Asset Overrides +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Overriding static assets that are being hosted using +:meth:`pyramid.config.Configurator.add_static_view` can affect your cache +busting strategy when using any cache busters that are asset-aware such as +:class:`pyramid.static.ManifestCacheBuster`. What sets asset-aware cache +busters apart is that they have logic tied to specific assets. For example, +a manifest is only generated for a specific set of pre-defined assets. Now, +imagine you have overridden an asset defined in this manifest with a new, +unknown version. By default, the cache buster will be invoked for an asset +it has never seen before and will likely end up returning a cache busting +token for the original asset rather than the asset that will actually end up +being served! In order to get around this issue it's possible to attach a +different :class:`pyramid.interfaces.ICacheBuster` implementation to the +new assets. This would cause the original assets to be served by their +manifest, and the new assets served by their own cache buster. To do this, +:meth:`pyramid.config.Configurator.add_cache_buster` supports an ``explicit`` +option. For example: + +.. code-block:: python + :linenos: + + from pyramid.static import ManifestCacheBuster + + # define a static view for myapp:static assets + config.add_static_view('static', 'myapp:static') + + # setup a cache buster for your app based on the myapp:static assets + my_cb = ManifestCacheBuster('myapp:static/manifest.json') + config.add_cache_buster('myapp:static', my_cb) + + # override an asset + config.override_asset( + to_override='myapp:static/background.png', + override_with='theme:static/background.png') + + # override the cache buster for theme:static assets + theme_cb = ManifestCacheBuster('theme:static/manifest.json') + config.add_cache_buster('theme:static', theme_cb, explicit=True) + +In the above example there is a default cache buster, ``my_cb`` for all assets +served from the ``myapp:static`` folder. This would also affect +``theme:static/background.png`` when generating URLs via +``request.static_url('myapp:static/background.png')``. + +The ``theme_cb`` is defined explicitly for any assets loaded from the +``theme:static`` folder. Explicit cache busters have priority and thus +``theme_cb`` would be invoked for +``request.static_url('myapp:static/background.png')`` but ``my_cb`` would be +used for any other assets like +``request.static_url('myapp:static/favicon.ico')``. -- cgit v1.2.3 From 897d1deeab710233565f97d216e4d112b2a466e3 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Thu, 17 Dec 2015 20:52:19 -0600 Subject: grammar updates from stevepiercy --- docs/narr/assets.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/narr/assets.rst b/docs/narr/assets.rst index b28e6b5b3..054c58247 100644 --- a/docs/narr/assets.rst +++ b/docs/narr/assets.rst @@ -557,7 +557,7 @@ use some of the following options to get started: * Configure your asset pipeline to rewrite URL references inline in CSS and JavaScript. This is the best approach because then the files - may be hosted by :app:`Pyramid` or an external CDN without haven't to + may be hosted by :app:`Pyramid` or an external CDN without having to change anything. They really are static. * Templatize JS and CSS, and call ``request.static_url()`` inside their @@ -825,7 +825,7 @@ imagine you have overridden an asset defined in this manifest with a new, unknown version. By default, the cache buster will be invoked for an asset it has never seen before and will likely end up returning a cache busting token for the original asset rather than the asset that will actually end up -being served! In order to get around this issue it's possible to attach a +being served! In order to get around this issue, it's possible to attach a different :class:`pyramid.interfaces.ICacheBuster` implementation to the new assets. This would cause the original assets to be served by their manifest, and the new assets served by their own cache buster. To do this, @@ -853,14 +853,14 @@ option. For example: theme_cb = ManifestCacheBuster('theme:static/manifest.json') config.add_cache_buster('theme:static', theme_cb, explicit=True) -In the above example there is a default cache buster, ``my_cb`` for all assets -served from the ``myapp:static`` folder. This would also affect +In the above example there is a default cache buster, ``my_cb``, for all +assets served from the ``myapp:static`` folder. This would also affect ``theme:static/background.png`` when generating URLs via ``request.static_url('myapp:static/background.png')``. The ``theme_cb`` is defined explicitly for any assets loaded from the ``theme:static`` folder. Explicit cache busters have priority and thus ``theme_cb`` would be invoked for -``request.static_url('myapp:static/background.png')`` but ``my_cb`` would be -used for any other assets like +``request.static_url('myapp:static/background.png')``, but ``my_cb`` would +be used for any other assets like ``request.static_url('myapp:static/favicon.ico')``. -- cgit v1.2.3 From bc8fa64a416ce52ec5cc9fd819ce1a3caa427a19 Mon Sep 17 00:00:00 2001 From: Michael Merickel Date: Thu, 17 Dec 2015 21:18:34 -0600 Subject: update changelog for #2171 --- CHANGES.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index e9dc975a7..3c3dd6e79 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -53,11 +53,12 @@ Features See https://github.com/Pylons/pyramid/pull/1471 - Cache busting for static resources has been added and is available via a new - argument to ``pyramid.config.Configurator.add_static_view``: ``cachebust``. - Core APIs are shipped for both cache busting via query strings and - path segments and may be extended to fit into custom asset pipelines. + ``pyramid.config.Configurator.add_cache_buster`` API. Core APIs are shipped + for both cache busting via query strings and via asset manifests for + integrating into custom asset pipelines. See https://github.com/Pylons/pyramid/pull/1380 and - https://github.com/Pylons/pyramid/pull/1583 + https://github.com/Pylons/pyramid/pull/1583 and + https://github.com/Pylons/pyramid/pull/2171 - Add ``pyramid.config.Configurator.root_package`` attribute and init parameter to assist with includeable packages that wish to resolve -- cgit v1.2.3