diff options
| author | Chris McDonough <chrism@plope.com> | 2014-07-28 21:07:09 -0400 |
|---|---|---|
| committer | Chris McDonough <chrism@plope.com> | 2014-07-28 21:07:09 -0400 |
| commit | 3587a53dc28b8f6411816ccd7fd8fdee0d88acb4 (patch) | |
| tree | 5228473a3f2f93a39ab55a920c1c8bd50a48ac1e | |
| parent | 330cb23bcb601466fd51d637ca8036399fd29465 (diff) | |
| parent | 6b88bdf7680151345debec0c8651f164a149a53a (diff) | |
| download | pyramid-3587a53dc28b8f6411816ccd7fd8fdee0d88acb4.tar.gz pyramid-3587a53dc28b8f6411816ccd7fd8fdee0d88acb4.tar.bz2 pyramid-3587a53dc28b8f6411816ccd7fd8fdee0d88acb4.zip | |
Merge branch 'feature-cachebust'
| -rw-r--r-- | CHANGES.txt | 6 | ||||
| -rw-r--r-- | docs/api/interfaces.rst | 2 | ||||
| -rw-r--r-- | docs/api/static.rst | 8 | ||||
| -rw-r--r-- | docs/narr/assets.rst | 175 | ||||
| -rw-r--r-- | docs/narr/environment.rst | 22 | ||||
| -rw-r--r-- | pyramid/config/settings.py | 9 | ||||
| -rw-r--r-- | pyramid/config/views.py | 82 | ||||
| -rw-r--r-- | pyramid/interfaces.py | 64 | ||||
| -rw-r--r-- | pyramid/static.py | 103 | ||||
| -rw-r--r-- | pyramid/tests/test_config/test_settings.py | 31 | ||||
| -rw-r--r-- | pyramid/tests/test_config/test_views.py | 97 | ||||
| -rw-r--r-- | pyramid/tests/test_static.py | 163 |
12 files changed, 702 insertions, 60 deletions
diff --git a/CHANGES.txt b/CHANGES.txt index 51af8ee01..63987d980 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,12 @@ Next release ============ +Features +-------- + +- Cache busting for static resources has been added and is available via a new + argument to ``pyramid.config.Configurator.add_static_view``: ``cachebust``. + Bug Fixes --------- diff --git a/docs/api/interfaces.rst b/docs/api/interfaces.rst index d8d935afd..a62976d8a 100644 --- a/docs/api/interfaces.rst +++ b/docs/api/interfaces.rst @@ -86,3 +86,5 @@ Other Interfaces .. autointerface:: IResourceURL :members: + .. autointerface:: ICacheBuster + :members: diff --git a/docs/api/static.rst b/docs/api/static.rst index c28473584..543e526ad 100644 --- a/docs/api/static.rst +++ b/docs/api/static.rst @@ -9,3 +9,11 @@ :members: :inherited-members: + .. autoclass:: PathSegmentMd5CacheBuster + :members: + + .. autoclass:: QueryStringMd5CacheBuster + :members: + + .. autoclass:: QueryStringConstantCacheBuster + :members: diff --git a/docs/narr/assets.rst b/docs/narr/assets.rst index b0a8d18b0..95863848b 100644 --- a/docs/narr/assets.rst +++ b/docs/narr/assets.rst @@ -287,6 +287,181 @@ suggestion for a pattern; any setting name other than ``media_location`` could be used. .. index:: + single: Cache Busting + +.. _cache_busting: + +Cache Busting +------------- + +.. versionadded:: 1.6 + +In order to maximize performance of a web application, you generally want to +limit the number of times a particular client requests the same static asset. +Ideally a client would cache a particular static asset "forever", requiring +it to be sent to the client a single time. The HTTP protocol allows you to +send headers with an HTTP response that can instruct a client to cache a +particular asset for an amount of time. As long as the client has a copy of +the asset in its cache and that cache hasn't expired, the client will use the +cached copy rather than request a new copy from the server. The drawback to +sending cache headers to the client for a static asset is that at some point +the static asset may change, and then you'll want the client to load a new copy +of the asset. Under normal circumstances you'd just need to wait for the +client's cached copy to expire before they get the new version of the static +resource. + +A commonly used workaround to this problem is a technique known as "cache +busting". Cache busting schemes generally involve generating a URL for a +static asset that changes when the static asset changes. This way headers can +be sent along with the static asset instructing the client to cache the asset +for a very long time. When a static asset is changed, the URL used to refer to +it in a web page also changes, so the client sees it as a new resource and +requests a copy, regardless of any caching policy set for the resource's old +URL. + +:app:`Pyramid` can be configured to produce cache busting URLs for static +assets by passing the optional argument, ``cachebust`` to +:meth:`~pyramid.config.Configurator.add_static_view`: + +.. code-block:: python + :linenos: + + # config is an instance of pyramid.config.Configurator + config.add_static_view(name='static', path='mypackage:folder/static', + cachebust=True) + +Setting the ``cachebust`` argument instructs :app:`Pyramid` to use a cache +busting scheme which adds the md5 checksum for a static asset as a path segment +in the asset's URL: + +.. code-block:: python + :linenos: + + js_url = request.static_url('mypackage:folder/static/js/myapp.js') + # Returns: 'http://www.example.com/static/c9658b3c0a314a1ca21e5988e662a09e/js/myapp.js` + +When the asset changes, so will its md5 checksum, and therefore so will its +URL. Supplying the ``cachebust`` argument also causes the static view to set +headers instructing clients to cache the asset for ten years, unless the +``max_cache_age`` argument is also passed, in which case that value is used. + +.. note:: + + md5 checksums are cached in RAM so if you change a static resource without + restarting your application, you may still generate URLs with a stale md5 + checksum. + +Disabling the Cache Buster +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It can be useful in some situations (e.g. development) to globally disable all +configured cache busters without changing calls to +:meth:`~pyramid.config.Configurator.add_static_view`. To do this set the +``PYRAMID_PREVENT_CACHEBUST`` environment variable or the +``pyramid.prevent_cachebust`` configuration value to a true value. + +Customizing the Cache Buster +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Revisiting from the previous section: + +.. code-block:: python + :linenos: + + # config is an instance of pyramid.config.Configurator + config.add_static_view(name='static', path='mypackage:folder/static', + cachebust=True) + +Setting ``cachebust`` to ``True`` instructs :app:`Pyramid` to use a default +cache busting implementation that should work for many situations. The +``cachebust`` may be set to any object that implements the interface, +:class:`~pyramid.interfaces.ICacheBuster`. The above configuration is exactly +equivalent to: + +.. code-block:: python + :linenos: + + from pyramid.static import PathSegmentMd5CacheBuster + + # config is an instance of pyramid.config.Configurator + config.add_static_view(name='static', path='mypackage:folder/static', + cachebust=PathSegmentMd5CacheBuster()) + +:app:`Pyramid` includes a handful of ready to use cache buster implementations: +:class:`~pyramid.static.PathSegmentMd5CacheBuster`, which inserts an md5 +checksum token in the path portion of the asset's URL, +:class:`~pyramid.static.QueryStringMd5CacheBuster`, which adds an md5 checksum +token to the query string of the asset's URL, and +:class:`~pyramid.static.QueryStringConstantCacheBuster`, which adds an +arbitrary token you provide to the query string of the asset's URL. + +In order to implement your own cache buster, you can write your own class from +scratch which implements the :class:`~pyramid.interfaces.ICacheBuster` +interface. Alternatively you may choose to subclass one of the existing +implementations. One of the most likely scenarios is you'd want to change the +way the asset token is generated. To do this just subclass an existing +implementation and replace the :meth:`~pyramid.interfaces.ICacheBuster.token` +method. Here is an example which just uses Git to get the hash of the +currently checked out code: + +.. code-block:: python + :linenos: + + import os + import subprocess + from pyramid.static import PathSegmentMd5CacheBuster + + class GitCacheBuster(PathSegmentMd5CacheBuster): + """ + Assuming your code is installed as a Git checkout, as opposed to as an + egg from an egg repository like PYPI, you can use this cachebuster to + get the current commit's SHA1 to use as the cache bust token. + """ + def __init__(self): + here = os.path.dirname(os.path.abspath(__file__)) + self.sha1 = subprocess.check_output( + ['git', 'rev-parse', 'HEAD'], + cwd=here).strip() + + def token(self, pathspec): + return self.sha1 + +Choosing a Cache Buster +~~~~~~~~~~~~~~~~~~~~~~~ + +The default cache buster implementation, +:class:`~pyramid.static.PathSegmentMd5CacheBuster`, works very well assuming +that you're using :app:`Pyramid` to serve your static assets. The md5 checksum +is fine grained enough that browsers should only request new versions of +specific assets that have changed. Many caching HTTP proxies will fail to +cache a resource if the URL contains a query string. In general, therefore, +you should prefer a cache busting strategy which modifies the path segment to +a strategy which adds a query string. + +It is possible, however, that your static assets are being served by another +web server or externally on a CDN. In these cases modifying the path segment +for a static asset URL would cause the external service to fail to find the +asset, causing your customer to get a 404. In these cases you would need to +fall back to a cache buster which adds a query string. It is even possible +that there isn't a copy of your static assets available to the :app:`Pyramid` +application, so a cache busting implementation that generates md5 checksums +would fail since it can't access the assets. In such a case, +:class:`~pyramid.static.QueryStringConstantCacheBuster` is a reasonable +fallback. The following code would set up a cachebuster that just uses the +time at start up as a cachebust token: + +.. code-block:: python + :linenos: + + import time + from pyramid.static import QueryStringConstantCacheBuster + + config.add_static_view( + name='http://mycdn.example.com/', + path='mypackage:static', + cachebust=QueryStringConstantCacheBuster(str(time.time()))) + +.. index:: single: static assets view .. _advanced_static: diff --git a/docs/narr/environment.rst b/docs/narr/environment.rst index 7bac12ea7..0b06fb80b 100644 --- a/docs/narr/environment.rst +++ b/docs/narr/environment.rst @@ -157,6 +157,28 @@ feature when this is true. | | | +---------------------------------+----------------------------------+ +Preventing Cache Busting +------------------------ + +Prevent the ``cachebust`` static view configuration argument from having any +effect globally in this process when this value is true. No cache buster will +be configured or used when this is true. + +.. versionadded:: 1.6 + +.. seealso:: + + See also :ref:`cache_busting`. + ++---------------------------------+----------------------------------+ +| Environment Variable Name | Config File Setting Name | ++=================================+==================================+ +| ``PYRAMID_PREVENT_CACHEBUST`` | ``pyramid.prevent_cachebust`` | +| | or ``prevent_cachebust`` | +| | | +| | | ++---------------------------------+----------------------------------+ + Debugging All ------------- diff --git a/pyramid/config/settings.py b/pyramid/config/settings.py index 565a6699c..492b7d524 100644 --- a/pyramid/config/settings.py +++ b/pyramid/config/settings.py @@ -17,7 +17,7 @@ class SettingsConfiguratorMixin(object): def add_settings(self, settings=None, **kw): """Augment the :term:`deployment settings` with one or more - key/value pairs. + key/value pairs. You may pass a dictionary:: @@ -117,6 +117,11 @@ class Settings(dict): config_prevent_http_cache) eff_prevent_http_cache = asbool(eget('PYRAMID_PREVENT_HTTP_CACHE', config_prevent_http_cache)) + config_prevent_cachebust = self.get('prevent_cachebust', '') + config_prevent_cachebust = self.get('pyramid.prevent_cachebust', + config_prevent_cachebust) + eff_prevent_cachebust = asbool(eget('PYRAMID_PREVENT_CACHEBUST', + config_prevent_cachebust)) update = { 'debug_authorization': eff_debug_all or eff_debug_auth, @@ -128,6 +133,7 @@ class Settings(dict): 'reload_assets':eff_reload_all or eff_reload_assets, 'default_locale_name':eff_locale_name, 'prevent_http_cache':eff_prevent_http_cache, + 'prevent_cachebust':eff_prevent_cachebust, 'pyramid.debug_authorization': eff_debug_all or eff_debug_auth, 'pyramid.debug_notfound': eff_debug_all or eff_debug_notfound, @@ -138,6 +144,7 @@ class Settings(dict): 'pyramid.reload_assets':eff_reload_all or eff_reload_assets, 'pyramid.default_locale_name':eff_locale_name, 'pyramid.prevent_http_cache':eff_prevent_http_cache, + 'pyramid.prevent_cachebust':eff_prevent_cachebust, } self.update(update) diff --git a/pyramid/config/views.py b/pyramid/config/views.py index 7a6157ec8..5ca696069 100644 --- a/pyramid/config/views.py +++ b/pyramid/config/views.py @@ -34,6 +34,7 @@ from pyramid.interfaces import ( ) from pyramid import renderers +from pyramid.static import PathSegmentMd5CacheBuster from pyramid.compat import ( string_types, @@ -44,11 +45,6 @@ from pyramid.compat import ( is_nonstr_iter ) -from pyramid.encode import ( - quote_plus, - urlencode, -) - from pyramid.exceptions import ( ConfigurationError, PredicateMismatch, @@ -302,7 +298,7 @@ class ViewDeriver(object): raise PredicateMismatch( 'predicate mismatch for view %s (%s)' % ( view_name, predicate.text())) - return view(context, request) + return view(context, request) def checker(context, request): return all((predicate(context, request) for predicate in preds)) @@ -894,8 +890,8 @@ class ViewsConfiguratorMixin(object): request_param - This value can be any string or any sequence of strings. A view - declaration with this argument ensures that the view will only be + This value can be any string or any sequence of strings. A view + declaration with this argument ensures that the view will only be called when the :term:`request` has a key in the ``request.params`` dictionary (an HTTP ``GET`` or ``POST`` variable) that has a name which matches the supplied value (if the value is a string) @@ -1001,7 +997,7 @@ class ViewsConfiguratorMixin(object): Note that using this feature requires a :term:`session factory` to have been configured. - + .. versionadded:: 1.4a2 physical_path @@ -1039,7 +1035,7 @@ class ViewsConfiguratorMixin(object): This value should be a sequence of references to custom predicate callables. Use custom predicates when no set of predefined predicates do what you need. Custom predicates - can be combined with predefined predicates as necessary. + can be combined with predefined predicates as necessary. Each custom predicate callable should accept two arguments: ``context`` and ``request`` and should return either ``True`` or ``False`` after doing arbitrary evaluation of @@ -1074,7 +1070,7 @@ class ViewsConfiguratorMixin(object): DeprecationWarning, stacklevel=4 ) - + view = self.maybe_dotted(view) context = self.maybe_dotted(context) for_ = self.maybe_dotted(for_) @@ -1160,7 +1156,7 @@ class ViewsConfiguratorMixin(object): view_desc = self.object_description(view) tmpl_intr = None - + view_intr = self.introspectable('views', discriminator, view_desc, @@ -1569,7 +1565,7 @@ class ViewsConfiguratorMixin(object): wrapper=None, route_name=None, request_type=None, - request_method=None, + request_method=None, request_param=None, containment=None, xhr=None, @@ -1612,7 +1608,7 @@ class ViewsConfiguratorMixin(object): '%s may not be used as an argument to add_forbidden_view' % arg ) - + settings = dict( view=view, context=HTTPForbidden, @@ -1623,7 +1619,7 @@ class ViewsConfiguratorMixin(object): containment=containment, xhr=xhr, accept=accept, - header=header, + header=header, path_info=path_info, custom_predicates=custom_predicates, decorator=decorator, @@ -1638,7 +1634,7 @@ class ViewsConfiguratorMixin(object): return self.add_view(**settings) set_forbidden_view = add_forbidden_view # deprecated sorta-bw-compat alias - + @viewdefaults @action_method def add_notfound_view( @@ -1649,7 +1645,7 @@ class ViewsConfiguratorMixin(object): wrapper=None, route_name=None, request_type=None, - request_method=None, + request_method=None, request_param=None, containment=None, xhr=None, @@ -1700,7 +1696,7 @@ class ViewsConfiguratorMixin(object): '%s may not be used as an argument to add_notfound_view' % arg ) - + settings = dict( view=view, context=HTTPNotFound, @@ -1711,7 +1707,7 @@ class ViewsConfiguratorMixin(object): containment=containment, xhr=xhr, accept=accept, - header=header, + header=header, path_info=path_info, custom_predicates=custom_predicates, decorator=decorator, @@ -1786,7 +1782,20 @@ class ViewsConfiguratorMixin(object): ``Expires`` and ``Cache-Control`` headers for static assets served. Note that this argument has no effect when the ``name`` is a *url prefix*. By default, this argument is ``None``, meaning that no - particular Expires or Cache-Control headers are set in the response. + particular Expires or Cache-Control headers are set in the response, + unless ``cachebust`` is specified. + + The ``cachebust`` keyword argument may be set to cause + :meth:`~pyramid.request.Request.static_url` to use cache busting when + generating URLs. See :ref:`cache_busting` for general information + about cache busting. The value of the ``cachebust`` argument may be + ``True``, in which case a default cache busting implementation is used. + The value of the ``cachebust`` argument may also be an object which + implements :class:`~pyramid.interfaces.ICacheBuster`. See the + :mod:`~pyramid.static` module for some implementations. If the + ``cachebust`` argument is provided, the default for ``cache_max_age`` + is modified to be ten years. ``cache_max_age`` may still be explicitly + provided to override this default. The ``permission`` keyword argument is used to specify the :term:`permission` required by a user to execute the static view. By @@ -1884,6 +1893,8 @@ def isexception(o): @implementer(IStaticURLInfo) class StaticURLInfo(object): + # Indirection for testing + _default_cachebust = PathSegmentMd5CacheBuster def _get_registrations(self, registry): try: @@ -1897,11 +1908,14 @@ class StaticURLInfo(object): registry = request.registry except AttributeError: # bw compat (for tests) registry = get_current_registry() - for (url, spec, route_name) in self._get_registrations(registry): + registrations = self._get_registrations(registry) + for (url, spec, route_name, cachebust) in registrations: if path.startswith(spec): subpath = path[len(spec):] if WIN: # pragma: no cover subpath = subpath.replace('\\', '/') # windows + if cachebust: + subpath, kw = cachebust(subpath, kw) if url is None: kw['subpath'] = subpath return request.route_url(route_name, **kw) @@ -1941,6 +1955,21 @@ class StaticURLInfo(object): # make sure it ends with a slash name = name + '/' + if config.registry.settings.get('pyramid.prevent_cachebust'): + cb = None + else: + cb = extra.pop('cachebust', None) + if cb is True: + cb = self._default_cachebust() + if cb: + def cachebust(subpath, kw): + token = cb.token(spec + subpath) + subpath_tuple = tuple(subpath.split('/')) + subpath_tuple, kw = cb.pregenerate(token, subpath_tuple, kw) + return '/'.join(subpath_tuple), kw + else: + cachebust = None + if url_parse(name).netloc: # it's a URL # url, spec, route_name @@ -1949,10 +1978,14 @@ class StaticURLInfo(object): else: # it's a view name url = None - cache_max_age = extra.pop('cache_max_age', None) + ten_years = 10 * 365 * 24 * 60 * 60 # more or less + default = ten_years if cb else None + cache_max_age = extra.pop('cache_max_age', default) + # create a view + cb_match = getattr(cb, 'match', None) view = static_view(spec, cache_max_age=cache_max_age, - use_subpath=True) + use_subpath=True, cachebust_match=cb_match) # Mutate extra to allow factory, etc to be passed through here. # Treat permission specially because we'd like to default to @@ -1993,7 +2026,7 @@ class StaticURLInfo(object): registrations.pop(idx) # url, spec, route_name - registrations.append((url, spec, route_name)) + registrations.append((url, spec, route_name, cachebust)) intr = config.introspectable('static views', name, @@ -2004,4 +2037,3 @@ class StaticURLInfo(object): config.action(None, callable=register, introspectables=(intr,)) - diff --git a/pyramid/interfaces.py b/pyramid/interfaces.py index aa2dbdafd..c5a70dbfd 100644 --- a/pyramid/interfaces.py +++ b/pyramid/interfaces.py @@ -708,7 +708,7 @@ class IRoute(Interface): pregenerator = Attribute('This attribute should either be ``None`` or ' 'a callable object implementing the ' '``IRoutePregenerator`` interface') - + def match(path): """ If the ``path`` passed to this function can be matched by the @@ -803,7 +803,7 @@ class IContextURL(IResourceURL): # <__main__.Fudge object at 0x1cda890> # <object object at 0x7fa678f3e2a0> <object object at 0x7fa678f3e2a0> # <__main__.Another object at 0x1cda850> - + def virtual_root(): """ Return the virtual root related to a request and the current context""" @@ -837,9 +837,9 @@ class IPEP302Loader(Interface): def get_code(fullname): """ Return the code object for the module identified by 'fullname'. - + Return 'None' if it's a built-in or extension module. - + If the loader doesn't have the code object but it does have the source code, return the compiled source code. @@ -848,16 +848,16 @@ class IPEP302Loader(Interface): def get_source(fullname): """ Return the source code for the module identified by 'fullname'. - + Return a string, using newline characters for line endings, or None if the source is not available. - + Raise ImportError if the module can't be found by the importer at all. """ def get_filename(fullname): """ Return the value of '__file__' if the named module was loaded. - + If the module is not found, raise ImportError. """ @@ -1164,6 +1164,56 @@ class IJSONAdapter(Interface): class IPredicateList(Interface): """ Interface representing a predicate list """ +class ICacheBuster(Interface): + """ + Instances of ``ICacheBuster`` may be provided as arguments to + :meth:`~pyramid.config.Configurator.add_static_view`. Instances of + ``ICacheBuster`` provide mechanisms for generating a cache bust token for + a static asset, modifying a static asset URL to include a cache bust token, + and, optionally, unmodifying a static asset URL in order to look up an + asset. See :ref:`cache_busting`. + + .. versionadded:: 1.6 + """ + def token(pathspec): + """ + Computes and returns a token string used for cache busting. + ``pathspec`` is the path specification for the resource to be cache + busted. """ + + def pregenerate(token, subpath, kw): + """ + Modifies a subpath and/or keyword arguments from which a static asset + URL will be computed during URL generation. The ``token`` argument is + a token string computed by + :meth:`~pyramid.interfaces.ICacheBuster.token` for a particular asset. + The ``subpath`` argument is a tuple of path elements that represent the + portion of the asset URL which is used to find the asset. The ``kw`` + argument is a dict of keywords that are to be passed eventually to + :meth:`~pyramid.request.Request.route_url` for URL generation. The + return value should be a two-tuple of ``(subpath, kw)`` which are + versions of the same arguments modified to include the cachebust token + in the generated URL. + """ + + def match(subpath): + """ + Performs the logical inverse of + :meth:`~pyramid.interfaces.ICacheBuster.pregenerate` by taking a + subpath from a cache busted URL and removing the cache bust token, so + that :app:`Pyramid` can find the underlying asset. + + ``subpath`` is the subpath portion of the URL for an incoming request + for a static asset. The return value should be the same tuple with the + cache busting token elided. + + If the cache busting scheme in use doesn't specifically modify the path + portion of the generated URL (e.g. it adds a query string), a method + which implements this interface may not be necessary. It is + permissible for an instance of + :class:`~pyramid.interfaces.ICacheBuster` to omit this method. + """ + # configuration phases: a lower phase number means the actions associated # with this phase will be executed earlier than those with later phase # numbers. The default phase number is 0, FTR. diff --git a/pyramid/static.py b/pyramid/static.py index aa67568d3..c4a9e3cc4 100644 --- a/pyramid/static.py +++ b/pyramid/static.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import hashlib import os from os.path import ( @@ -26,7 +27,7 @@ from pyramid.httpexceptions import ( HTTPMovedPermanently, ) -from pyramid.path import caller_package +from pyramid.path import AssetResolver, caller_package from pyramid.response import FileResponse from pyramid.traversal import traversal_path_info @@ -78,7 +79,7 @@ class static_view(object): """ def __init__(self, root_dir, cache_max_age=3600, package_name=None, - use_subpath=False, index='index.html'): + use_subpath=False, index='index.html', cachebust_match=None): # package_name is for bw compat; it is preferred to pass in a # package-relative path as root_dir # (e.g. ``anotherpackage:foo/static``). @@ -91,13 +92,15 @@ class static_view(object): self.docroot = docroot self.norm_docroot = normcase(normpath(docroot)) self.index = index + self.cachebust_match = cachebust_match def __call__(self, context, request): if self.use_subpath: path_tuple = request.subpath else: path_tuple = traversal_path_info(request.environ['PATH_INFO']) - + if self.cachebust_match: + path_tuple = self.cachebust_match(path_tuple) path = _secure_path(path_tuple) if path is None: @@ -154,3 +157,97 @@ def _secure_path(path_tuple): encoded = slash.join(path_tuple) # will be unicode return encoded +def _generate_md5(spec): + asset = AssetResolver(None).resolve(spec) + md5 = hashlib.md5() + with asset.stream() as stream: + for block in iter(lambda: stream.read(4096), b''): + md5.update(block) + return md5.hexdigest() + +class Md5AssetTokenGenerator(object): + """ + A mixin class which provides an implementation of + :meth:`~pyramid.interfaces.ICacheBuster.target` which generates an md5 + checksum token for an asset, caching it for subsequent calls. + """ + def __init__(self): + self.token_cache = {} + + def token(self, pathspec): + # An astute observer will notice that this use of token_cache doesn't + # look particularly thread safe. Basic read/write operations on Python + # dicts, however, are atomic, so simply accessing and writing values + # to the dict shouldn't cause a segfault or other catastrophic failure. + # (See: http://effbot.org/pyfaq/what-kinds-of-global-value-mutation-are-thread-safe.htm) + # + # We do have a race condition that could result in the same md5 + # checksum getting computed twice or more times in parallel. Since + # the program would still function just fine if this were to occur, + # the extra overhead of using locks to serialize access to the dict + # seems an unnecessary burden. + # + token = self.token_cache.get(pathspec) + if not token: + self.token_cache[pathspec] = token = _generate_md5(pathspec) + return token + +class PathSegmentMd5CacheBuster(Md5AssetTokenGenerator): + """ + An implementation of :class:`~pyramid.interfaces.ICacheBuster` which + inserts an md5 checksum token for cache busting in the path portion of an + asset URL. Generated md5 checksums are cached in order to speed up + subsequent calls. + + .. versionadded:: 1.6 + """ + def pregenerate(self, token, subpath, kw): + return (token,) + subpath, kw + + def match(self, subpath): + return subpath[1:] + +class QueryStringMd5CacheBuster(Md5AssetTokenGenerator): + """ + An implementation of :class:`~pyramid.interfaces.ICacheBuster` which adds + an md5 checksum token for cache busting in the query string of an asset + URL. Generated md5 checksums are cached in order to speed up subsequent + calls. + + The optional ``param`` argument determines the name of the parameter added + to the query string and defaults to ``'x'``. + + .. versionadded:: 1.6 + """ + def __init__(self, param='x'): + super(QueryStringMd5CacheBuster, self).__init__() + self.param = param + + def pregenerate(self, token, subpath, kw): + query = kw.setdefault('_query', {}) + if isinstance(query, dict): + query[self.param] = token + else: + kw['_query'] = tuple(query) + ((self.param, token),) + return subpath, kw + +class QueryStringConstantCacheBuster(QueryStringMd5CacheBuster): + """ + An implementation of :class:`~pyramid.interfaces.ICacheBuster` which adds + an arbitrary token for cache busting in the query string of an asset URL. + + The ``token`` parameter is the token string to use for cache busting and + will be the same for every request. + + The optional ``param`` argument determines the name of the parameter added + to the query string and defaults to ``'x'``. + + .. versionadded:: 1.6 + """ + def __init__(self, token, param='x'): + self._token = token + self.param = param + + def token(self, pathspec): + return self._token + diff --git a/pyramid/tests/test_config/test_settings.py b/pyramid/tests/test_config/test_settings.py index c74f96375..d2a98b347 100644 --- a/pyramid/tests/test_config/test_settings.py +++ b/pyramid/tests/test_config/test_settings.py @@ -57,7 +57,7 @@ class TestSettingsConfiguratorMixin(unittest.TestCase): self.assertEqual(settings['a'], 1) class TestSettings(unittest.TestCase): - + def _getTargetClass(self): from pyramid.config.settings import Settings return Settings @@ -131,6 +131,35 @@ class TestSettings(unittest.TestCase): self.assertEqual(result['prevent_http_cache'], True) self.assertEqual(result['pyramid.prevent_http_cache'], True) + def test_prevent_cachebust(self): + settings = self._makeOne({}) + self.assertEqual(settings['prevent_cachebust'], False) + self.assertEqual(settings['pyramid.prevent_cachebust'], False) + result = self._makeOne({'prevent_cachebust':'false'}) + self.assertEqual(result['prevent_cachebust'], False) + self.assertEqual(result['pyramid.prevent_cachebust'], False) + result = self._makeOne({'prevent_cachebust':'t'}) + self.assertEqual(result['prevent_cachebust'], True) + self.assertEqual(result['pyramid.prevent_cachebust'], True) + result = self._makeOne({'prevent_cachebust':'1'}) + self.assertEqual(result['prevent_cachebust'], True) + self.assertEqual(result['pyramid.prevent_cachebust'], True) + result = self._makeOne({'pyramid.prevent_cachebust':'t'}) + self.assertEqual(result['prevent_cachebust'], True) + self.assertEqual(result['pyramid.prevent_cachebust'], True) + result = self._makeOne({}, {'PYRAMID_PREVENT_CACHEBUST':'1'}) + self.assertEqual(result['prevent_cachebust'], True) + self.assertEqual(result['pyramid.prevent_cachebust'], True) + result = self._makeOne({'prevent_cachebust':'false', + 'pyramid.prevent_cachebust':'1'}) + self.assertEqual(result['prevent_cachebust'], True) + self.assertEqual(result['pyramid.prevent_cachebust'], True) + result = self._makeOne({'prevent_cachebust':'false', + 'pyramid.prevent_cachebust':'f'}, + {'PYRAMID_PREVENT_CACHEBUST':'1'}) + self.assertEqual(result['prevent_cachebust'], True) + self.assertEqual(result['pyramid.prevent_cachebust'], True) + def test_reload_templates(self): settings = self._makeOne({}) self.assertEqual(settings['reload_templates'], False) diff --git a/pyramid/tests/test_config/test_views.py b/pyramid/tests/test_config/test_views.py index 57bb5e9d0..a0d9ee0c3 100644 --- a/pyramid/tests/test_config/test_views.py +++ b/pyramid/tests/test_config/test_views.py @@ -113,7 +113,7 @@ class TestViewsConfigurationMixin(unittest.TestCase): config.add_view(renderer='dummy.pt') view = self._getViewCallable(config) self.assertRaises(ValueError, view, None, None) - + def test_add_view_with_tmpl_renderer_factory_no_renderer_factory(self): config = self._makeOne(autocommit=True) introspector = DummyIntrospector() @@ -136,7 +136,7 @@ class TestViewsConfigurationMixin(unittest.TestCase): ('renderer factories', '.pt') in introspector.related[-1]) view = self._getViewCallable(config) self.assertTrue(b'Hello!' in view(None, None).body) - + def test_add_view_wrapped_view_is_decorated(self): def view(request): # request-only wrapper """ """ @@ -3742,8 +3742,9 @@ class TestStaticURLInfo(unittest.TestCase): def test_generate_registration_miss(self): inst = self._makeOne() - registrations = [(None, 'spec', 'route_name'), - ('http://example.com/foo/', 'package:path/', None)] + registrations = [ + (None, 'spec', 'route_name', None), + ('http://example.com/foo/', 'package:path/', None, None)] inst._get_registrations = lambda *x: registrations request = self._makeRequest() result = inst.generate('package:path/abc', request) @@ -3751,7 +3752,8 @@ class TestStaticURLInfo(unittest.TestCase): def test_generate_registration_no_registry_on_request(self): inst = self._makeOne() - registrations = [('http://example.com/foo/', 'package:path/', None)] + registrations = [ + ('http://example.com/foo/', 'package:path/', None, None)] inst._get_registrations = lambda *x: registrations request = self._makeRequest() del request.registry @@ -3760,7 +3762,8 @@ class TestStaticURLInfo(unittest.TestCase): def test_generate_slash_in_name1(self): inst = self._makeOne() - registrations = [('http://example.com/foo/', 'package:path/', None)] + registrations = [ + ('http://example.com/foo/', 'package:path/', None, None)] inst._get_registrations = lambda *x: registrations request = self._makeRequest() result = inst.generate('package:path/abc', request) @@ -3768,7 +3771,8 @@ class TestStaticURLInfo(unittest.TestCase): def test_generate_slash_in_name2(self): inst = self._makeOne() - registrations = [('http://example.com/foo/', 'package:path/', None)] + registrations = [ + ('http://example.com/foo/', 'package:path/', None, None)] inst._get_registrations = lambda *x: registrations request = self._makeRequest() result = inst.generate('package:path/', request) @@ -3788,7 +3792,7 @@ class TestStaticURLInfo(unittest.TestCase): def test_generate_route_url(self): inst = self._makeOne() - registrations = [(None, 'package:path/', '__viewname/')] + registrations = [(None, 'package:path/', '__viewname/', None)] inst._get_registrations = lambda *x: registrations def route_url(n, **kw): self.assertEqual(n, '__viewname/') @@ -3801,7 +3805,7 @@ class TestStaticURLInfo(unittest.TestCase): def test_generate_url_unquoted_local(self): inst = self._makeOne() - registrations = [(None, 'package:path/', '__viewname/')] + registrations = [(None, 'package:path/', '__viewname/', None)] inst._get_registrations = lambda *x: registrations def route_url(n, **kw): self.assertEqual(n, '__viewname/') @@ -3814,7 +3818,7 @@ class TestStaticURLInfo(unittest.TestCase): def test_generate_url_quoted_remote(self): inst = self._makeOne() - registrations = [('http://example.com/', 'package:path/', None)] + registrations = [('http://example.com/', 'package:path/', None, None)] inst._get_registrations = lambda *x: registrations request = self._makeRequest() result = inst.generate('package:path/abc def', request, a=1) @@ -3822,7 +3826,7 @@ class TestStaticURLInfo(unittest.TestCase): def test_generate_url_with_custom_query(self): inst = self._makeOne() - registrations = [('http://example.com/', 'package:path/', None)] + registrations = [('http://example.com/', 'package:path/', None, None)] inst._get_registrations = lambda *x: registrations request = self._makeRequest() result = inst.generate('package:path/abc def', request, a=1, @@ -3832,7 +3836,7 @@ class TestStaticURLInfo(unittest.TestCase): def test_generate_url_with_custom_anchor(self): inst = self._makeOne() - registrations = [('http://example.com/', 'package:path/', None)] + registrations = [('http://example.com/', 'package:path/', None, None)] inst._get_registrations = lambda *x: registrations request = self._makeRequest() uc = text_(b'La Pe\xc3\xb1a', 'utf-8') @@ -3841,33 +3845,50 @@ class TestStaticURLInfo(unittest.TestCase): self.assertEqual(result, 'http://example.com/abc%20def#La%20Pe%C3%B1a') + def test_generate_url_cachebust(self): + def cachebust(subpath, kw): + kw['foo'] = 'bar' + return 'foo' + '/' + subpath, kw + inst = self._makeOne() + registrations = [(None, 'package:path/', '__viewname', cachebust)] + inst._get_registrations = lambda *x: registrations + request = self._makeRequest() + def route_url(n, **kw): + self.assertEqual(n, '__viewname') + self.assertEqual(kw, {'subpath':'foo/abc', 'foo':'bar'}) + request.route_url = route_url + inst.generate('package:path/abc', request) + def test_add_already_exists(self): inst = self._makeOne() config = self._makeConfig( [('http://example.com/', 'package:path/', None)]) inst.add(config, 'http://example.com', 'anotherpackage:path') - expected = [('http://example.com/', 'anotherpackage:path/', None)] + expected = [ + ('http://example.com/', 'anotherpackage:path/', None, None)] self._assertRegistrations(config, expected) def test_add_url_withendslash(self): inst = self._makeOne() config = self._makeConfig() inst.add(config, 'http://example.com/', 'anotherpackage:path') - expected = [('http://example.com/', 'anotherpackage:path/', None)] + expected = [ + ('http://example.com/', 'anotherpackage:path/', None, None)] self._assertRegistrations(config, expected) def test_add_url_noendslash(self): inst = self._makeOne() config = self._makeConfig() inst.add(config, 'http://example.com', 'anotherpackage:path') - expected = [('http://example.com/', 'anotherpackage:path/', None)] + expected = [ + ('http://example.com/', 'anotherpackage:path/', None, None)] self._assertRegistrations(config, expected) def test_add_url_noscheme(self): inst = self._makeOne() config = self._makeConfig() inst.add(config, '//example.com', 'anotherpackage:path') - expected = [('//example.com/', 'anotherpackage:path/', None)] + expected = [('//example.com/', 'anotherpackage:path/', None, None)] self._assertRegistrations(config, expected) def test_add_viewname(self): @@ -3876,7 +3897,7 @@ class TestStaticURLInfo(unittest.TestCase): config = self._makeConfig() inst = self._makeOne() inst.add(config, 'view', 'anotherpackage:path', cache_max_age=1) - expected = [(None, 'anotherpackage:path/', '__view/')] + expected = [(None, 'anotherpackage:path/', '__view/', None)] self._assertRegistrations(config, expected) self.assertEqual(config.route_args, ('__view/', 'view/*subpath')) self.assertEqual(config.view_kw['permission'], NO_PERMISSION_REQUIRED) @@ -3887,7 +3908,7 @@ class TestStaticURLInfo(unittest.TestCase): config.route_prefix = '/abc' inst = self._makeOne() inst.add(config, 'view', 'anotherpackage:path',) - expected = [(None, 'anotherpackage:path/', '__/abc/view/')] + expected = [(None, 'anotherpackage:path/', '__/abc/view/', None)] self._assertRegistrations(config, expected) self.assertEqual(config.route_args, ('__/abc/view/', 'view/*subpath')) @@ -3904,7 +3925,7 @@ class TestStaticURLInfo(unittest.TestCase): inst.add(config, 'view', 'anotherpackage:path', cache_max_age=1, context=DummyContext) self.assertEqual(config.view_kw['context'], DummyContext) - + def test_add_viewname_with_for_(self): config = self._makeConfig() inst = self._makeOne() @@ -3920,6 +3941,34 @@ class TestStaticURLInfo(unittest.TestCase): self.assertEqual(config.view_kw['renderer'], 'mypackage:templates/index.pt') + def test_add_cachebust_default(self): + config = self._makeConfig() + inst = self._makeOne() + inst._default_cachebust = DummyCacheBuster + inst.add(config, 'view', 'mypackage:path', cachebust=True) + cachebust = config.registry._static_url_registrations[0][3] + subpath, kw = cachebust('some/path', {}) + self.assertEqual(subpath, 'some/path') + self.assertEqual(kw['x'], 'foo') + + def test_add_cachebust_prevented(self): + config = self._makeConfig() + config.registry.settings['pyramid.prevent_cachebust'] = True + inst = self._makeOne() + inst.add(config, 'view', 'mypackage:path', cachebust=True) + cachebust = config.registry._static_url_registrations[0][3] + self.assertEqual(cachebust, None) + + def test_add_cachebust_custom(self): + config = self._makeConfig() + inst = self._makeOne() + inst.add(config, 'view', 'mypackage:path', + cachebust=DummyCacheBuster()) + cachebust = config.registry._static_url_registrations[0][3] + subpath, kw = cachebust('some/path', {}) + self.assertEqual(subpath, 'some/path') + self.assertEqual(kw['x'], 'foo') + class Test_view_description(unittest.TestCase): def _callFUT(self, view): from pyramid.config.views import view_description @@ -3939,7 +3988,8 @@ class Test_view_description(unittest.TestCase): class DummyRegistry: - pass + def __init__(self): + self.settings = {} from zope.interface import implementer from pyramid.interfaces import IResponse @@ -4025,6 +4075,13 @@ class DummyMultiView: def __permitted__(self, context, request): """ """ +class DummyCacheBuster(object): + def token(self, pathspec): + return 'foo' + def pregenerate(self, token, subpath, kw): + kw['x'] = token + return subpath, kw + def parse_httpdate(s): import datetime # cannot use %Z, must use literal GMT; Jython honors timezone diff --git a/pyramid/tests/test_static.py b/pyramid/tests/test_static.py index 94497d4f6..2f4de249e 100644 --- a/pyramid/tests/test_static.py +++ b/pyramid/tests/test_static.py @@ -26,7 +26,7 @@ class Test_static_view_use_subpath_False(unittest.TestCase): if kw is not None: environ.update(kw) return Request(environ=environ) - + def test_ctor_defaultargs(self): inst = self._makeOne('package:resource_name') self.assertEqual(inst.package_name, 'package') @@ -110,6 +110,14 @@ class Test_static_view_use_subpath_False(unittest.TestCase): response = inst(context, request) self.assertTrue(b'<html>static</html>' in response.body) + def test_cachebust_match(self): + inst = self._makeOne('pyramid.tests:fixtures/static') + inst.cachebust_match = lambda subpath: subpath[1:] + request = self._makeRequest({'PATH_INFO':'/foo/index.html'}) + context = DummyContext() + response = inst(context, request) + self.assertTrue(b'<html>static</html>' in response.body) + def test_resource_is_file_with_wsgi_file_wrapper(self): from pyramid.response import _BLOCK_SIZE inst = self._makeOne('pyramid.tests:fixtures/static') @@ -218,7 +226,7 @@ class Test_static_view_use_subpath_True(unittest.TestCase): if kw is not None: environ.update(kw) return Request(environ=environ) - + def test_ctor_defaultargs(self): inst = self._makeOne('package:resource_name') self.assertEqual(inst.package_name, 'package') @@ -273,7 +281,7 @@ class Test_static_view_use_subpath_True(unittest.TestCase): context = DummyContext() from pyramid.httpexceptions import HTTPNotFound self.assertRaises(HTTPNotFound, inst, context, request) - + def test_oob_os_sep(self): import os inst = self._makeOne('pyramid.tests:fixtures/static') @@ -360,6 +368,155 @@ class Test_static_view_use_subpath_True(unittest.TestCase): from pyramid.httpexceptions import HTTPNotFound self.assertRaises(HTTPNotFound, inst, context, request) +class TestMd5AssetTokenGenerator(unittest.TestCase): + _fspath = None + _tmp = None + + @property + def fspath(self): + if self._fspath: + return self._fspath + + import os + import tempfile + self._tmp = tmp = tempfile.mkdtemp() + self._fspath = os.path.join(tmp, 'test.txt') + return self._fspath + + def tearDown(self): + import shutil + if self._tmp: + shutil.rmtree(self._tmp) + + def _makeOne(self): + from pyramid.static import Md5AssetTokenGenerator as cls + return cls() + + def test_package_resource(self): + fut = self._makeOne().token + expected = '76d653a3a044e2f4b38bb001d283e3d9' + token = fut('pyramid.tests:fixtures/static/index.html') + self.assertEqual(token, expected) + + def test_filesystem_resource(self): + fut = self._makeOne().token + expected = 'd5155f250bef0e9923e894dbc713c5dd' + with open(self.fspath, 'w') as f: + f.write("Are we rich yet?") + token = fut(self.fspath) + self.assertEqual(token, expected) + + def test_cache(self): + fut = self._makeOne().token + expected = 'd5155f250bef0e9923e894dbc713c5dd' + with open(self.fspath, 'w') as f: + f.write("Are we rich yet?") + token = fut(self.fspath) + self.assertEqual(token, expected) + + # md5 shouldn't change because we've cached it + with open(self.fspath, 'w') as f: + f.write("Sorry for the convenience.") + token = fut(self.fspath) + self.assertEqual(token, expected) + +class TestPathSegmentMd5CacheBuster(unittest.TestCase): + + def _makeOne(self): + from pyramid.static import PathSegmentMd5CacheBuster as cls + inst = cls() + inst.token = lambda pathspec: 'foo' + return inst + + def test_token(self): + fut = self._makeOne().token + self.assertEqual(fut('whatever'), 'foo') + + def test_pregenerate(self): + fut = self._makeOne().pregenerate + self.assertEqual(fut('foo', ('bar',), 'kw'), (('foo', 'bar'), 'kw')) + + def test_match(self): + fut = self._makeOne().match + self.assertEqual(fut(('foo', 'bar')), ('bar',)) + +class TestQueryStringMd5CacheBuster(unittest.TestCase): + + def _makeOne(self, param=None): + from pyramid.static import QueryStringMd5CacheBuster as cls + if param: + inst = cls(param) + else: + inst = cls() + inst.token = lambda pathspec: 'foo' + return inst + + def test_token(self): + fut = self._makeOne().token + self.assertEqual(fut('whatever'), 'foo') + + def test_pregenerate(self): + fut = self._makeOne().pregenerate + self.assertEqual( + fut('foo', ('bar',), {}), + (('bar',), {'_query': {'x': 'foo'}})) + + def test_pregenerate_change_param(self): + fut = self._makeOne('y').pregenerate + self.assertEqual( + fut('foo', ('bar',), {}), + (('bar',), {'_query': {'y': 'foo'}})) + + def test_pregenerate_query_is_already_tuples(self): + fut = self._makeOne().pregenerate + self.assertEqual( + fut('foo', ('bar',), {'_query': [('a', 'b')]}), + (('bar',), {'_query': (('a', 'b'), ('x', 'foo'))})) + + def test_pregenerate_query_is_tuple_of_tuples(self): + fut = self._makeOne().pregenerate + self.assertEqual( + fut('foo', ('bar',), {'_query': (('a', 'b'),)}), + (('bar',), {'_query': (('a', 'b'), ('x', 'foo'))})) + +class TestQueryStringConstantCacheBuster(TestQueryStringMd5CacheBuster): + + def _makeOne(self, param=None): + from pyramid.static import QueryStringConstantCacheBuster as cls + if param: + inst = cls('foo', param) + else: + inst = cls('foo') + return inst + + def test_token(self): + fut = self._makeOne().token + self.assertEqual(fut('whatever'), 'foo') + + def test_pregenerate(self): + fut = self._makeOne().pregenerate + self.assertEqual( + fut('foo', ('bar',), {}), + (('bar',), {'_query': {'x': 'foo'}})) + + def test_pregenerate_change_param(self): + fut = self._makeOne('y').pregenerate + self.assertEqual( + fut('foo', ('bar',), {}), + (('bar',), {'_query': {'y': 'foo'}})) + + def test_pregenerate_query_is_already_tuples(self): + fut = self._makeOne().pregenerate + self.assertEqual( + fut('foo', ('bar',), {'_query': [('a', 'b')]}), + (('bar',), {'_query': (('a', 'b'), ('x', 'foo'))})) + + def test_pregenerate_query_is_tuple_of_tuples(self): + fut = self._makeOne().pregenerate + self.assertEqual( + fut('foo', ('bar',), {'_query': (('a', 'b'),)}), + (('bar',), {'_query': (('a', 'b'), ('x', 'foo'))})) + class DummyContext: pass |
