From d809ac74d19342bcc84e4fe043697709b2001cc0 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Tue, 30 Jun 2009 21:02:00 +0000 Subject: - Add a ``reload_resources`` configuration file setting (aka the ``BFG_RELOAD_RESOURCES`` environment variable). When this is set to true, the server never needs to be restarted when moving files between directory resource overrides (esp. for templates currently). - Add a ``reload_all`` configuration file setting (aka the ``BFG_RELOAD_ALL`` environment variable) that implies both ``reload_resources`` and ``reload_templates``. - The ``static`` helper view class now uses a ``PackageURLParser`` in order to allow for the overriding of static resources (CSS / logo files, etc) using the ``resource`` ZCML directive. The ``PackageURLParser`` class was added to a (new) ``static`` module in BFG; it is a subclass of the ``StaticURLParser`` class in ``paste.urlparser``. - The ``repoze.bfg.templating.renderer_from_cache`` function now checks for the ``reload_resources`` setting; if it's true, it does not register a template renderer (it won't use the registry as a template renderer cache). - Add ``pkg_resources`` to the glossary. - Update the "Environment" docs to note the existence of ``reload_resources`` and ``reload_all``. - Use a colon instead of a tab as the separator between package name and relpath to form the "spec" when register a ITemplateRenderer. --- CHANGES.txt | 29 ++++ docs/glossary.rst | 7 + docs/narr/environment.rst | 54 ++++++- docs/narr/hooks.rst | 2 + repoze/bfg/settings.py | 9 +- repoze/bfg/static.py | 78 ++++++++++ repoze/bfg/templating.py | 19 ++- repoze/bfg/tests/fixtures/static/index.html | 1 + repoze/bfg/tests/fixtures/static/subdir/index.html | 1 + repoze/bfg/tests/test_settings.py | 36 +++++ repoze/bfg/tests/test_static.py | 157 +++++++++++++++++++++ repoze/bfg/tests/test_templating.py | 31 +++- repoze/bfg/tests/test_view.py | 26 +++- repoze/bfg/view.py | 51 +++++-- 14 files changed, 474 insertions(+), 27 deletions(-) create mode 100644 repoze/bfg/static.py create mode 100644 repoze/bfg/tests/fixtures/static/index.html create mode 100644 repoze/bfg/tests/fixtures/static/subdir/index.html create mode 100644 repoze/bfg/tests/test_static.py diff --git a/CHANGES.txt b/CHANGES.txt index d8a72991b..11a3407e2 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,35 @@ Next release ============ +- Add a ``reload_resources`` configuration file setting (aka the + ``BFG_RELOAD_RESOURCES`` environment variable). When this is set to + true, the server never needs to be restarted when moving files + between directory resource overrides (esp. for templates currently). + +- Add a ``reload_all`` configuration file setting (aka the + ``BFG_RELOAD_ALL`` environment variable) that implies both + ``reload_resources`` and ``reload_templates``. + +- The ``static`` helper view class now uses a ``PackageURLParser`` in + order to allow for the overriding of static resources (CSS / logo + files, etc) using the ``resource`` ZCML directive. The + ``PackageURLParser`` class was added to a (new) ``static`` module in + BFG; it is a subclass of the ``StaticURLParser`` class in + ``paste.urlparser``. + +- The ``repoze.bfg.templating.renderer_from_cache`` function now + checks for the ``reload_resources`` setting; if it's true, it does + not register a template renderer (it won't use the registry as a + template renderer cache). + +- Add ``pkg_resources`` to the glossary. + +- Update the "Environment" docs to note the existence of + ``reload_resources`` and ``reload_all``. + +- Use a colon instead of a tab as the separator between package name + and relpath to form the "spec" when register a ITemplateRenderer. + - Updated the ``bfg_alchemy`` paster template to include two views: the view on the root shows a list of links to records; the view on a record shows the details for that object. diff --git a/docs/glossary.rst b/docs/glossary.rst index 303c20703..3abe31ea6 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -17,6 +17,13 @@ Glossary `Setuptools `_ builds on Python's ``distutils`` to provide easier building, distribution, and installation of libraries and applications. + pkg_resources + A module which ships with :term:`setuptools` that provides an API + for addressing "resource files" within Python packages. Resource + files are static files, template files, etc; basically anything + non-Python-source that lives in a Python package can be considered + a resource file. See also `PkgResources + `_ Package A directory on disk which contains an ``__init__.py`` file, making it recognizable to Python as a location which can be ``import`` -ed. diff --git a/docs/narr/environment.rst b/docs/narr/environment.rst index 6df4157e2..45b544df5 100644 --- a/docs/narr/environment.rst +++ b/docs/narr/environment.rst @@ -27,6 +27,11 @@ application-specific configuration settings. | | | See also: | | | | :ref:`reload_templates_section` | +---------------------------------+-----------------------------+----------------------------------------+ +| ``BFG_RELOAD_RESOURCES`` | ``reload_resources`` | Don't cache any resource file data | +| | | when true | +| | | See also: | +| | | :ref:`overriding_resources_section` | ++---------------------------------+-----------------------------+----------------------------------------+ | ``BFG_DEBUG_AUTHORIZATION`` | ``debug_authorization`` | Print view authorization failure & | | | | success info to stderr when true | | | | See also: | @@ -39,6 +44,8 @@ application-specific configuration settings. +---------------------------------+-----------------------------+----------------------------------------+ | ``BFG_DEBUG_ALL`` | ``debug_all`` | Turns all debug_* settings on. | +---------------------------------+-----------------------------+----------------------------------------+ +| ``BFG_RELOAD_ALL`` | ``reload_all`` | Turns all reload_* settings on. | ++---------------------------------+-----------------------------+----------------------------------------+ Examples -------- @@ -66,9 +73,52 @@ application would behave in the same manner as if you had placed the respective settings in the ``[app:main]`` section of your application's ``.ini`` file. -If you want to turn all ``debug`` settings (every debug setting that -starts with ``debug_``). on in one fell swoop, you can use +If you want to turn all ``debug`` settings (every setting that starts +with ``debug_``). on in one fell swoop, you can use ``BFG_DEBUG_ALL=1`` as an environment variable setting or you may use ``debug_all=true`` in the config file. Note that this does not effect settings that do not start with ``debug_*`` such as ``reload_templates``. + +If you want to turn all ``reload`` settings (everysetting that starts +with ``reload_``). on in one fell swoop, you can use +``BFG_RELOAD_ALL=1`` as an environment variable setting or you may use +``reload_all=true`` in the config file. Note that this does not +effect settings that do not start with ``reload_*`` such as +``debug_notfound``. + +Understanding the Distinction Between ``reload_templates`` and ``reload_resources`` +----------------------------------------------------------------------------------- + +The difference between ``reload_resources`` and ``reload_templates`` +is a bit subtle. Templates are themselves also treated by +:mod:`repoze.bfg` as :term:`pkg_resources` resource files (along with +static files and other resources), so the distinction can be +confusing. It's helpful to read :ref:`overriding_resources_section` +for some context about resources in general. + +When ``reload_templates`` is true, :mod:`repoze.bfg`` takes advantage +of the underlying templating systems' ability to check for file +modifications to an individual template file. When +``reload_templates`` is true but ``reload_resources`` is *not* true, +the template filename returned by pkg_resources is cached by +:mod:`repoze.bfg` on the first request. Subsequent requests for the +same template file will return a cached template filename. The +underlying templating system checks for modifications to this +particular file for every request. Setting ``reload_templates`` to +``True`` doesn't effect performance dramatically (although it should +still not be used in production because it has some effect). + +However, when ``reload_resources`` is true, :mod:`repoze.bfg` will not +cache the template filename, meaning you can see the effect of +changing the content of an overridden resource directory for templates +without restarting the server after every change. Subsequent requests +for the same template file may return different filenames based on the +current state of overridden resource directories. Setting +``reload_resources`` to ``True`` effects performance *dramatically* +(slowing things down by an order of magnitude for each template +rendering) but it's convenient when moving files around in overridden +resource directories. ``reload_resources`` makes the system *very +slow* when templates are in use. Never set ``reload_resources`` to +``True`` on a production system. + diff --git a/docs/narr/hooks.rst b/docs/narr/hooks.rst index 85b1131bb..9eba8a92c 100644 --- a/docs/narr/hooks.rst +++ b/docs/narr/hooks.rst @@ -118,6 +118,8 @@ an object that implements any particular interface; it simply needs have a ``status`` attribute, a ``headerlist`` attribute, and and ``app_iter`` attribute. +.. _overriding_resources_section: + Overriding Resources -------------------- diff --git a/repoze/bfg/settings.py b/repoze/bfg/settings.py index df2a5e99e..08607f756 100644 --- a/repoze/bfg/settings.py +++ b/repoze/bfg/settings.py @@ -40,6 +40,9 @@ def get_options(kw, environ=os.environ): config_debug_all = kw.get('debug_all', '') effective_debug_all = asbool(eget('BFG_DEBUG_ALL', config_debug_all)) + config_reload_all = kw.get('reload_all', '') + effective_reload_all = asbool(eget('BFG_RELOAD_ALL', + config_reload_all)) config_debug_auth = kw.get('debug_authorization', '') effective_debug_auth = asbool(eget('BFG_DEBUG_AUTHORIZATION', config_debug_auth)) @@ -49,10 +52,14 @@ def get_options(kw, environ=os.environ): config_reload_templates = kw.get('reload_templates', '') effective_reload_templates = asbool(eget('BFG_RELOAD_TEMPLATES', config_reload_templates)) + config_reload_resources = kw.get('reload_resources', '') + effective_reload_resources = asbool(eget('BFG_RELOAD_RESOURCES', + config_reload_resources)) update = { 'debug_authorization': effective_debug_all or effective_debug_auth, 'debug_notfound': effective_debug_all or effective_debug_notfound, - 'reload_templates': effective_reload_templates, + 'reload_templates': effective_reload_all or effective_reload_templates, + 'reload_resources':effective_reload_all or effective_reload_resources, } kw.update(update) diff --git a/repoze/bfg/static.py b/repoze/bfg/static.py new file mode 100644 index 000000000..01ce98a30 --- /dev/null +++ b/repoze/bfg/static.py @@ -0,0 +1,78 @@ +import os +import pkg_resources + +from paste.httpheaders import ETAG +from paste.urlparser import StaticURLParser +from paste import httpexceptions +from paste import request + +class PackageURLParser(StaticURLParser): + """ This probably won't work with zipimported resources """ + def __init__(self, package_name, resource_name, root_resource=None, + cache_max_age=None): + self.package_name = package_name + self.resource_name = os.path.normpath(resource_name) + if root_resource is None: + root_resource = self.resource_name + self.root_resource = root_resource + self.cache_max_age = cache_max_age + + def __call__(self, environ, start_response): + path_info = environ.get('PATH_INFO', '') + if not path_info: + return self.add_slash(environ, start_response) + if path_info == '/': + # @@: This should obviously be configurable + filename = 'index.html' + else: + filename = request.path_info_pop(environ) + resource = os.path.normcase(os.path.normpath( + self.resource_name + '/' + filename)) + if ( (self.root_resource is not None) and + (not resource.startswith(self.root_resource)) ): + # Out of bounds + return self.not_found(environ, start_response) + if not pkg_resources.resource_exists(self.package_name, resource): + return self.not_found(environ, start_response) + if pkg_resources.resource_isdir(self.package_name, resource): + # @@: Cache? + child_root = (self.root_resource is not None and + self.root_resource or self.resource_name) + return self.__class__( + self.package_name, resource, root_resource=child_root, + cache_max_age=self.cache_max_age)(environ, start_response) + if (environ.get('PATH_INFO') + and environ.get('PATH_INFO') != '/'): # pragma: no cover + return self.error_extra_path(environ, start_response) + full = pkg_resources.resource_filename(self.package_name, resource) + if_none_match = environ.get('HTTP_IF_NONE_MATCH') + if if_none_match: + mytime = os.stat(full).st_mtime + if str(mytime) == if_none_match: + headers = [] + ETAG.update(headers, mytime) + start_response('304 Not Modified', headers) + return [''] # empty body + + fa = self.make_app(full) + if self.cache_max_age: + fa.cache_control(max_age=self.cache_max_age) + return fa(environ, start_response) + + def not_found(self, environ, start_response, debug_message=None): + comment=('SCRIPT_NAME=%r; PATH_INFO=%r; looking in package %s; ' + 'subdir %s ;debug: %s' % (environ.get('SCRIPT_NAME'), + environ.get('PATH_INFO'), + self.package_name, + self.resource_name, + debug_message or '(none)')) + exc = httpexceptions.HTTPNotFound( + 'The resource at %s could not be found' + % request.construct_url(environ), + comment=comment) + return exc.wsgi_application(environ, start_response) + + def __repr__(self): + return '<%s %s:%s at %s>' % (self.__class__.__name__, self.package_name, + self.root_resource, id(self)) + diff --git a/repoze/bfg/templating.py b/repoze/bfg/templating.py index a81726bb9..3287f0808 100644 --- a/repoze/bfg/templating.py +++ b/repoze/bfg/templating.py @@ -1,9 +1,12 @@ +import os import pkg_resources + from zope.component import queryUtility -from repoze.bfg.interfaces import ITemplateRenderer from zope.component import getSiteManager + +from repoze.bfg.interfaces import ITemplateRenderer from repoze.bfg.path import caller_package -import os +from repoze.bfg.settings import get_settings def renderer_from_cache(path, factory, level=3, **kw): if os.path.isabs(path): @@ -21,7 +24,7 @@ def renderer_from_cache(path, factory, level=3, **kw): # 'path' is a relative filename package = caller_package(level=level) spec = (package.__name__, path) - utility_name = '%s\t%s' % spec # utility name must be a string :-( + utility_name = '%s:%s' % spec # utility name must be a string renderer = queryUtility(ITemplateRenderer, name=utility_name) if renderer is None: # service unit tests here by trying the relative path @@ -29,11 +32,15 @@ def renderer_from_cache(path, factory, level=3, **kw): renderer = queryUtility(ITemplateRenderer, name=path) if renderer is None: if not pkg_resources.resource_exists(*spec): - raise ValueError('Missing template resource: %s:%s' % spec) + raise ValueError('Missing template resource: %s' % utility_name) abspath = pkg_resources.resource_filename(*spec) renderer = factory(abspath, **kw) - sm = getSiteManager() - sm.registerUtility(renderer, ITemplateRenderer, name=utility_name) + settings = get_settings() + if (not settings) or (not settings.get('reload_resources')): + # cache the template + sm = getSiteManager() + sm.registerUtility(renderer, ITemplateRenderer, + name=utility_name) return renderer diff --git a/repoze/bfg/tests/fixtures/static/index.html b/repoze/bfg/tests/fixtures/static/index.html new file mode 100644 index 000000000..6498787a5 --- /dev/null +++ b/repoze/bfg/tests/fixtures/static/index.html @@ -0,0 +1 @@ +static diff --git a/repoze/bfg/tests/fixtures/static/subdir/index.html b/repoze/bfg/tests/fixtures/static/subdir/index.html new file mode 100644 index 000000000..bb84fad04 --- /dev/null +++ b/repoze/bfg/tests/fixtures/static/subdir/index.html @@ -0,0 +1 @@ +subdir diff --git a/repoze/bfg/tests/test_settings.py b/repoze/bfg/tests/test_settings.py index 903a23f5d..8319a302d 100644 --- a/repoze/bfg/tests/test_settings.py +++ b/repoze/bfg/tests/test_settings.py @@ -61,6 +61,42 @@ class TestGetOptions(unittest.TestCase): {'BFG_RELOAD_TEMPLATES':'1'}) self.assertEqual(result['reload_templates'], True) + def test_reload_resources(self): + result = self._callFUT({}) + self.assertEqual(result['reload_resources'], False) + result = self._callFUT({'reload_resources':'false'}) + self.assertEqual(result['reload_resources'], False) + result = self._callFUT({'reload_resources':'t'}) + self.assertEqual(result['reload_resources'], True) + result = self._callFUT({'reload_resources':'1'}) + self.assertEqual(result['reload_resources'], True) + result = self._callFUT({}, {'BFG_RELOAD_RESOURCES':'1'}) + self.assertEqual(result['reload_resources'], True) + result = self._callFUT({'reload_resources':'false'}, + {'BFG_RELOAD_RESOURCES':'1'}) + self.assertEqual(result['reload_resources'], True) + + def test_reload_all(self): + result = self._callFUT({}) + self.assertEqual(result['reload_templates'], False) + self.assertEqual(result['reload_resources'], False) + result = self._callFUT({'reload_all':'false'}) + self.assertEqual(result['reload_templates'], False) + self.assertEqual(result['reload_resources'], False) + result = self._callFUT({'reload_all':'t'}) + self.assertEqual(result['reload_templates'], True) + self.assertEqual(result['reload_resources'], True) + result = self._callFUT({'reload_all':'1'}) + self.assertEqual(result['reload_templates'], True) + self.assertEqual(result['reload_resources'], True) + result = self._callFUT({}, {'BFG_RELOAD_ALL':'1'}) + self.assertEqual(result['reload_templates'], True) + self.assertEqual(result['reload_resources'], True) + result = self._callFUT({'reload_all':'false'}, + {'BFG_RELOAD_ALL':'1'}) + self.assertEqual(result['reload_templates'], True) + self.assertEqual(result['reload_resources'], True) + def test_debug_authorization(self): result = self._callFUT({}) self.assertEqual(result['debug_authorization'], False) diff --git a/repoze/bfg/tests/test_static.py b/repoze/bfg/tests/test_static.py new file mode 100644 index 000000000..7c7b5627c --- /dev/null +++ b/repoze/bfg/tests/test_static.py @@ -0,0 +1,157 @@ +import unittest + +class TestPackageURLParser(unittest.TestCase): + def _getTargetClass(self): + from repoze.bfg.static import PackageURLParser + return PackageURLParser + + def _makeOne(self, *arg, **kw): + return self._getTargetClass()(*arg, **kw) + + + def _makeEnviron(self, **kw): + environ = { + 'wsgi.url_scheme':'http', + 'wsgi.version':(1,0), + 'SERVER_NAME':'example.com', + 'SERVER_PORT':'6543', + 'PATH_INFO':'/', + 'SCRIPT_NAME':'', + 'REQUEST_METHOD':'GET', + } + environ.update(kw) + return environ + + def test_ctor_allargs(self): + inst = self._makeOne('package', 'resource/name', root_resource='root', + cache_max_age=100) + self.assertEqual(inst.package_name, 'package') + self.assertEqual(inst.resource_name, 'resource/name') + self.assertEqual(inst.root_resource, 'root') + self.assertEqual(inst.cache_max_age, 100) + + def test_ctor_defaultargs(self): + inst = self._makeOne('package', 'resource/name') + self.assertEqual(inst.package_name, 'package') + self.assertEqual(inst.resource_name, 'resource/name') + self.assertEqual(inst.root_resource, 'resource/name') + self.assertEqual(inst.cache_max_age, None) + + def test_call_adds_slash_path_info_empty(self): + environ = self._makeEnviron(PATH_INFO='') + inst = self._makeOne('repoze.bfg.tests', 'fixtures/static') + sr = DummyStartResponse() + response = inst(environ, sr) + body = response[0] + self.failUnless('301 Moved Permanently' in body) + self.failUnless('http://example.com:6543/' in body) + + def test_path_info_slash_means_index_html(self): + environ = self._makeEnviron() + inst = self._makeOne('repoze.bfg.tests', 'fixtures/static') + sr = DummyStartResponse() + response = inst(environ, sr) + body = response[0] + self.failUnless('static' in body) + + def test_resource_out_of_bounds(self): + environ = self._makeEnviron() + inst = self._makeOne('repoze.bfg.tests', 'fixtures/static') + inst.root_resource = 'abcdef' + sr = DummyStartResponse() + response = inst(environ, sr) + body = response[0] + self.failUnless('404 Not Found' in body) + self.failUnless('http://example.com:6543/' in body) + + def test_resource_doesnt_exist(self): + environ = self._makeEnviron(PATH_INFO='/notthere') + inst = self._makeOne('repoze.bfg.tests', 'fixtures/static') + sr = DummyStartResponse() + response = inst(environ, sr) + body = response[0] + self.failUnless('404 Not Found' in body) + self.failUnless('http://example.com:6543/' in body) + + def test_resource_isdir(self): + environ = self._makeEnviron(PATH_INFO='/subdir/') + inst = self._makeOne('repoze.bfg.tests', 'fixtures/static') + sr = DummyStartResponse() + response = inst(environ, sr) + body = response[0] + self.failUnless('subdir' in body) + + def test_resource_is_file(self): + environ = self._makeEnviron(PATH_INFO='/index.html') + inst = self._makeOne('repoze.bfg.tests', 'fixtures/static') + sr = DummyStartResponse() + response = inst(environ, sr) + body = response[0] + self.failUnless('static' in body) + + def test_resource_is_file_with_cache_max_age(self): + environ = self._makeEnviron(PATH_INFO='/index.html') + inst = self._makeOne('repoze.bfg.tests', 'fixtures/static', + cache_max_age=600) + sr = DummyStartResponse() + response = inst(environ, sr) + body = response[0] + self.failUnless('static' in body) + self.assertEqual(len(sr.headerlist), 8) + header_names = [ x[0] for x in sr.headerlist ] + header_names.sort() + self.assertEqual(header_names, + ['Accept-Ranges', 'Cache-Control', + 'Content-Length', 'Content-Range', + 'Content-Type', 'ETag', 'Expires', 'Last-Modified']) + + def test_resource_is_file_with_no_cache_max_age(self): + environ = self._makeEnviron(PATH_INFO='/index.html') + inst = self._makeOne('repoze.bfg.tests', 'fixtures/static') + sr = DummyStartResponse() + response = inst(environ, sr) + body = response[0] + self.failUnless('static' in body) + self.assertEqual(len(sr.headerlist), 6) + header_names = [ x[0] for x in sr.headerlist ] + header_names.sort() + self.assertEqual(header_names, + ['Accept-Ranges', 'Content-Length', 'Content-Range', + 'Content-Type', 'ETag', 'Last-Modified']) + + def test_if_none_match(self): + class DummyEq(object): + def __eq__(self, other): + return True + dummy_eq = DummyEq() + environ = self._makeEnviron(HTTP_IF_NONE_MATCH=dummy_eq) + inst = self._makeOne('repoze.bfg.tests', 'fixtures/static') + sr = DummyStartResponse() + response = inst(environ, sr) + self.assertEqual(len(sr.headerlist), 1) + self.assertEqual(sr.status, '304 Not Modified') + self.assertEqual(sr.headerlist[0][0], 'ETag') + self.assertEqual(response[0], '') + + def test_repr(self): + inst = self._makeOne('repoze.bfg.tests', 'fixtures/static') + self.failUnless( + repr(inst).startswith( + '