From 434c0514dc7dd9c7881e1d30db4d988680220af2 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Sat, 23 Aug 2008 00:03:29 +0000 Subject: - Read and write a pickled ZCML actions list, stored as ``configure.zcml.pck`` next to the applications's "normal" configuration file. A given bfg app will usually start faster if it's able to read the pickle data. It fails gracefully to reading the real ZCML file if it cannot read the pickle. --- CHANGES.txt | 8 ++ repoze/bfg/path.py | 15 +++ repoze/bfg/registry.py | 5 +- repoze/bfg/template.py | 14 +-- repoze/bfg/tests/fixtureapp/another.zcml | 12 ++ repoze/bfg/tests/fixtureapp/configure.zcml | 3 + repoze/bfg/tests/test_zcml.py | 183 +++++++++++++++++++++++++++++ repoze/bfg/zcml.py | 85 ++++++++++++++ 8 files changed, 310 insertions(+), 15 deletions(-) create mode 100644 repoze/bfg/path.py create mode 100644 repoze/bfg/tests/fixtureapp/another.zcml diff --git a/CHANGES.txt b/CHANGES.txt index 50ba6cb6c..7a04efa08 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,11 @@ +Next release + + - Read and write a pickled ZCML actions list, stored as + ``configure.zcml.pck`` next to the applications's "normal" + configuration file. A given bfg app will usually start faster + if it's able to read the pickle data. It fails gracefully + to reading the real ZCML file if it cannot read the pickle. + 0.3.1 (8/20/2008) - Generated application differences: ``make_app`` entry point diff --git a/repoze/bfg/path.py b/repoze/bfg/path.py new file mode 100644 index 000000000..fcd317fba --- /dev/null +++ b/repoze/bfg/path.py @@ -0,0 +1,15 @@ +import os +import sys + +def caller_path(path, level=2): + if not os.path.isabs(path): + package_globals = sys._getframe(level).f_globals + package_name = package_globals['__name__'] + package = sys.modules[package_name] + prefix = package_path(package) + path = os.path.join(prefix, path) + return path + +def package_path(package): + return os.path.abspath(os.path.dirname(package.__file__)) + diff --git a/repoze/bfg/registry.py b/repoze/bfg/registry.py index e02e5124e..9301b950b 100644 --- a/repoze/bfg/registry.py +++ b/repoze/bfg/registry.py @@ -1,4 +1,5 @@ import threading + import zope.component from zope.component import getGlobalSiteManager @@ -6,11 +7,11 @@ from zope.component.interfaces import ComponentLookupError from zope.component.interfaces import IComponentLookup from zope.component.registry import Components from zope.component import getSiteManager as original_getSiteManager -from zope.configuration import xmlconfig from zope.interface import implements from repoze.bfg.interfaces import ISettings +from repoze.bfg.zcml import zcml_configure class ThreadLocalRegistryManager(threading.local): registry = getGlobalSiteManager() @@ -52,7 +53,7 @@ def makeRegistry(filename, package, options=None, lock=threading.Lock()): registry_manager.set(registry) original_getSiteManager.sethook(getSiteManager) zope.component.getGlobalSiteManager = registry_manager.get - xmlconfig.file(filename, package=package) + zcml_configure(filename, package=package) if options is None: options = {} settings = Settings(options) diff --git a/repoze/bfg/template.py b/repoze/bfg/template.py index de049f5fa..05331aad9 100644 --- a/repoze/bfg/template.py +++ b/repoze/bfg/template.py @@ -1,5 +1,4 @@ import os -import sys from webob import Response @@ -10,6 +9,7 @@ from zope.component import getSiteManager from zope.interface import classProvides from zope.interface import implements +from repoze.bfg.path import caller_path from repoze.bfg.interfaces import ITemplateFactory from repoze.bfg.interfaces import ITemplate from repoze.bfg.interfaces import INodeTemplate @@ -63,9 +63,6 @@ def get_processor(xslt_fn, auto_reload=False): xslt_pool.processors[xslt_fn] = proc return proc -def package_path(package): - return os.path.abspath(os.path.dirname(package.__file__)) - def registerTemplate(type, template, path): try: sm = getSiteManager() @@ -134,13 +131,4 @@ def render_transform_to_response(path, node, **kw): result = render_transform(path, node, **kw) return Response(result) -def caller_path(path): - if not os.path.isabs(path): - package_globals = sys._getframe(2).f_globals - package_name = package_globals['__name__'] - package = sys.modules[package_name] - prefix = package_path(package) - path = os.path.join(prefix, path) - return path - diff --git a/repoze/bfg/tests/fixtureapp/another.zcml b/repoze/bfg/tests/fixtureapp/another.zcml new file mode 100644 index 000000000..f29e7e59e --- /dev/null +++ b/repoze/bfg/tests/fixtureapp/another.zcml @@ -0,0 +1,12 @@ + + + + + diff --git a/repoze/bfg/tests/fixtureapp/configure.zcml b/repoze/bfg/tests/fixtureapp/configure.zcml index dfedda7bb..a56c1cbdf 100644 --- a/repoze/bfg/tests/fixtureapp/configure.zcml +++ b/repoze/bfg/tests/fixtureapp/configure.zcml @@ -18,4 +18,7 @@ request_type=".views.IDummy" /> + + + diff --git a/repoze/bfg/tests/test_zcml.py b/repoze/bfg/tests/test_zcml.py index 663022796..5fa5d6290 100644 --- a/repoze/bfg/tests/test_zcml.py +++ b/repoze/bfg/tests/test_zcml.py @@ -136,6 +136,189 @@ class TestSampleApp(unittest.TestCase, PlacelessSetup): new = cPickle.loads(dumped) self.assertEqual(len(actions), len(new)) +class TestZCMLPickling(unittest.TestCase, PlacelessSetup): + i = 0 + def setUp(self): + self.tempdir = None + PlacelessSetup.setUp(self) + import sys + import os + import tempfile + from repoze.bfg.path import package_path + from repoze.bfg.tests import fixtureapp as package + import shutil + tempdir = tempfile.mkdtemp() + modname = 'myfixture%s' % self.i + self.i += 1 + self.packagepath = os.path.join(tempdir, modname) + fixturedir = package_path(package) + pckname = os.path.join(fixturedir, 'configure.zcml.pck') + if os.path.isfile(pckname): + os.remove(pckname) + shutil.copytree(fixturedir, self.packagepath) + sys.path.insert(0, tempdir) + self.module = __import__(modname) + self.tempdir = tempdir + + def tearDown(self): + PlacelessSetup.tearDown(self) + import sys + import shutil + if self.module is not None: + del sys.modules[self.module.__name__] + if self.tempdir is not None: + sys.path.pop(0) + shutil.rmtree(self.tempdir) + + def test_file_configure(self): + import os + import cPickle + from repoze.bfg.zcml import file_configure + self.assertEqual(False, file_configure('configure.zcml', self.module)) + picklename = os.path.join(self.packagepath, 'configure.zcml.pck') + self.failUnless(os.path.exists(picklename)) + actions = cPickle.load(open(picklename, 'rb')) + self.failUnless(actions) + + def test_file_configure_nonexistent_configure_dot_zcml(self): + import os + from repoze.bfg.zcml import file_configure + os.remove(os.path.join(self.packagepath, 'configure.zcml')) + self.assertRaises(IOError, file_configure, 'configure.zcml', + self.module) + + def test_file_configure_pickling_error(self): + import os + from repoze.bfg.zcml import file_configure + def dumpfail(actions, f): + raise IOError + self.assertEqual(False, + file_configure('configure.zcml', self.module, dumpfail)) + picklename = os.path.join(self.packagepath, 'configure.zcml.pck') + self.failIf(os.path.exists(picklename)) + + def test_zcml_configure_writes_pickle_when_none_exists(self): + import os + import cPickle + from repoze.bfg.zcml import zcml_configure + self.assertEqual(False, zcml_configure('configure.zcml', self.module)) + picklename = os.path.join(self.packagepath, 'configure.zcml.pck') + self.failUnless(os.path.exists(picklename)) + actions = cPickle.load(open(picklename, 'rb')) + self.failUnless(actions) + + def test_zcml_configure_uses_file_configure_with_bad_pickle1(self): + import os + import cPickle + from repoze.bfg.zcml import zcml_configure + picklename = os.path.join(self.packagepath, 'configure.zcml.pck') + f = open(picklename, 'wb') + cPickle.dump((), f) + f.close() + self.assertEqual(False, zcml_configure('configure.zcml', self.module)) + + def test_zcml_configure_uses_file_configure_with_bad_pickle2(self): + import os + from repoze.bfg.zcml import zcml_configure + picklename = os.path.join(self.packagepath, 'configure.zcml.pck') + f = open(picklename, 'wb') + f.write('garbage') + f.close() + self.assertEqual(False, zcml_configure('configure.zcml', self.module)) + + def test_zcml_configure_uses_file_configure_with_outofdate_pickle1(self): + import os + import cPickle + import time + from repoze.bfg.zcml import zcml_configure + basename = os.path.join(self.packagepath, 'configure.zcml') + picklename = os.path.join(self.packagepath, 'configure.zcml.pck') + self.assertEqual(False, zcml_configure('configure.zcml', self.module)) + self.failUnless(os.path.exists(picklename)) + actions = cPickle.load(open(picklename, 'rb')) + self.failUnless(actions) + os.utime(basename, (-1, time.time() + 100)) + self.assertEqual(False, zcml_configure('configure.zcml', self.module)) + + def test_zcml_configure_uses_file_configure_with_outofdate_pickle2(self): + import os + import cPickle + import time + from repoze.bfg.zcml import zcml_configure + basename = os.path.join(self.packagepath, 'another.zcml') + picklename = os.path.join(self.packagepath, 'configure.zcml.pck') + self.assertEqual(False, zcml_configure('configure.zcml', self.module)) + self.failUnless(os.path.exists(picklename)) + actions = cPickle.load(open(picklename, 'rb')) + self.failUnless(actions) + os.utime(basename, (-1, time.time() + 100)) + self.assertEqual(False, zcml_configure('configure.zcml', self.module)) + + def test_zcml_configure_uses_file_configure_with_missing_dependent(self): + import os + import cPickle + from repoze.bfg.zcml import zcml_configure + from zope.configuration.xmlconfig import ZopeXMLConfigurationError + basename = os.path.join(self.packagepath, 'another.zcml') + picklename = os.path.join(self.packagepath, 'configure.zcml.pck') + self.assertEqual(False, zcml_configure('configure.zcml', self.module)) + self.failUnless(os.path.exists(picklename)) + actions = cPickle.load(open(picklename, 'rb')) + self.failUnless(actions) + os.remove(basename) + self.assertRaises(ZopeXMLConfigurationError, zcml_configure, + 'configure.zcml', self.module) + + def test_zcml_configure_uses_file_configure_with_bad_version(self): + import os + from repoze.bfg.zcml import zcml_configure + from repoze.bfg.zcml import PVERSION + picklename = os.path.join(self.packagepath, 'configure.zcml.pck') + f = open(picklename, 'wb') + import cPickle + data = (PVERSION+1, 0, []) + cPickle.dump(data, open(picklename, 'wb')) + self.assertEqual(False, zcml_configure('configure.zcml', self.module)) + + def test_zcml_configure_uses_file_configure_with_bad_time(self): + import os + from repoze.bfg.zcml import zcml_configure + from repoze.bfg.zcml import PVERSION + picklename = os.path.join(self.packagepath, 'configure.zcml.pck') + f = open(picklename, 'wb') + import cPickle + data = (PVERSION, None, []) + cPickle.dump(data, open(picklename, 'wb')) + self.assertEqual(False, zcml_configure('configure.zcml', self.module)) + + def test_zcml_configure_uses_file_configure_with_bad_actions(self): + import os + from repoze.bfg.zcml import zcml_configure + from repoze.bfg.zcml import PVERSION + import time + picklename = os.path.join(self.packagepath, 'configure.zcml.pck') + f = open(picklename, 'wb') + import cPickle + data = (PVERSION, time.time()+500, None) + cPickle.dump(data, open(picklename, 'wb')) + self.assertEqual(False, zcml_configure('configure.zcml', self.module)) + + def test_zcml_configure_uses_good_pickle(self): + import os + import cPickle + import time + from repoze.bfg.zcml import zcml_configure + from repoze.bfg.zcml import PVERSION + basename = os.path.join(self.packagepath, 'another.zcml') + picklename = os.path.join(self.packagepath, 'configure.zcml.pck') + self.assertEqual(False, zcml_configure('configure.zcml', self.module)) + self.failUnless(os.path.exists(picklename)) + actions = cPickle.load(open(picklename, 'rb')) + self.failUnless(actions) + actions = (PVERSION, time.time()+100, actions[2]) + cPickle.dump(actions, open(picklename, 'wb')) + self.assertEqual(True, zcml_configure('configure.zcml', self.module)) + class Dummy: pass diff --git a/repoze/bfg/zcml.py b/repoze/bfg/zcml.py index 343abc1d7..0bb049516 100644 --- a/repoze/bfg/zcml.py +++ b/repoze/bfg/zcml.py @@ -1,3 +1,11 @@ +import cPickle +import os +from os.path import realpath +import time + +from zope.configuration import xmlconfig +import zope.configuration.config + from zope.component.zcml import handler from zope.component.interface import provideInterface from zope.configuration.exceptions import ConfigurationError @@ -10,6 +18,7 @@ from zope.schema import TextLine from repoze.bfg.interfaces import IRequest from repoze.bfg.interfaces import IViewPermission from repoze.bfg.interfaces import IView +from repoze.bfg.path import package_path from repoze.bfg.security import ViewPermissionFactory @@ -84,4 +93,80 @@ class IViewDirective(Interface): required=False ) +PVERSION = 0 + +def pickle_name(name, package): + path = package_path(package) + basename = os.path.join(path, name) + return os.path.join(path, basename + '.pck') + +def zcml_configure(name, package, load=cPickle.load): + """ Execute pickled zcml actions or fall back to parsing from file + """ + pckname = pickle_name(name, package) + + if not (os.path.isfile(pckname) or os.path.islink(pckname)): + return file_configure(name, package) + + try: + vers, ptime, actions = load(open(pckname, 'rb')) + except (IOError, cPickle.UnpicklingError, EOFError, TypeError, ValueError): + return file_configure(name, package) + + if vers != PVERSION: + return file_configure(name, package) + + try: + ptime = int(ptime) + except: + return file_configure(name, package) + + if not hasattr(actions, '__iter__'): + return file_configure(name, package) + + files = set() + for action in actions: + # files list used by pickled action is an element of the tuple + try: + files.update(action[4]) + except (TypeError, IndexError): + return file_configure(name, package) + + for file in files: + if not(os.path.isfile(file) or os.path.islink(file)): + return file_configure(name, package) + + mtime = os.stat(realpath(file)).st_mtime + + if mtime >= ptime: + return file_configure(name, package) + + context = zope.configuration.config.ConfigurationMachine() + xmlconfig.registerCommonDirectives(context) + context.actions = actions + context.execute_actions() + return True + +def file_configure(name, package, dump=cPickle.dump): + context = zope.configuration.config.ConfigurationMachine() + xmlconfig.registerCommonDirectives(context) + context.package = package + + xmlconfig.include(context, name, package) + context.execute_actions(clear=False) + + actions = context.actions + pckname = pickle_name(name, package) + + try: + data = (PVERSION, time.time(), actions) + dump(data, open(pckname, 'wb'), -1) + except (OSError, IOError, TypeError, cPickle.PickleError): + try: + os.remove(pckname) + except: + pass + + return False + -- cgit v1.2.3