- Read and write a pickled ZCML actions list, stored as

    ``configure.zcml.pck`` next to the applications's "normal" 
    configuration file.  A given bfg app will usually start faster
    if it's able to read the pickle data.  It fails gracefully
    to reading the real ZCML file if it cannot read the pickle.

8 files changed, 310 insertions, 15 deletions

diff --git a/CHANGES.txt b/CHANGES.txt
index 50ba6cb6c..7a04efa08 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,11 @@
+Next release
+
+  - Read and write a pickled ZCML actions list, stored as
+    ``configure.zcml.pck`` next to the applications's "normal" 
+    configuration file.  A given bfg app will usually start faster
+    if it's able to read the pickle data.  It fails gracefully
+    to reading the real ZCML file if it cannot read the pickle.
+
 0.3.1 (8/20/2008)
 
   - Generated application differences: ``make_app`` entry point
diff --git a/repoze/bfg/path.py b/repoze/bfg/path.py
new file mode 100644
index 000000000..fcd317fba
--- /dev/null
+++ b/repoze/bfg/path.py
@@ -0,0 +1,15 @@
+import os
+import sys
+
+def caller_path(path, level=2):
+    if not os.path.isabs(path):
+        package_globals = sys._getframe(level).f_globals
+        package_name = package_globals['__name__']
+        package = sys.modules[package_name]
+        prefix = package_path(package)
+        path = os.path.join(prefix, path)
+    return path
+
+def package_path(package):
+    return os.path.abspath(os.path.dirname(package.__file__))
+
diff --git a/repoze/bfg/registry.py b/repoze/bfg/registry.py
index e02e5124e..9301b950b 100644
--- a/repoze/bfg/registry.py
+++ b/repoze/bfg/registry.py
@@ -1,4 +1,5 @@
 import threading
+
 import zope.component
 
 from zope.component import getGlobalSiteManager
@@ -6,11 +7,11 @@ from zope.component.interfaces import ComponentLookupError
 from zope.component.interfaces import IComponentLookup
 from zope.component.registry import Components
 from zope.component import getSiteManager as original_getSiteManager
-from zope.configuration import xmlconfig
 
 from zope.interface import implements
 
 from repoze.bfg.interfaces import ISettings
+from repoze.bfg.zcml import zcml_configure
 
 class ThreadLocalRegistryManager(threading.local):
     registry = getGlobalSiteManager()
@@ -52,7 +53,7 @@ def makeRegistry(filename, package, options=None, lock=threading.Lock()):
         registry_manager.set(registry)
         original_getSiteManager.sethook(getSiteManager)
         zope.component.getGlobalSiteManager = registry_manager.get
-        xmlconfig.file(filename, package=package)
+        zcml_configure(filename, package=package)
         if options is None:
             options = {}
         settings = Settings(options)
diff --git a/repoze/bfg/template.py b/repoze/bfg/template.py
index de049f5fa..05331aad9 100644
--- a/repoze/bfg/template.py
+++ b/repoze/bfg/template.py
@@ -1,5 +1,4 @@
 import os
-import sys
 
 from webob import Response
 
@@ -10,6 +9,7 @@ from zope.component import getSiteManager
 from zope.interface import classProvides
 from zope.interface import implements
 
+from repoze.bfg.path import caller_path
 from repoze.bfg.interfaces import ITemplateFactory
 from repoze.bfg.interfaces import ITemplate
 from repoze.bfg.interfaces import INodeTemplate
@@ -63,9 +63,6 @@ def get_processor(xslt_fn, auto_reload=False):
     xslt_pool.processors[xslt_fn] = proc
     return proc
 
-def package_path(package):
-    return os.path.abspath(os.path.dirname(package.__file__))
-
 def registerTemplate(type, template, path):
     try:
         sm = getSiteManager()
@@ -134,13 +131,4 @@ def render_transform_to_response(path, node, **kw):
     result = render_transform(path, node, **kw)
     return Response(result)
 
-def caller_path(path):
-    if not os.path.isabs(path):
-        package_globals = sys._getframe(2).f_globals
-        package_name = package_globals['__name__']
-        package = sys.modules[package_name]
-        prefix = package_path(package)
-        path = os.path.join(prefix, path)
-    return path
-
     
diff --git a/repoze/bfg/tests/fixtureapp/another.zcml b/repoze/bfg/tests/fixtureapp/another.zcml
new file mode 100644
index 000000000..f29e7e59e
--- /dev/null
+++ b/repoze/bfg/tests/fixtureapp/another.zcml
@@ -0,0 +1,12 @@
+<configure xmlns="http://namespaces.zope.org/zope"
+    xmlns:bfg="http://namespaces.repoze.org/bfg"
+	           i18n_domain="repoze.bfg">
+
+  <bfg:view
+      view=".views.fixture_view"
+      for="*"
+      name="another.html"
+      permission="repoze.view"
+      />
+
+</configure>
diff --git a/repoze/bfg/tests/fixtureapp/configure.zcml b/repoze/bfg/tests/fixtureapp/configure.zcml
index dfedda7bb..a56c1cbdf 100644
--- a/repoze/bfg/tests/fixtureapp/configure.zcml
+++ b/repoze/bfg/tests/fixtureapp/configure.zcml
@@ -18,4 +18,7 @@
       request_type=".views.IDummy"
       />
 
+  <include file="another.zcml"/>
+
+
 </configure>
diff --git a/repoze/bfg/tests/test_zcml.py b/repoze/bfg/tests/test_zcml.py
index 663022796..5fa5d6290 100644
--- a/repoze/bfg/tests/test_zcml.py
+++ b/repoze/bfg/tests/test_zcml.py
@@ -136,6 +136,189 @@ class TestSampleApp(unittest.TestCase, PlacelessSetup):
         new = cPickle.loads(dumped)
         self.assertEqual(len(actions), len(new))
 
+class TestZCMLPickling(unittest.TestCase, PlacelessSetup):
+    i = 0
+    def setUp(self):
+        self.tempdir = None
+        PlacelessSetup.setUp(self)
+        import sys
+        import os
+        import tempfile
+        from repoze.bfg.path import package_path
+        from repoze.bfg.tests import fixtureapp as package
+        import shutil
+        tempdir = tempfile.mkdtemp()
+        modname = 'myfixture%s' % self.i
+        self.i += 1
+        self.packagepath = os.path.join(tempdir, modname)
+        fixturedir = package_path(package)
+        pckname = os.path.join(fixturedir, 'configure.zcml.pck')
+        if os.path.isfile(pckname):
+            os.remove(pckname)
+        shutil.copytree(fixturedir, self.packagepath)
+        sys.path.insert(0, tempdir)
+        self.module = __import__(modname)
+        self.tempdir = tempdir
+
+    def tearDown(self):
+        PlacelessSetup.tearDown(self)
+        import sys
+        import shutil
+        if self.module is not None:
+            del sys.modules[self.module.__name__]
+        if self.tempdir is not None:
+            sys.path.pop(0)
+            shutil.rmtree(self.tempdir)
+
+    def test_file_configure(self):
+        import os
+        import cPickle
+        from repoze.bfg.zcml import file_configure
+        self.assertEqual(False, file_configure('configure.zcml', self.module))
+        picklename = os.path.join(self.packagepath, 'configure.zcml.pck')
+        self.failUnless(os.path.exists(picklename))
+        actions = cPickle.load(open(picklename, 'rb'))
+        self.failUnless(actions)
+
+    def test_file_configure_nonexistent_configure_dot_zcml(self):
+        import os
+        from repoze.bfg.zcml import file_configure
+        os.remove(os.path.join(self.packagepath, 'configure.zcml'))
+        self.assertRaises(IOError, file_configure, 'configure.zcml',
+                          self.module)
+
+    def test_file_configure_pickling_error(self):
+        import os
+        from repoze.bfg.zcml import file_configure
+        def dumpfail(actions, f):
+            raise IOError
+        self.assertEqual(False,
+                      file_configure('configure.zcml', self.module, dumpfail))
+        picklename = os.path.join(self.packagepath, 'configure.zcml.pck')
+        self.failIf(os.path.exists(picklename))
+
+    def test_zcml_configure_writes_pickle_when_none_exists(self):
+        import os
+        import cPickle
+        from repoze.bfg.zcml import zcml_configure
+        self.assertEqual(False, zcml_configure('configure.zcml', self.module))
+        picklename = os.path.join(self.packagepath, 'configure.zcml.pck')
+        self.failUnless(os.path.exists(picklename))
+        actions = cPickle.load(open(picklename, 'rb'))
+        self.failUnless(actions)
+
+    def test_zcml_configure_uses_file_configure_with_bad_pickle1(self):
+        import os
+        import cPickle
+        from repoze.bfg.zcml import zcml_configure
+        picklename = os.path.join(self.packagepath, 'configure.zcml.pck')
+        f = open(picklename, 'wb')
+        cPickle.dump((), f)
+        f.close()
+        self.assertEqual(False, zcml_configure('configure.zcml', self.module))
+
+    def test_zcml_configure_uses_file_configure_with_bad_pickle2(self):
+        import os
+        from repoze.bfg.zcml import zcml_configure
+        picklename = os.path.join(self.packagepath, 'configure.zcml.pck')
+        f = open(picklename, 'wb')
+        f.write('garbage')
+        f.close()
+        self.assertEqual(False, zcml_configure('configure.zcml', self.module))
+
+    def test_zcml_configure_uses_file_configure_with_outofdate_pickle1(self):
+        import os
+        import cPickle
+        import time
+        from repoze.bfg.zcml import zcml_configure
+        basename = os.path.join(self.packagepath, 'configure.zcml')
+        picklename = os.path.join(self.packagepath, 'configure.zcml.pck')
+        self.assertEqual(False, zcml_configure('configure.zcml', self.module))
+        self.failUnless(os.path.exists(picklename))
+        actions = cPickle.load(open(picklename, 'rb'))
+        self.failUnless(actions)
+        os.utime(basename, (-1, time.time() + 100))
+        self.assertEqual(False, zcml_configure('configure.zcml', self.module))
+
+    def test_zcml_configure_uses_file_configure_with_outofdate_pickle2(self):
+        import os
+        import cPickle
+        import time
+        from repoze.bfg.zcml import zcml_configure
+        basename = os.path.join(self.packagepath, 'another.zcml')
+        picklename = os.path.join(self.packagepath, 'configure.zcml.pck')
+        self.assertEqual(False, zcml_configure('configure.zcml', self.module))
+        self.failUnless(os.path.exists(picklename))
+        actions = cPickle.load(open(picklename, 'rb'))
+        self.failUnless(actions)
+        os.utime(basename, (-1, time.time() + 100))
+        self.assertEqual(False, zcml_configure('configure.zcml', self.module))
+
+    def test_zcml_configure_uses_file_configure_with_missing_dependent(self):
+        import os
+        import cPickle
+        from repoze.bfg.zcml import zcml_configure
+        from zope.configuration.xmlconfig import ZopeXMLConfigurationError
+        basename = os.path.join(self.packagepath, 'another.zcml')
+        picklename = os.path.join(self.packagepath, 'configure.zcml.pck')
+        self.assertEqual(False, zcml_configure('configure.zcml', self.module))
+        self.failUnless(os.path.exists(picklename))
+        actions = cPickle.load(open(picklename, 'rb'))
+        self.failUnless(actions)
+        os.remove(basename)
+        self.assertRaises(ZopeXMLConfigurationError, zcml_configure,
+                          'configure.zcml', self.module)
+
+    def test_zcml_configure_uses_file_configure_with_bad_version(self):
+        import os
+        from repoze.bfg.zcml import zcml_configure
+        from repoze.bfg.zcml import PVERSION
+        picklename = os.path.join(self.packagepath, 'configure.zcml.pck')
+        f = open(picklename, 'wb')
+        import cPickle
+        data = (PVERSION+1, 0, [])
+        cPickle.dump(data, open(picklename, 'wb'))
+        self.assertEqual(False, zcml_configure('configure.zcml', self.module))
+
+    def test_zcml_configure_uses_file_configure_with_bad_time(self):
+        import os
+        from repoze.bfg.zcml import zcml_configure
+        from repoze.bfg.zcml import PVERSION
+        picklename = os.path.join(self.packagepath, 'configure.zcml.pck')
+        f = open(picklename, 'wb')
+        import cPickle
+        data = (PVERSION, None, [])
+        cPickle.dump(data, open(picklename, 'wb'))
+        self.assertEqual(False, zcml_configure('configure.zcml', self.module))
+
+    def test_zcml_configure_uses_file_configure_with_bad_actions(self):
+        import os
+        from repoze.bfg.zcml import zcml_configure
+        from repoze.bfg.zcml import PVERSION
+        import time
+        picklename = os.path.join(self.packagepath, 'configure.zcml.pck')
+        f = open(picklename, 'wb')
+        import cPickle
+        data = (PVERSION, time.time()+500, None)
+        cPickle.dump(data, open(picklename, 'wb'))
+        self.assertEqual(False, zcml_configure('configure.zcml', self.module))
+
+    def test_zcml_configure_uses_good_pickle(self):
+        import os
+        import cPickle
+        import time
+        from repoze.bfg.zcml import zcml_configure
+        from repoze.bfg.zcml import PVERSION
+        basename = os.path.join(self.packagepath, 'another.zcml')
+        picklename = os.path.join(self.packagepath, 'configure.zcml.pck')
+        self.assertEqual(False, zcml_configure('configure.zcml', self.module))
+        self.failUnless(os.path.exists(picklename))
+        actions = cPickle.load(open(picklename, 'rb'))
+        self.failUnless(actions)
+        actions = (PVERSION, time.time()+100, actions[2])
+        cPickle.dump(actions, open(picklename, 'wb'))
+        self.assertEqual(True, zcml_configure('configure.zcml', self.module))
+
 class Dummy:
     pass
 
diff --git a/repoze/bfg/zcml.py b/repoze/bfg/zcml.py
index 343abc1d7..0bb049516 100644
--- a/repoze/bfg/zcml.py
+++ b/repoze/bfg/zcml.py
@@ -1,3 +1,11 @@
+import cPickle
+import os
+from os.path import realpath
+import time
+
+from zope.configuration import xmlconfig
+import zope.configuration.config
+
 from zope.component.zcml import handler
 from zope.component.interface import provideInterface
 from zope.configuration.exceptions import ConfigurationError
@@ -10,6 +18,7 @@ from zope.schema import TextLine
 from repoze.bfg.interfaces import IRequest
 from repoze.bfg.interfaces import IViewPermission
 from repoze.bfg.interfaces import IView
+from repoze.bfg.path import package_path
 
 from repoze.bfg.security import ViewPermissionFactory
 
@@ -84,4 +93,80 @@ class IViewDirective(Interface):
         required=False
         )
 
+PVERSION = 0
+
+def pickle_name(name, package):
+    path = package_path(package)
+    basename = os.path.join(path, name)
+    return os.path.join(path, basename + '.pck')
+
+def zcml_configure(name, package, load=cPickle.load):
+    """ Execute pickled zcml actions or fall back to parsing from file
+    """
+    pckname = pickle_name(name, package)
+
+    if not (os.path.isfile(pckname) or os.path.islink(pckname)):
+        return file_configure(name, package)
+
+    try:
+        vers, ptime, actions = load(open(pckname, 'rb'))
+    except (IOError, cPickle.UnpicklingError, EOFError, TypeError, ValueError):
+        return file_configure(name, package)
+
+    if vers != PVERSION:
+        return file_configure(name, package)
+
+    try:
+        ptime = int(ptime)
+    except:
+        return file_configure(name, package)
+
+    if not hasattr(actions, '__iter__'):
+        return file_configure(name, package)
+
+    files = set()
+    for action in actions:
+        # files list used by pickled action is an element of the tuple
+        try:
+            files.update(action[4])
+        except (TypeError, IndexError):
+            return file_configure(name, package)
+
+    for file in files:
+        if not(os.path.isfile(file) or os.path.islink(file)):
+            return file_configure(name, package)
+
+        mtime = os.stat(realpath(file)).st_mtime
+
+        if  mtime >= ptime:
+            return file_configure(name, package)
+
+    context = zope.configuration.config.ConfigurationMachine()
+    xmlconfig.registerCommonDirectives(context)
+    context.actions = actions
+    context.execute_actions()
+    return True
+
+def file_configure(name, package, dump=cPickle.dump):
+    context = zope.configuration.config.ConfigurationMachine()
+    xmlconfig.registerCommonDirectives(context)
+    context.package = package
+
+    xmlconfig.include(context, name, package)
+    context.execute_actions(clear=False)
+
+    actions = context.actions
+    pckname = pickle_name(name, package)
+
+    try:
+        data = (PVERSION, time.time(), actions)
+        dump(data, open(pckname, 'wb'), -1)
+    except (OSError, IOError, TypeError, cPickle.PickleError):
+        try:
+            os.remove(pckname)
+        except:
+            pass
+
+    return False
+