- In the past, during traversal, the ModelGraphTraverser (the

    default traverser) always passed each URL path segment to any
    ``__getitem__`` method of a model object as a byte string (a
    ``str`` object).  Now, by default the ModelGraphTraverser attempts
    to decode the path segment to Unicode (a ``unicode`` object) using
    the UTF-8 encoding before passing it to the ``__getitem__`` method
    of a model object.  This makes it possible for model objects to be
    dumber in ``__getitem__`` when trying to resolve a subobject, as
    model objects themselves no longer need to try to divine whether
    or not to try to decode the path segment passed by the
    traverser.

    Note that since 0.5.4, URLs generated by repoze.bfg's
    ``model_url`` API will contain UTF-8 encoded path segments as
    necessary, so any URL generated by BFG itself will be decodeable
    by the traverser.  If another application generates URLs to a BFG
    application, to be resolved successully, it should generate the
    URL with UTF-8 encoded path segments to be successfully resolved.
    The decoder is not at all magical: if a non-UTF-8-decodeable path
    segment (e.g. one encoded using UTF-16 or some other insanity) is
    passed in the URL, BFG will raise a ``TypeError`` with a message
    indicating it could not decode the path segment.

    To turn on the older behavior, where path segments were not
    decoded to Unicode before being passed to model object
    ``__getitem__`` by the traverser, and were passed as a raw byte
    string, set the ``unicode_path_segments`` configuration setting to
    a false value in your BFG application's section of the paste .ini
    file, for example::

      unicode_path_segments = False

    Or start the application using the ``BFG_UNICODE_PATH_SEGMENT``
    envvar set to a false value::

      BFG_UNICODE_PATH_SEGMENTS=0

2 files changed, 57 insertions, 0 deletions

diff --git a/repoze/bfg/tests/test_registry.py b/repoze/bfg/tests/test_registry.py
index 0b7eda586..f10fe6a52 100644
--- a/repoze/bfg/tests/test_registry.py
+++ b/repoze/bfg/tests/test_registry.py
@@ -117,6 +117,22 @@ class TestGetOptions(unittest.TestCase):
         self.assertEqual(result['debug_notfound'], True)
         self.assertEqual(result['debug_authorization'], True)
 
+    def test_unicode_path_segments(self):
+        get_options = self._getFUT()
+        result = get_options({})
+        self.assertEqual(result['unicode_path_segments'], False)
+        result = get_options({'unicode_path_segments':'false'})
+        self.assertEqual(result['unicode_path_segments'], False)
+        result = get_options({'unicode_path_segments':'t'})
+        self.assertEqual(result['unicode_path_segments'], True)
+        result = get_options({'unicode_path_segments':'1'})
+        self.assertEqual(result['unicode_path_segments'], True)
+        result = get_options({}, {'BFG_UNICODE_PATH_SEGMENTS':'1'})
+        self.assertEqual(result['unicode_path_segments'], True)
+        result = get_options({'unicode_path_segments':'false'},
+                             {'BFG_UNICODE_PATH_SEGMENTS':'1'})
+        self.assertEqual(result['unicode_path_segments'], True)
+
 class TestSettings(unittest.TestCase):
     def _getTargetClass(self):
         from repoze.bfg.registry import Settings
@@ -131,12 +147,14 @@ class TestSettings(unittest.TestCase):
         self.assertEqual(settings.reload_templates, False)
         self.assertEqual(settings.debug_notfound, False)
         self.assertEqual(settings.debug_authorization, False)
+        self.assertEqual(settings.unicode_path_segments, True)
 
     def test_with_option(self):
         settings = self._makeOne(reload_templates=True)
         self.assertEqual(settings.reload_templates, True)
         self.assertEqual(settings.debug_notfound, False)
         self.assertEqual(settings.debug_authorization, False)
+        self.assertEqual(settings.unicode_path_segments, True)
 
 class TestThreadLocalRegistryManager(unittest.TestCase, PlacelessSetup):
     def setUp(self):
diff --git a/repoze/bfg/tests/test_traversal.py b/repoze/bfg/tests/test_traversal.py
index 32be1ebbb..21e2b6180 100644
--- a/repoze/bfg/tests/test_traversal.py
+++ b/repoze/bfg/tests/test_traversal.py
@@ -162,6 +162,42 @@ class ModelGraphTraverserTests(unittest.TestCase, PlacelessSetup):
         self.assertEqual(ctx.__parent__, bar)
         self.failIf(isProxy(ctx.__parent__))
 
+    def test_non_utf8_path_segment_unicode_path_segments_fails(self):
+        foo = DummyContext()
+        root = DummyContext(foo)
+        policy = self._makeOne(root)
+        segment = unicode('LaPe\xc3\xb1a', 'utf-8').encode('utf-16')
+        environ = self._getEnviron(PATH_INFO='/%s' % segment)
+        self.assertRaises(TypeError, policy, environ)
+
+    def test_non_utf8_path_segment_settings_unicode_path_segments_fails(self):
+        defaultkw = {'unicode_path_segments':True}
+        settings = DummySettings(**defaultkw)
+        from repoze.bfg.interfaces import ISettings
+        import zope.component
+        gsm = zope.component.getGlobalSiteManager()
+        gsm.registerUtility(settings, ISettings)
+        foo = DummyContext()
+        root = DummyContext(foo)
+        policy = self._makeOne(root)
+        segment = unicode('LaPe\xc3\xb1a', 'utf-8').encode('utf-16')
+        environ = self._getEnviron(PATH_INFO='/%s' % segment)
+        self.assertRaises(TypeError, policy, environ)
+
+    def test_non_utf8_path_segment_str_path_segments_succeeds(self):
+        defaultkw = {'unicode_path_segments':False}
+        settings = DummySettings(**defaultkw)
+        from repoze.bfg.interfaces import ISettings
+        import zope.component
+        gsm = zope.component.getGlobalSiteManager()
+        gsm.registerUtility(settings, ISettings)
+        foo = DummyContext()
+        root = DummyContext(foo)
+        policy = self._makeOne(root)
+        segment = unicode('LaPe\xc3\xb1a', 'utf-8').encode('utf-16')
+        environ = self._getEnviron(PATH_INFO='/%s' % segment)
+        ctx, name, subpath = policy(environ) # test is: this doesn't fail
+
 class FindInterfaceTests(unittest.TestCase):
     def _callFUT(self, context, iface):
         from repoze.bfg.traversal import find_interface
@@ -422,3 +458,6 @@ class DummyContext(object):
 class DummyRequest:
     application_url = 'http://example.com:5432/'
 
+class DummySettings:
+    def __init__(self, **kw):
+        self.__dict__.update(kw)