- In the past, during traversal, the ModelGraphTraverser (the

    default traverser) always passed each URL path segment to any
    ``__getitem__`` method of a model object as a byte string (a
    ``str`` object).  Now, by default the ModelGraphTraverser attempts
    to decode the path segment to Unicode (a ``unicode`` object) using
    the UTF-8 encoding before passing it to the ``__getitem__`` method
    of a model object.  This makes it possible for model objects to be
    dumber in ``__getitem__`` when trying to resolve a subobject, as
    model objects themselves no longer need to try to divine whether
    or not to try to decode the path segment passed by the
    traverser.

    Note that since 0.5.4, URLs generated by repoze.bfg's
    ``model_url`` API will contain UTF-8 encoded path segments as
    necessary, so any URL generated by BFG itself will be decodeable
    by the traverser.  If another application generates URLs to a BFG
    application, to be resolved successully, it should generate the
    URL with UTF-8 encoded path segments to be successfully resolved.
    The decoder is not at all magical: if a non-UTF-8-decodeable path
    segment (e.g. one encoded using UTF-16 or some other insanity) is
    passed in the URL, BFG will raise a ``TypeError`` with a message
    indicating it could not decode the path segment.

    To turn on the older behavior, where path segments were not
    decoded to Unicode before being passed to model object
    ``__getitem__`` by the traverser, and were passed as a raw byte
    string, set the ``unicode_path_segments`` configuration setting to
    a false value in your BFG application's section of the paste .ini
    file, for example::

      unicode_path_segments = False

    Or start the application using the ``BFG_UNICODE_PATH_SEGMENT``
    envvar set to a false value::

      BFG_UNICODE_PATH_SEGMENTS=0

1 files changed, 17 insertions, 3 deletions

diff --git a/repoze/bfg/traversal.py b/repoze/bfg/traversal.py
index e599ee6c1..89495a223 100644
--- a/repoze/bfg/traversal.py
+++ b/repoze/bfg/traversal.py
@@ -1,5 +1,7 @@
 import urllib
 import urlparse
+
+from zope.component import queryUtility
    
 from zope.interface import classProvides
 from zope.interface import implements
@@ -9,6 +11,7 @@ from repoze.bfg.location import lineage
 from repoze.bfg.interfaces import ILocation
 from repoze.bfg.interfaces import ITraverser
 from repoze.bfg.interfaces import ITraverserFactory
+from repoze.bfg.interfaces import ISettings
 
 def split_path(path):
     while path.startswith('/'):
@@ -26,7 +29,13 @@ def split_path(path):
             clean.append(segment)
     return clean
 
-def step(ob, name, default):
+def step(ob, name, default, as_unicode=True):
+    if as_unicode:
+        try:
+            name = name.decode('utf-8')
+        except UnicodeDecodeError:
+            raise TypeError('Could not decode path segment "%s" using the '
+                            'UTF-8 decoding scheme' % name)
     if name.startswith('@@'):
         return name[2:], default
     if not hasattr(ob, '__getitem__'):
@@ -36,7 +45,7 @@ def step(ob, name, default):
     except KeyError:
         return name, default
 
-_marker = ()
+_marker = []
 
 class ModelGraphTraverser(object):
     classProvides(ITraverserFactory)
@@ -44,8 +53,13 @@ class ModelGraphTraverser(object):
     def __init__(self, root):
         self.root = root
         self.locatable = ILocation.providedBy(root)
+        self.unicode_path_segments = True
+        settings = queryUtility(ISettings)
+        if settings is not None:
+            self.unicode_path_segments = settings.unicode_path_segments
 
     def __call__(self, environ):
+        unicode_path_segments = self.unicode_path_segments
         path = environ.get('PATH_INFO', '/')
         path = split_path(path)
         ob = self.root
@@ -54,7 +68,7 @@ class ModelGraphTraverser(object):
 
         while path:
             segment = path.pop(0)
-            segment, next = step(ob, segment, _marker)
+            segment, next = step(ob, segment, _marker, unicode_path_segments)
             if next is _marker:
                 name = segment
                 break