- The ``repoze.bfg.traversal.traversal_path`` API now eagerly attempts

  to encode a Unicode ``path`` into ASCII before attempting to split
  it and decode its segments.  This is for convenience, effectively to
  allow a (stored-as-Unicode-in-a-database, or
  retrieved-as-Unicode-from-a-request-parameter) Unicode path to be
  passed to ``find_model``, which eventually internally uses the
  ``traversal_path`` function under the hood.  In version 1.2 and
  prior, if the ``path`` was Unicode, that Unicode was split on
  slashes and each resulting segment value was Unicode.  An
  inappropriate call to the ``decode()`` method of a resulting Unicode
  path segment could cause a ``UnicodeDecodeError`` to occur even if
  the Unicode representation of the path contained no 'high order'
  characters (it effectively did a "double decode").  By converting
  the Unicode path argument to ASCII before we attempt to decode and
  split, genuine errors will occur in a more obvious place while also
  allowing us to handle (for convenience) the case that it's a Unicode
  representation formed entirely from ASCII-compatible characters.

1 files changed, 12 insertions, 0 deletions

diff --git a/repoze/bfg/tests/test_traversal.py b/repoze/bfg/tests/test_traversal.py
index a9727902e..1cae0a05f 100644
--- a/repoze/bfg/tests/test_traversal.py
+++ b/repoze/bfg/tests/test_traversal.py
@@ -50,6 +50,18 @@ class TraversalPathTests(unittest.TestCase):
         path = '/'.join([encoded, encoded])
         self.assertRaises(URLDecodeError, self._callFUT, path)
 
+    def test_unicode_highorder_chars(self):
+        path = u'/%E6%B5%81%E8%A1%8C%E8%B6%8B%E5%8A%BF'
+        self.assertEqual(self._callFUT(path), (u'\u6d41\u884c\u8d8b\u52bf',))
+
+    def test_unicode_simple(self):
+        path = u'/abc'
+        self.assertEqual(self._callFUT(path), (u'abc',))
+
+    def test_unicode_undecodeable_to_ascii(self):
+        path = unicode('/La Pe\xc3\xb1a', 'utf-8')
+        self.assertRaises(UnicodeEncodeError, self._callFUT, path)
+
 class ModelGraphTraverserTests(unittest.TestCase):
     def setUp(self):
         cleanUp()