Merge branch 'master' into extract_http_basic

18 files changed, 130 insertions, 45 deletions

diff --git a/CHANGES.txt b/CHANGES.txt
index 3e659ee9a..8cb4c602e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -19,10 +19,15 @@ Features
 
 Bug Fixes
 ---------
+
 - Fixed bug in `proutes` such that it now shows the correct view when a class
   and `attr` is involved.
   See: https://github.com/Pylons/pyramid/pull/2687
 
+- The JSON renderers now encode their result as UTF-8. The renderer helper
+  will now warn the user and encode the result as UTF-8 if a renderer returns a
+  text type and the response does not have a valid character set. See
+  https://github.com/Pylons/pyramid/pull/2706
 
 Deprecations
 ------------
@@ -30,4 +35,4 @@ Deprecations
 Documentation Changes
 ---------------------
 - Updated Windows installation instructions and related bits.
-  See: https://github.com/Pylons/pyramid/issues/2661
\ No newline at end of file
+  See: https://github.com/Pylons/pyramid/issues/2661
diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt
index 768dbccf9..bb21337e2 100644
--- a/CONTRIBUTORS.txt
+++ b/CONTRIBUTORS.txt
@@ -281,3 +281,6 @@ Contributors
 
 - Dariusz Gorecki, 2016/07/15
 
+- Jon Davidson, 2016/07/18
+
+- Keith Yang, 2016/07/22
diff --git a/docs/narr/hooks.rst b/docs/narr/hooks.rst
index 49ef29d3f..c54b213f1 100644
--- a/docs/narr/hooks.rst
+++ b/docs/narr/hooks.rst
@@ -26,7 +26,7 @@ Not Found View by using the
    :linenos:
 
    def notfound(request):
-       return Response('Not Found, dude', status='404 Not Found')
+       return Response('Not Found', status='404 Not Found')
 
    def main(globals, **settings):
        config = Configurator()
@@ -45,7 +45,7 @@ and a :term:`scan`, you can replace the Not Found View by using the
 
    @notfound_view_config()
    def notfound(request):
-       return Response('Not Found, dude', status='404 Not Found')
+       return Response('Not Found', status='404 Not Found')
 
    def main(globals, **settings):
        config = Configurator()
@@ -67,11 +67,11 @@ Views can carry predicates limiting their applicability.  For example:
 
    @notfound_view_config(request_method='GET')
    def notfound_get(request):
-       return Response('Not Found during GET, dude', status='404 Not Found')
+       return Response('Not Found during GET', status='404 Not Found')
 
    @notfound_view_config(request_method='POST')
    def notfound_post(request):
-       return Response('Not Found during POST, dude', status='404 Not Found')
+       return Response('Not Found during POST', status='404 Not Found')
 
    def main(globals, **settings):
       config = Configurator()
diff --git a/docs/narr/urldispatch.rst b/docs/narr/urldispatch.rst
index 7d37c04df..9ac01e24a 100644
--- a/docs/narr/urldispatch.rst
+++ b/docs/narr/urldispatch.rst
@@ -850,7 +850,7 @@ application:
    from pyramid.httpexceptions import HTTPNotFound
 
    def notfound(request):
-       return HTTPNotFound('Not found, bro.')
+       return HTTPNotFound()
 
    def no_slash(request):
        return Response('No slash')
@@ -871,7 +871,7 @@ If a request enters the application with the ``PATH_INFO`` value of
 However, if a request enters the application with the ``PATH_INFO`` value of
 ``/no_slash/``, *no* route will match, and the slash-appending not found view
 will not find a matching route with an appended slash.  As a result, the
-``notfound`` view will be called and it will return a "Not found, bro." body.
+``notfound`` view will be called and it will return a "Not found" body.
 
 If a request enters the application with the ``PATH_INFO`` value of
 ``/has_slash/``, the second route will match.  If a request enters the
@@ -892,7 +892,7 @@ exactly the same job:
 
    @notfound_view_config(append_slash=True)
    def notfound(request):
-       return HTTPNotFound('Not found, bro.')
+       return HTTPNotFound()
 
    @view_config(route_name='noslash')
    def no_slash(request):
diff --git a/docs/quick_tutorial/authentication.rst b/docs/quick_tutorial/authentication.rst
index acff97f3b..892beb3ec 100644
--- a/docs/quick_tutorial/authentication.rst
+++ b/docs/quick_tutorial/authentication.rst
@@ -1,7 +1,7 @@
 .. _qtut_authentication:
 
 ==============================
-20: Logins With Authentication
+20: Logins with authentication
 ==============================
 
 Login views that authenticate a username and password against a list of users.
@@ -34,6 +34,18 @@ Steps
    .. code-block:: bash
 
     $ cd ..; cp -r view_classes authentication; cd authentication
+
+#. Add ``bcrypt`` as a dependency in ``authentication/setup.py``:
+
+   .. literalinclude:: authentication/setup.py
+    :language: python
+    :emphasize-lines: 5-6
+    :linenos:
+
+#. We can now install our project in development mode:
+
+   .. code-block:: bash
+
     $ $VENV/bin/pip install -e .
 
 #. Put the security hash in the ``authentication/development.ini``
@@ -96,8 +108,8 @@ Unlike many web frameworks, Pyramid includes a built-in but optional security
 model for authentication and authorization. This security system is intended to
 be flexible and support many needs. In this security model, authentication (who
 are you) and authorization (what are you allowed to do) are not just pluggable,
-but de-coupled. To learn one step at a time, we provide a system that
-identifies users and lets them log out.
+but decoupled. To learn one step at a time, we provide a system that identifies
+users and lets them log out.
 
 In this example we chose to use the bundled :ref:`AuthTktAuthenticationPolicy
 <authentication_module>` policy. We enabled it in our configuration and
@@ -108,6 +120,20 @@ returned a login form. When reached via ``POST``, it processed the submitted
 username and password against the "groupfinder" callable that we registered in
 the configuration.
 
+The function ``hash_password`` uses a one-way hashing algorithm with a salt on
+the user's password via ``bcrypt``, instead of storing the password in plain
+text. This is considered to be a "best practice" for security.
+
+.. note::
+    There are alternative libraries to ``bcrypt`` if it is an issue on your
+    system. Just make sure that the library uses an algorithm approved for
+    storing passwords securely.
+
+The function ``check_password`` will compare the two hashed values of the
+submitted password and the user's password stored in the database. If the
+hashed values are equivalent, then the user is authenticated, else
+authentication fails.
+
 In our template, we fetched the ``logged_in`` value from the view class. We use
 this to calculate the logged-in user, if any. In the template we can then
 choose to show a login link to anonymous visitors or a logout link to logged-in
@@ -125,4 +151,5 @@ Extra credit
    request? Use ``import pdb; pdb.set_trace()`` to answer this.
 
 .. seealso:: See also :ref:`security_chapter`,
-   :ref:`AuthTktAuthenticationPolicy <authentication_module>`.
+   :ref:`AuthTktAuthenticationPolicy <authentication_module>`, `bcrypt
+   <https://pypi.python.org/pypi/bcrypt>`_
diff --git a/docs/quick_tutorial/authentication/setup.py b/docs/quick_tutorial/authentication/setup.py
index 2221b72e9..7a6ff4226 100644
--- a/docs/quick_tutorial/authentication/setup.py
+++ b/docs/quick_tutorial/authentication/setup.py
@@ -2,7 +2,8 @@ from setuptools import setup
 
 requires = [
     'pyramid',
-    'pyramid_chameleon'
+    'pyramid_chameleon',
+    'bcrypt'
 ]
 
 setup(name='tutorial',
diff --git a/docs/quick_tutorial/authentication/tutorial/security.py b/docs/quick_tutorial/authentication/tutorial/security.py
index ab90bab2c..e585e2642 100644
--- a/docs/quick_tutorial/authentication/tutorial/security.py
+++ b/docs/quick_tutorial/authentication/tutorial/security.py
@@ -1,5 +1,17 @@
-USERS = {'editor': 'editor',
-         'viewer': 'viewer'}
+import bcrypt
+
+
+def hash_password(pw):
+    pwhash = bcrypt.hashpw(pw.encode('utf8'), bcrypt.gensalt())
+    return pwhash.decode('utf8')
+
+def check_password(pw, hashed_pw):
+    expected_hash = hashed_pw.encode('utf8')
+    return bcrypt.checkpw(pw.encode('utf8'), expected_hash)
+
+
+USERS = {'editor': hash_password('editor'),
+         'viewer': hash_password('viewer')}
 GROUPS = {'editor': ['group:editors']}
 
 
diff --git a/docs/quick_tutorial/authentication/tutorial/views.py b/docs/quick_tutorial/authentication/tutorial/views.py
index ab46eb2dd..b07538d5e 100644
--- a/docs/quick_tutorial/authentication/tutorial/views.py
+++ b/docs/quick_tutorial/authentication/tutorial/views.py
@@ -9,7 +9,10 @@ from pyramid.view import (
     view_defaults
     )
 
-from .security import USERS
+from .security import (
+    USERS,
+    check_password
+)
 
 
 @view_defaults(renderer='home.pt')
@@ -40,7 +43,7 @@ class TutorialViews:
         if 'form.submitted' in request.params:
             login = request.params['login']
             password = request.params['password']
-            if USERS.get(login) == password:
+            if check_password(password, USERS.get(login)):
                 headers = remember(request, login)
                 return HTTPFound(location=came_from,
                                  headers=headers)
diff --git a/docs/quick_tutorial/authorization/setup.py b/docs/quick_tutorial/authorization/setup.py
index 2221b72e9..7a6ff4226 100644
--- a/docs/quick_tutorial/authorization/setup.py
+++ b/docs/quick_tutorial/authorization/setup.py
@@ -2,7 +2,8 @@ from setuptools import setup
 
 requires = [
     'pyramid',
-    'pyramid_chameleon'
+    'pyramid_chameleon',
+    'bcrypt'
 ]
 
 setup(name='tutorial',
diff --git a/docs/quick_tutorial/authorization/tutorial/security.py b/docs/quick_tutorial/authorization/tutorial/security.py
index ab90bab2c..e585e2642 100644
--- a/docs/quick_tutorial/authorization/tutorial/security.py
+++ b/docs/quick_tutorial/authorization/tutorial/security.py
@@ -1,5 +1,17 @@
-USERS = {'editor': 'editor',
-         'viewer': 'viewer'}
+import bcrypt
+
+
+def hash_password(pw):
+    pwhash = bcrypt.hashpw(pw.encode('utf8'), bcrypt.gensalt())
+    return pwhash.decode('utf8')
+
+def check_password(pw, hashed_pw):
+    expected_hash = hashed_pw.encode('utf8')
+    return bcrypt.checkpw(pw.encode('utf8'), expected_hash)
+
+
+USERS = {'editor': hash_password('editor'),
+         'viewer': hash_password('viewer')}
 GROUPS = {'editor': ['group:editors']}
 
 
diff --git a/docs/quick_tutorial/authorization/tutorial/views.py b/docs/quick_tutorial/authorization/tutorial/views.py
index 43d14455a..b2dc905c0 100644
--- a/docs/quick_tutorial/authorization/tutorial/views.py
+++ b/docs/quick_tutorial/authorization/tutorial/views.py
@@ -10,7 +10,10 @@ from pyramid.view import (
     forbidden_view_config
     )
 
-from .security import USERS
+from .security import (
+    USERS,
+    check_password
+)
 
 
 @view_defaults(renderer='home.pt')
@@ -42,7 +45,7 @@ class TutorialViews:
         if 'form.submitted' in request.params:
             login = request.params['login']
             password = request.params['password']
-            if USERS.get(login) == password:
+            if check_password(password, USERS.get(login)):
                 headers = remember(request, login)
                 return HTTPFound(location=came_from,
                                  headers=headers)
diff --git a/docs/quick_tutorial/forms.rst b/docs/quick_tutorial/forms.rst
index 1f421ee67..84ceb13d6 100644
--- a/docs/quick_tutorial/forms.rst
+++ b/docs/quick_tutorial/forms.rst
@@ -41,6 +41,7 @@ Steps
    pulls in Colander as a dependency:
 
    .. literalinclude:: forms/setup.py
+    :emphasize-lines: 5-6
     :linenos:
 
 #. We can now install our project in development mode:
diff --git a/docs/tutorials/wiki2/basiclayout.rst b/docs/tutorials/wiki2/basiclayout.rst
index ce67bb9e3..98a14c644 100644
--- a/docs/tutorials/wiki2/basiclayout.rst
+++ b/docs/tutorials/wiki2/basiclayout.rst
@@ -114,7 +114,7 @@ Finally ``main`` is finished configuring things, so it uses the
 Route declarations
 ------------------
 
-Open the ``tutorials/routes.py`` file. It should already contain the following:
+Open the ``tutorial/routes.py`` file. It should already contain the following:
 
 .. literalinclude:: src/basiclayout/tutorial/routes.py
   :linenos:
diff --git a/docs/tutorials/wiki2/installation.rst b/docs/tutorials/wiki2/installation.rst
index a214b1306..0440c2d1d 100644
--- a/docs/tutorials/wiki2/installation.rst
+++ b/docs/tutorials/wiki2/installation.rst
@@ -402,13 +402,6 @@ initialize our database.
    already have a database, you should delete it before running
    ``initialize_tutorial_db`` again.
 
-.. note::
-
-   The ``initialize_tutorial_db`` command is not performing a migration but
-   rather simply creating missing tables and adding some dummy data. If you
-   already have a database, you should delete it before running
-   ``initialize_tutorial_db`` again.
-
 Type the following command, making sure you are still in the ``tutorial``
 directory (the directory with a ``development.ini`` in it):
 
diff --git a/pyramid/renderers.py b/pyramid/renderers.py
index 9b3f19510..5b915ffdf 100644
--- a/pyramid/renderers.py
+++ b/pyramid/renderers.py
@@ -1,6 +1,7 @@
 import json
 import os
 import re
+import warnings
 
 from zope.interface import (
     implementer,
@@ -272,7 +273,7 @@ class JSON(object):
                 if ct == response.default_content_type:
                     response.content_type = 'application/json'
             default = self._make_default(request)
-            return self.serializer(value, default=default, **self.kw)
+            return self.serializer(value, default=default, **self.kw).encode('UTF-8')
 
         return _render
 
@@ -379,7 +380,7 @@ class JSONP(JSON):
                         raise HTTPBadRequest('Invalid JSONP callback function name.')
 
                     ct = 'application/javascript'
-                    body = '/**/{0}({1});'.format(callback, val)
+                    body = '/**/{0}({1});'.format(callback, val).encode('UTF-8')
                 response = request.response
                 if response.content_type == response.default_content_type:
                     response.content_type = ct
@@ -467,7 +468,17 @@ class RendererHelper(object):
 
         if result is not None:
             if isinstance(result, text_type):
-                response.text = result
+                if response.charset is None:
+                    warnings.warn(
+                        "Renderer returned a result of type {0}, "
+                        "however the response Content-Type <{1}> does not "
+                        "have a charset. Implicitly encoding the result as "
+                        "UTF-8.".format(type(result), response.content_type),
+                        RuntimeWarning
+                    )
+                    response.body = result.encode('UTF-8')
+                else:
+                    response.text = result
             elif isinstance(result, bytes):
                 response.body = result
             elif hasattr(result, '__iter__'):
diff --git a/pyramid/tests/test_config/test_views.py b/pyramid/tests/test_config/test_views.py
index 878574e88..c57deec7a 100644
--- a/pyramid/tests/test_config/test_views.py
+++ b/pyramid/tests/test_config/test_views.py
@@ -2168,7 +2168,7 @@ class TestViewsConfigurationMixin(unittest.TestCase):
                                      ctx_iface=implementedBy(HTTPNotFound),
                                      request_iface=IRequest)
         result = view(None, request)
-        self._assertBody(result, '{}')
+        self._assertBody(result, b'{}')
 
     def test_add_forbidden_view_with_renderer(self):
         from zope.interface import implementedBy
@@ -2185,7 +2185,7 @@ class TestViewsConfigurationMixin(unittest.TestCase):
                                      ctx_iface=implementedBy(HTTPForbidden),
                                      request_iface=IRequest)
         result = view(None, request)
-        self._assertBody(result, '{}')
+        self._assertBody(result, b'{}')
 
     def test_set_view_mapper(self):
         from pyramid.interfaces import IViewMapperFactory
diff --git a/pyramid/tests/test_renderers.py b/pyramid/tests/test_renderers.py
index 65bfa5582..ce337cd99 100644
--- a/pyramid/tests/test_renderers.py
+++ b/pyramid/tests/test_renderers.py
@@ -18,7 +18,7 @@ class TestJSON(unittest.TestCase):
     def test_it(self):
         renderer = self._makeOne()(None)
         result = renderer({'a':1}, {})
-        self.assertEqual(result, '{"a": 1}')
+        self.assertEqual(result, b'{"a": 1}')
 
     def test_with_request_content_type_notset(self):
         request = testing.DummyRequest()
@@ -43,7 +43,7 @@ class TestJSON(unittest.TestCase):
         renderer = self._makeOne()
         renderer.add_adapter(datetime, adapter)
         result = renderer(None)({'a':now}, {'request':request})
-        self.assertEqual(result, '{"a": "%s"}' % now.isoformat())
+        self.assertEqual(result, '{{"a": "{0}"}}'.format(now.isoformat()).encode('UTF-8'))
 
     def test_with_custom_adapter2(self):
         request = testing.DummyRequest()
@@ -54,7 +54,7 @@ class TestJSON(unittest.TestCase):
         now = datetime.utcnow()
         renderer = self._makeOne(adapters=((datetime, adapter),))
         result = renderer(None)({'a':now}, {'request':request})
-        self.assertEqual(result, '{"a": "%s"}' % now.isoformat())
+        self.assertEqual(result, '{{"a": "{0}"}}'.format(now.isoformat()).encode('UTF-8'))
 
     def test_with_custom_serializer(self):
         class Serializer(object):
@@ -66,7 +66,7 @@ class TestJSON(unittest.TestCase):
         renderer = self._makeOne(serializer=serializer, baz=5)
         obj = {'a':'b'}
         result = renderer(None)(obj, {})
-        self.assertEqual(result, 'foo')
+        self.assertEqual(result, b'foo')
         self.assertEqual(serializer.obj, obj)
         self.assertEqual(serializer.kw['baz'], 5)
         self.assertTrue('default' in serializer.kw)
@@ -84,7 +84,7 @@ class TestJSON(unittest.TestCase):
         objects = [MyObject(1), MyObject(2)]
         renderer = self._makeOne()(None)
         result = renderer(objects, {'request':request})
-        self.assertEqual(result, '[{"x": 1}, {"x": 2}]')
+        self.assertEqual(result, b'[{"x": 1}, {"x": 2}]')
 
     def test_with_object_adapter_no___json__(self):
         class MyObject(object):
@@ -290,6 +290,19 @@ class TestRendererHelper(unittest.TestCase):
         response = helper._make_response(la.encode('utf-8'), request)
         self.assertEqual(response.body, la.encode('utf-8'))
 
+    def test__make_response_result_is_str_no_charset(self):
+        from pyramid.response import Response
+        request = testing.DummyRequest()
+        request.response = Response(content_type='application/json', charset=None)
+
+        self.assertIsNone(request.response.charset)
+
+        helper = self._makeOne('loo.foo')
+        la = text_('/La Pe\xc3\xb1a', 'utf-8')
+        response = helper._make_response(la, request)
+        self.assertIsNone(response.charset)
+        self.assertEqual(response.body, la.encode('utf-8'))
+
     def test__make_response_result_is_iterable(self):
         from pyramid.response import Response
         request = testing.DummyRequest()
@@ -492,7 +505,7 @@ class Test_render(unittest.TestCase):
         request.response = response
         # use a json renderer, which will mutate the response
         result = self._callFUT('json', dict(a=1), request=request)
-        self.assertEqual(result, '{"a": 1}')
+        self.assertEqual(result, b'{"a": 1}')
         self.assertEqual(request.response, response)
 
     def test_no_response_to_preserve(self):
@@ -507,7 +520,7 @@ class Test_render(unittest.TestCase):
         request = DummyRequestWithClassResponse()
         # use a json renderer, which will mutate the response
         result = self._callFUT('json', dict(a=1), request=request)
-        self.assertEqual(result, '{"a": 1}')
+        self.assertEqual(result, b'{"a": 1}')
         self.assertFalse('response' in request.__dict__)
 
 class Test_render_to_response(unittest.TestCase):
@@ -627,7 +640,7 @@ class TestJSONP(unittest.TestCase):
         request = testing.DummyRequest()
         request.GET['callback'] = 'callback'
         result = renderer({'a':'1'}, {'request':request})
-        self.assertEqual(result, '/**/callback({"a": "1"});')
+        self.assertEqual(result, b'/**/callback({"a": "1"});')
         self.assertEqual(request.response.content_type,
                          'application/javascript')
 
@@ -637,7 +650,7 @@ class TestJSONP(unittest.TestCase):
         request = testing.DummyRequest()
         request.GET['callback'] = 'angular.callbacks._0'
         result = renderer({'a':'1'}, {'request':request})
-        self.assertEqual(result, '/**/angular.callbacks._0({"a": "1"});')
+        self.assertEqual(result, b'/**/angular.callbacks._0({"a": "1"});')
         self.assertEqual(request.response.content_type,
                          'application/javascript')
 
diff --git a/pyramid/view.py b/pyramid/view.py
index 88c6397af..0ef2d65d4 100644
--- a/pyramid/view.py
+++ b/pyramid/view.py
@@ -341,7 +341,7 @@ class notfound_view_config(object):
 
         @notfound_view_config()
         def notfound(request):
-            return Response('Not found, dude!', status='404 Not Found')
+            return Response('Not found!', status='404 Not Found')
 
     All arguments except ``append_slash`` have the same meaning as
     :meth:`pyramid.view.view_config` and each predicate