diff options
| -rw-r--r-- | .travis.yml | 6 | ||||
| -rw-r--r-- | CHANGES.txt | 47 | ||||
| -rw-r--r-- | CONTRIBUTORS.txt | 2 | ||||
| -rw-r--r-- | HACKING.txt | 8 | ||||
| -rw-r--r-- | RELEASING.txt | 4 | ||||
| -rw-r--r-- | docs/api/authentication.rst | 5 | ||||
| -rw-r--r-- | docs/narr/commandline.rst | 31 | ||||
| -rw-r--r-- | docs/narr/hooks.rst | 29 | ||||
| -rw-r--r-- | docs/narr/install.rst | 5 | ||||
| -rw-r--r-- | docs/narr/introduction.rst | 2 | ||||
| -rw-r--r-- | docs/quick_tour.rst | 2 | ||||
| -rw-r--r-- | docs/quick_tutorial/requirements.rst | 2 | ||||
| -rw-r--r-- | docs/quick_tutorial/scaffolds.rst | 6 | ||||
| -rw-r--r-- | pyramid/authentication.py | 100 | ||||
| -rw-r--r-- | pyramid/config/__init__.py | 206 | ||||
| -rw-r--r-- | pyramid/config/tweens.py | 15 | ||||
| -rw-r--r-- | pyramid/paster.py | 14 | ||||
| -rw-r--r-- | pyramid/router.py | 9 | ||||
| -rw-r--r-- | pyramid/scripting.py | 35 | ||||
| -rw-r--r-- | pyramid/scripts/proutes.py | 2 | ||||
| -rw-r--r-- | pyramid/testing.py | 1 | ||||
| -rw-r--r-- | pyramid/tests/test_authentication.py | 78 | ||||
| -rw-r--r-- | pyramid/tests/test_config/test_init.py | 123 | ||||
| -rw-r--r-- | pyramid/tests/test_scripting.py | 21 | ||||
| -rw-r--r-- | setup.py | 10 | ||||
| -rw-r--r-- | tox.ini | 11 |
26 files changed, 555 insertions, 219 deletions
diff --git a/.travis.yml b/.travis.yml index fbdd88224..b46f677a6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,6 @@ matrix: include: - python: 2.7 env: TOXENV=py27 - - python: 3.3 - env: TOXENV=py33 - python: 3.4 env: TOXENV=py34 - python: 3.5 @@ -20,6 +18,10 @@ matrix: env: TOXENV=docs - python: 3.5 env: TOXENV=pep8 + - python: nightly + env: TOXENV=py36 + allow_failures: + - env: TOXENV=py36 install: - travis_retry pip install tox diff --git a/CHANGES.txt b/CHANGES.txt index 02e3271ce..f17a04f92 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -14,9 +14,27 @@ Backward Incompatibilities To run your server as a daemon you should use a process manager instead of pserve. + See https://github.com/Pylons/pyramid/pull/2615 + Features -------- +- The `_get_credentials` private method of `BasicAuthAuthenticationPolicy` + has been extracted into standalone function ``extract_http_basic_credentials` + in `pyramid.authentication` module, this function extracts HTTP Basic + credentials from a ``request`` object, and returns them as a named tuple. + See https://github.com/Pylons/pyramid/pull/2662 + +- Pyramid 1.4 silently dropped a feature of the configurator that has been + restored. It's again possible for action discriminators to conflict across + different action orders. + See https://github.com/Pylons/pyramid/pull/2757 + +- ``pyramid.paster.bootstrap`` and its sibling ``pyramid.scripting.prepare`` + can now be used as context managers to automatically invoke the ``closer`` + and pop threadlocals off of the stack to prevent memory leaks. + See https://github.com/Pylons/pyramid/pull/2760 + Bug Fixes --------- @@ -24,10 +42,37 @@ Bug Fixes and `attr` is involved. See: https://github.com/Pylons/pyramid/pull/2687 +- Fix a ``FutureWarning`` in Python 3.5 when using ``re.split`` on the + ``format`` setting to the ``proutes`` script. + See https://github.com/Pylons/pyramid/pull/2714 + +- Fix a ``RuntimeWarning`` emitted by WebOb when using arbitrary objects + as the ``userid`` in the ``AuthTktAuthenticationPolicy``. This is now caught + by the policy and the object is serialized as a base64 string to avoid + the cryptic warning. Since the userid will be read back as a string on + subsequent requests a more useful warning is emitted encouraging you to + use a primitive type instead. + See https://github.com/Pylons/pyramid/pull/2715 + +- Pyramid 1.6 introduced the ability for an action to invoke another action. + There was a bug in the way that ``config.add_view`` would interact with + custom view derivers introduced in Pyramid 1.7 because the view's + discriminator cannot be computed until view derivers and view predicates + have been created in earlier orders. Invoking an action from another action + would trigger an unrolling of the pipeline and would compute discriminators + before they were ready. The new behavior respects the ``order`` of the action + and ensures the discriminators are not computed until dependent actions + from previous orders have executed. + See https://github.com/Pylons/pyramid/pull/2757 + Deprecations ------------ Documentation Changes --------------------- - Updated Windows installation instructions and related bits. - See: https://github.com/Pylons/pyramid/issues/2661 + See https://github.com/Pylons/pyramid/issues/2661 + +- Fix an inconsistency in the documentation between view predicates and + route predicates and highlight the differences in their APIs. + See https://github.com/Pylons/pyramid/pull/2764 diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 12b6fedcf..bb21337e2 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -279,6 +279,8 @@ Contributors - Jean-Christophe Bohin, 2016/06/13 +- Dariusz Gorecki, 2016/07/15 + - Jon Davidson, 2016/07/18 - Keith Yang, 2016/07/22 diff --git a/HACKING.txt b/HACKING.txt index 5bbdce0c6..4b237b56c 100644 --- a/HACKING.txt +++ b/HACKING.txt @@ -124,10 +124,10 @@ In order to add a feature to Pyramid: - The feature must be documented in both the API and narrative documentation (in ``docs/``). -- The feature must work fully on the following CPython versions: 2.6, 2.7, 3.2, - 3.3, 3.4, and 3.5 on both UNIX and Windows. +- The feature must work fully on the following CPython versions: 2.7, 3.4, + and 3.5 on both UNIX and Windows. -- The feature must work on the latest version of PyPy and PyPy3. +- The feature must work on the latest version of PyPy. - The feature must not cause installation or runtime failure on App Engine. If it doesn't cause installation or runtime failure, but doesn't actually @@ -199,7 +199,7 @@ Running Tests Alternately:: - $ tox -e{py26,py27,py32,py33,py34,py35,pypy,pypy3}-scaffolds, + $ tox -e{py27,py34,py35,pypy}-scaffolds, Test Coverage ------------- diff --git a/RELEASING.txt b/RELEASING.txt index 326dea993..4690fbd37 100644 --- a/RELEASING.txt +++ b/RELEASING.txt @@ -33,8 +33,8 @@ Prepare new release branch - Run tests on Windows if feasible. -- Make sure all scaffold tests pass (Py 2.7, 3.3, 3.4, 3.5, and pypy on UNIX; - this doesn't work on Windows): +- Make sure all scaffold tests pass (CPython 2.7, 3.4, and 3.5, and PyPy on + UNIX; this doesn't work on Windows): $ ./scaffoldtests.sh diff --git a/docs/api/authentication.rst b/docs/api/authentication.rst index 19d08618b..57f32327a 100644 --- a/docs/api/authentication.rst +++ b/docs/api/authentication.rst @@ -34,5 +34,10 @@ Helper Classes .. autoclass:: AuthTktCookieHelper :members: + .. autoclass:: HTTPBasicCredentials + :members: +Helper Functions +~~~~~~~~~~~~~~~~ + .. autofunction:: extract_http_basic_credentials diff --git a/docs/narr/commandline.rst b/docs/narr/commandline.rst index 6cd90d42f..242bc7ec7 100644 --- a/docs/narr/commandline.rst +++ b/docs/narr/commandline.rst @@ -649,6 +649,10 @@ using the :func:`pyramid.paster.bootstrap` command in the body of your script. .. versionadded:: 1.1 :func:`pyramid.paster.bootstrap` +.. versionchanged:: 1.8 + Added the ability for ``bootstrap`` to cleanup automatically via the + ``with`` statement. + In the simplest case, :func:`pyramid.paster.bootstrap` can be used with a single argument, which accepts the :term:`PasteDeploy` ``.ini`` file representing your Pyramid application's configuration as a single argument: @@ -656,8 +660,9 @@ representing your Pyramid application's configuration as a single argument: .. code-block:: python from pyramid.paster import bootstrap - env = bootstrap('/path/to/my/development.ini') - print(env['request'].route_url('home')) + + with bootstrap('/path/to/my/development.ini') as env: + print(env['request'].route_url('home')) :func:`pyramid.paster.bootstrap` returns a dictionary containing framework-related information. This dictionary will always contain a @@ -723,8 +728,9 @@ load instead of ``main``: .. code-block:: python from pyramid.paster import bootstrap - env = bootstrap('/path/to/my/development.ini#another') - print(env['request'].route_url('home')) + + with bootstrap('/path/to/my/development.ini#another') as env: + print(env['request'].route_url('home')) The above example specifies the ``another`` ``app``, ``pipeline``, or ``composite`` section of your PasteDeploy configuration file. The ``app`` @@ -761,9 +767,9 @@ desired request and passing it into :func:`~pyramid.paster.bootstrap`: from pyramid.request import Request request = Request.blank('/', base_url='https://example.com/prefix') - env = bootstrap('/path/to/my/development.ini#another', request=request) - print(env['request'].application_url) - # will print 'https://example.com/prefix' + with bootstrap('/path/to/my/development.ini#another', request=request) as env: + print(env['request'].application_url) + # will print 'https://example.com/prefix' Now you can readily use Pyramid's APIs for generating URLs: @@ -776,7 +782,9 @@ Now you can readily use Pyramid's APIs for generating URLs: Cleanup ~~~~~~~ -When your scripting logic finishes, it's good manners to call the ``closer`` +If you're using the ``with``-statement variant then there's nothing to +worry about. However if you're using the returned environment directly then +when your scripting logic finishes, it's good manners to call the ``closer`` callback: .. code-block:: python @@ -891,15 +899,12 @@ contains the following code: omit = options.omit if omit is None: omit = [] - env = bootstrap(config_uri) - settings, closer = env['registry'].settings, env['closer'] - try: + with bootstrap(config_uri) as env: + settings = env['registry'].settings for k, v in settings.items(): if any([k.startswith(x) for x in omit]): continue print('%-40s %-20s' % (k, v)) - finally: - closer() This script uses the Python ``optparse`` module to allow us to make sense out of extra arguments passed to the script. It uses the diff --git a/docs/narr/hooks.rst b/docs/narr/hooks.rst index c54b213f1..6d0a2a5a3 100644 --- a/docs/narr/hooks.rst +++ b/docs/narr/hooks.rst @@ -1481,7 +1481,7 @@ method. For example: phash = text def __call__(self, context, request): - return getattr(context, 'content_type', None) == self.val + return request.content_type == self.val The constructor of a predicate factory takes two arguments: ``val`` and ``config``. The ``val`` argument will be the argument passed to @@ -1500,13 +1500,28 @@ with the name and the value serialized. The result of ``phash`` is not seen in output anywhere, it just informs the uniqueness constraints for view configuration. -The ``__call__`` method of a predicate factory must accept a resource -(``context``) and a request, and must return ``True`` or ``False``. It is the -"meat" of the predicate. +The ``__call__`` method differs depending on whether the predicate is used as +a :term:`view predicate` or a :term:`route predicate`: -You can use the same predicate factory as both a view predicate and as a route -predicate, but you'll need to call ``add_view_predicate`` and -``add_route_predicate`` separately with the same factory. +- When used as a route predicate, the ``__call__`` signature is + ``(info, request)``. The ``info`` object is a dictionary containing two + keys: ``match`` and ``route``. ``info['match']`` is the matchdict containing + the patterns matched in the route pattern. ``info['route']`` is the + :class:`pyramid.interfaces.IRoute` object for the current route. + +- When used as a view predicate, the ``__call__`` signature is + ``(context, request)``. The ``context`` is the result of :term:`traversal` + performed using either the route's :term:`root factory` or the app's + :term:`default root factory`. + +In both cases the ``__call__`` method is expected to return ``True`` or +``False``. + +It is possible to use the same predicate factory as both a view predicate and +as a route predicate, but they'll need to handle the ``info`` or ``context`` +argument specially (many predicates do not need this argument) and you'll need +to call ``add_view_predicate`` and ``add_route_predicate`` separately with +the same factory. .. _subscriber_predicates: diff --git a/docs/narr/install.rst b/docs/narr/install.rst index c59ced2a5..677c27e4a 100644 --- a/docs/narr/install.rst +++ b/docs/narr/install.rst @@ -21,9 +21,8 @@ the following sections. .. sidebar:: Python Versions - As of this writing, :app:`Pyramid` has been tested under Python 2.7, - Python 3.3, Python 3.4, Python 3.5, PyPy, and PyPy3. :app:`Pyramid` does - not run under any version of Python before 2.7. + As of this writing, :app:`Pyramid` is tested against Python 2.7, + Python 3.4, Python 3.5, PyPy. :app:`Pyramid` is known to run on all popular UNIX-like systems such as Linux, Mac OS X, and FreeBSD, as well as on Windows platforms. It is also known to diff --git a/docs/narr/introduction.rst b/docs/narr/introduction.rst index de6ac408b..47638579b 100644 --- a/docs/narr/introduction.rst +++ b/docs/narr/introduction.rst @@ -860,7 +860,7 @@ Every release of Pyramid has 100% statement coverage via unit and integration tests, as measured by the ``coverage`` tool available on PyPI. It also has greater than 95% decision/condition coverage as measured by the ``instrumental`` tool available on PyPI. It is automatically tested by Travis, -and Jenkins on Python 2.7, Python 3.3, Python 3.4, Python 3.5, PyPy, and PyPy3 +and Jenkins on Python 2.7, Python 3.4, Python 3.5, and PyPy after each commit to its GitHub repository. Official Pyramid add-ons are held to a similar testing standard. We still find bugs in Pyramid and its official add-ons, but we've noticed we find a lot more of them while working on other diff --git a/docs/quick_tour.rst b/docs/quick_tour.rst index 88cd69400..b2dec77e9 100644 --- a/docs/quick_tour.rst +++ b/docs/quick_tour.rst @@ -52,7 +52,7 @@ For Windows: # or for a specific released version c:\\> %VENV%\\Scripts\\pip install "pyramid==\ |release|\ " -Of course Pyramid runs fine on Python 2.6+, as do the examples in this *Quick +Of course Pyramid runs fine on Python 2.7+, as do the examples in this *Quick Tour*. We're showing Python 3 for simplicity. (Pyramid had production support for Python 3 in October 2011.) Also for simplicity, the remaining examples will show only UNIX commands. diff --git a/docs/quick_tutorial/requirements.rst b/docs/quick_tutorial/requirements.rst index 958347a90..1de9a8acf 100644 --- a/docs/quick_tutorial/requirements.rst +++ b/docs/quick_tutorial/requirements.rst @@ -19,7 +19,7 @@ virtual environment.) This *Quick Tutorial* is based on: -* **Python 3.5**. Pyramid fully supports Python 3.3+ and Python 2.7+. This +* **Python 3.5**. Pyramid fully supports Python 3.4+ and Python 2.7+. This tutorial uses **Python 3.5** but runs fine under Python 2.7. * **venv**. We believe in virtual environments. For this tutorial, we use diff --git a/docs/quick_tutorial/scaffolds.rst b/docs/quick_tutorial/scaffolds.rst index 7845f2b71..ad002f4fd 100644 --- a/docs/quick_tutorial/scaffolds.rst +++ b/docs/quick_tutorial/scaffolds.rst @@ -38,9 +38,9 @@ Steps $ $VENV/bin/pcreate --list Available scaffolds: - alchemy: Pyramid SQLAlchemy project using url dispatch - starter: Pyramid starter project - zodb: Pyramid ZODB project using traversal + alchemy: Pyramid project using SQLAlchemy, SQLite, URL dispatch, and Jinja2 + starter: Pyramid starter project using URL dispatch and Chameleon + zodb: Pyramid project using ZODB, traversal, and Chameleon #. Tell ``pcreate`` to use the ``starter`` scaffold to make our project: diff --git a/pyramid/authentication.py b/pyramid/authentication.py index e6b888db2..2ee5576d9 100644 --- a/pyramid/authentication.py +++ b/pyramid/authentication.py @@ -1,10 +1,12 @@ import binascii from codecs import utf_8_decode from codecs import utf_8_encode +from collections import namedtuple import hashlib import base64 import re import time as time_mod +import warnings from zope.interface import implementer @@ -947,8 +949,19 @@ class AuthTktCookieHelper(object): if encoding_data: encoding, encoder = encoding_data - userid = encoder(userid) - user_data = 'userid_type:%s' % encoding + else: + warnings.warn( + "userid is of type {}, and is not supported by the " + "AuthTktAuthenticationPolicy. Explicitly converting to string " + "and storing as base64. Subsequent requests will receive a " + "string as the userid, it will not be decoded back to the type " + "provided.".format(type(userid)), RuntimeWarning + ) + encoding, encoder = self.userid_type_encoders.get(text_type) + userid = str(userid) + + userid = encoder(userid) + user_data = 'userid_type:%s' % encoding new_tokens = [] for token in tokens: @@ -1083,7 +1096,7 @@ class BasicAuthAuthenticationPolicy(CallbackAuthenticationPolicy): def unauthenticated_userid(self, request): """ The userid parsed from the ``Authorization`` request header.""" - credentials = self._get_credentials(request) + credentials = extract_http_basic_credentials(request) if credentials: return credentials[0] @@ -1100,42 +1113,15 @@ class BasicAuthAuthenticationPolicy(CallbackAuthenticationPolicy): return [('WWW-Authenticate', 'Basic realm="%s"' % self.realm)] def callback(self, username, request): - # Username arg is ignored. Unfortunately _get_credentials winds up - # getting called twice when authenticated_userid is called. Avoiding - # that, however, winds up duplicating logic from the superclass. - credentials = self._get_credentials(request) + # Username arg is ignored. Unfortunately + # extract_http_basic_credentials winds up getting called twice when + # authenticated_userid is called. Avoiding that, however, + # winds up duplicating logic from the superclass. + credentials = extract_http_basic_credentials(request) if credentials: username, password = credentials return self.check(username, password, request) - def _get_credentials(self, request): - authorization = request.headers.get('Authorization') - if not authorization: - return None - try: - authmeth, auth = authorization.split(' ', 1) - except ValueError: # not enough values to unpack - return None - if authmeth.lower() != 'basic': - return None - - try: - authbytes = b64decode(auth.strip()) - except (TypeError, binascii.Error): # can't decode - return None - - # try utf-8 first, then latin-1; see discussion in - # https://github.com/Pylons/pyramid/issues/898 - try: - auth = authbytes.decode('utf-8') - except UnicodeDecodeError: - auth = authbytes.decode('latin-1') - - try: - username, password = auth.split(':', 1) - except ValueError: # not enough values to unpack - return None - return username, password class _SimpleSerializer(object): def loads(self, bstruct): @@ -1143,3 +1129,47 @@ class _SimpleSerializer(object): def dumps(self, appstruct): return bytes_(appstruct) + + +HTTPBasicCredentials = namedtuple( + 'HTTPBasicCredentials', ['username', 'password']) + + +def extract_http_basic_credentials(request): + """ A helper function for extraction of HTTP Basic credentials + from a given :term:`request`. + + Returns a :class:`.HTTPBasicCredentials` 2-tuple with ``username`` and + ``password`` attributes or ``None`` if no credentials could be found. + + """ + authorization = request.headers.get('Authorization') + if not authorization: + return None + + try: + authmeth, auth = authorization.split(' ', 1) + except ValueError: # not enough values to unpack + return None + + if authmeth.lower() != 'basic': + return None + + try: + authbytes = b64decode(auth.strip()) + except (TypeError, binascii.Error): # can't decode + return None + + # try utf-8 first, then latin-1; see discussion in + # https://github.com/Pylons/pyramid/issues/898 + try: + auth = authbytes.decode('utf-8') + except UnicodeDecodeError: + auth = authbytes.decode('latin-1') + + try: + username, password = auth.split(':', 1) + except ValueError: # not enough values to unpack + return None + + return HTTPBasicCredentials(username, password) diff --git a/pyramid/config/__init__.py b/pyramid/config/__init__.py index 553f32c9b..d4064dc78 100644 --- a/pyramid/config/__init__.py +++ b/pyramid/config/__init__.py @@ -27,7 +27,6 @@ from pyramid.compat import ( text_, reraise, string_types, - zip_longest, ) from pyramid.events import ApplicationCreated @@ -380,6 +379,7 @@ class Configurator( self.add_default_view_predicates() self.add_default_view_derivers() self.add_default_route_predicates() + self.add_default_tweens() if exceptionresponse_view is not None: exceptionresponse_view = self.maybe_dotted(exceptionresponse_view) @@ -1110,29 +1110,8 @@ class ActionState(object): try: all_actions = [] executed_actions = [] - pending_actions = iter([]) - - # resolve the new action list against what we have already - # executed -- if a new action appears intertwined in the list - # of already-executed actions then someone wrote a broken - # re-entrant action because it scheduled the action *after* it - # should have been executed (as defined by the action order) - def resume(actions): - for a, b in zip_longest(actions, executed_actions): - if b is None and a is not None: - # common case is that we are executing every action - yield a - elif b is not None and a != b: - raise ConfigurationError( - 'During execution a re-entrant action was added ' - 'that modified the planned execution order in a ' - 'way that is incompatible with what has already ' - 'been executed.') - else: - # resolved action is in the same location as before, - # so we are in good shape, but the action is already - # executed so we skip it - assert b is not None and a == b + action_iter = iter([]) + conflict_state = ConflictResolverState() while True: # We clear the actions list prior to execution so if there @@ -1141,26 +1120,14 @@ class ActionState(object): # ensures that the previously executed actions have no new # conflicts. if self.actions: - # Only resolve the new actions against executed_actions - # and pending_actions instead of everything to avoid - # redundant checks. - # Assume ``actions = resolveConflicts([A, B, C])`` which - # after conflict checks, resulted in ``actions == [A]`` - # then we know action A won out or a conflict would have - # been raised. Thus, when action D is added later, we only - # need to check the new action against A. - # ``actions = resolveConflicts([A, D]) should drop the - # number of redundant checks down from O(n^2) closer to - # O(n lg n). all_actions.extend(self.actions) - pending_actions = resume(resolveConflicts( - executed_actions + - list(pending_actions) + - self.actions - )) + action_iter = resolveConflicts( + self.actions, + state=conflict_state, + ) self.actions = [] - action = next(pending_actions, None) + action = next(action_iter, None) if action is None: # we are done! break @@ -1176,9 +1143,7 @@ class ActionState(object): try: if callable is not None: callable(*args, **kw) - except (KeyboardInterrupt, SystemExit): # pragma: no cover - raise - except: + except Exception: t, v, tb = sys.exc_info() try: reraise(ConfigurationExecutionError, @@ -1193,65 +1158,102 @@ class ActionState(object): executed_actions.append(action) + self.actions = all_actions + return executed_actions + finally: if clear: - del self.actions[:] - else: - self.actions = all_actions + self.actions = [] + + +class ConflictResolverState(object): + def __init__(self): + # keep a set of resolved discriminators to test against to ensure + # that a new action does not conflict with something already executed + self.resolved_ainfos = {} + + # actions left over from a previous iteration + self.remaining_actions = [] + + # after executing an action we memoize its order to avoid any new + # actions sending us backward + self.min_order = None + + # unique tracks the index of the action so we need it to increase + # monotonically across invocations to resolveConflicts + self.start = 0 + # this function is licensed under the ZPL (stolen from Zope) -def resolveConflicts(actions): +def resolveConflicts(actions, state=None): """Resolve conflicting actions Given an actions list, identify and try to resolve conflicting actions. Actions conflict if they have the same non-None discriminator. + Conflicting actions can be resolved if the include path of one of the actions is a prefix of the includepaths of the other conflicting actions and is unequal to the include paths in the other conflicting actions. + + Actions are resolved on a per-order basis because some discriminators + cannot be computed until earlier actions have executed. An action in an + earlier order may execute successfully only to find out later that it was + overridden by another action with a smaller include path. This will result + in a conflict as there is no way to revert the original action. + + ``state`` may be an instance of ``ConflictResolverState`` that + can be used to resume execution and resolve the new actions against the + list of executed actions from a previous call. + """ + if state is None: + state = ConflictResolverState() + + # pick up where we left off last time, but track the new actions as well + state.remaining_actions.extend(normalize_actions(actions)) + actions = state.remaining_actions def orderandpos(v): n, v = v - if not isinstance(v, dict): - # old-style tuple action - v = expand_action(*v) return (v['order'] or 0, n) - sactions = sorted(enumerate(actions), key=orderandpos) - def orderonly(v): n, v = v - if not isinstance(v, dict): - # old-style tuple action - v = expand_action(*v) return v['order'] or 0 + sactions = sorted(enumerate(actions, start=state.start), key=orderandpos) for order, actiongroup in itertools.groupby(sactions, orderonly): # "order" is an integer grouping. Actions in a lower order will be # executed before actions in a higher order. All of the actions in # one grouping will be executed (its callable, if any will be called) # before any of the actions in the next. - - unique = {} output = [] + unique = {} + + # error out if we went backward in order + if state.min_order is not None and order < state.min_order: + r = ['Actions were added to order={0} after execution had moved ' + 'on to order={1}. Conflicting actions: ' + .format(order, state.min_order)] + for i, action in actiongroup: + for line in str(action['info']).rstrip().split('\n'): + r.append(" " + line) + raise ConfigurationError('\n'.join(r)) for i, action in actiongroup: # Within an order, actions are executed sequentially based on # original action ordering ("i"). - if not isinstance(action, dict): - # old-style tuple action - action = expand_action(*action) - - # "ainfo" is a tuple of (order, i, action) where "order" is a - # user-supplied grouping, "i" is an integer expressing the relative - # position of this action in the action list being resolved, and - # "action" is an action dictionary. The purpose of an ainfo is to - # associate an "order" and an "i" with a particular action; "order" - # and "i" exist for sorting purposes after conflict resolution. - ainfo = (order, i, action) + # "ainfo" is a tuple of (i, action) where "i" is an integer + # expressing the relative position of this action in the action + # list being resolved, and "action" is an action dictionary. The + # purpose of an ainfo is to associate an "i" with a particular + # action; "i" exists for sorting after conflict resolution. + ainfo = (i, action) + # wait to defer discriminators until we are on their order because + # the discriminator may depend on state from a previous order discriminator = undefer(action['discriminator']) action['discriminator'] = discriminator @@ -1266,28 +1268,39 @@ def resolveConflicts(actions): # Check for conflicts conflicts = {} - for discriminator, ainfos in unique.items(): - # We use (includepath, order, i) as a sort key because we need to + # We use (includepath, i) as a sort key because we need to # sort the actions by the paths so that the shortest path with a # given prefix comes first. The "first" action is the one with the - # shortest include path. We break sorting ties using "order", then - # "i". + # shortest include path. We break sorting ties using "i". def bypath(ainfo): - path, order, i = ainfo[2]['includepath'], ainfo[0], ainfo[1] + path, i = ainfo[1]['includepath'], ainfo[0] return path, order, i ainfos.sort(key=bypath) ainfo, rest = ainfos[0], ainfos[1:] - output.append(ainfo) - _, _, action = ainfo - basepath, baseinfo, discriminator = ( - action['includepath'], - action['info'], - action['discriminator'], - ) + _, action = ainfo + + # ensure this new action does not conflict with a previously + # resolved action from an earlier order / invocation + prev_ainfo = state.resolved_ainfos.get(discriminator) + if prev_ainfo is not None: + _, paction = prev_ainfo + basepath, baseinfo = paction['includepath'], paction['info'] + includepath = action['includepath'] + # if the new action conflicts with the resolved action then + # note the conflict, otherwise drop the action as it's + # effectively overriden by the previous action + if (includepath[:len(basepath)] != basepath or + includepath == basepath): + L = conflicts.setdefault(discriminator, [baseinfo]) + L.append(action['info']) + + else: + output.append(ainfo) - for _, _, action in rest: + basepath, baseinfo = action['includepath'], action['info'] + for _, action in rest: includepath = action['includepath'] # Test whether path is a prefix of opath if (includepath[:len(basepath)] != basepath or # not a prefix @@ -1298,14 +1311,30 @@ def resolveConflicts(actions): if conflicts: raise ConfigurationConflictError(conflicts) - # sort conflict-resolved actions by (order, i) and yield them one - # by one - for a in [x[2] for x in sorted(output, key=operator.itemgetter(0, 1))]: - yield a + # sort resolved actions by "i" and yield them one by one + for i, action in sorted(output, key=operator.itemgetter(0)): + # do not memoize the order until we resolve an action inside it + state.min_order = action['order'] + state.start = i + 1 + state.remaining_actions.remove(action) + state.resolved_ainfos[action['discriminator']] = (i, action) + yield action -def expand_action(discriminator, callable=None, args=(), kw=None, - includepath=(), info=None, order=0, introspectables=()): +def normalize_actions(actions): + """Convert old-style tuple actions to new-style dicts.""" + result = [] + for v in actions: + if not isinstance(v, dict): + v = expand_action_tuple(*v) + result.append(v) + return result + + +def expand_action_tuple( + discriminator, callable=None, args=(), kw=None, includepath=(), + info=None, order=0, introspectables=(), +): if kw is None: kw = {} return dict( @@ -1319,4 +1348,5 @@ def expand_action(discriminator, callable=None, args=(), kw=None, introspectables=introspectables, ) + global_registries = WeakOrderedSet() diff --git a/pyramid/config/tweens.py b/pyramid/config/tweens.py index 8e1800f33..0aeb01fe3 100644 --- a/pyramid/config/tweens.py +++ b/pyramid/config/tweens.py @@ -10,7 +10,6 @@ from pyramid.compat import ( from pyramid.exceptions import ConfigurationError from pyramid.tweens import ( - excview_tween_factory, MAIN, INGRESS, EXCVIEW, @@ -107,6 +106,9 @@ class TweensConfiguratorMixin(object): return self._add_tween(tween_factory, under=under, over=over, explicit=False) + def add_default_tweens(self): + self.add_tween(EXCVIEW) + @action_method def _add_tween(self, tween_factory, under=None, over=None, explicit=False): @@ -142,17 +144,6 @@ class TweensConfiguratorMixin(object): if tweens is None: tweens = Tweens() registry.registerUtility(tweens, ITweens) - ex_intr = self.introspectable('tweens', - ('tween', EXCVIEW, False), - EXCVIEW, - 'implicit tween') - ex_intr['name'] = EXCVIEW - ex_intr['factory'] = excview_tween_factory - ex_intr['type'] = 'implicit' - ex_intr['under'] = None - ex_intr['over'] = MAIN - introspectables.append(ex_intr) - tweens.add_implicit(EXCVIEW, excview_tween_factory, over=MAIN) def register(): if explicit: diff --git a/pyramid/paster.py b/pyramid/paster.py index 3916be8f0..1b7afb5dc 100644 --- a/pyramid/paster.py +++ b/pyramid/paster.py @@ -129,8 +129,22 @@ def bootstrap(config_uri, request=None, options=None): {'http_port': 8080} and then use %(http_port)s in the config file. + This function may be used as a context manager to call the ``closer`` + automatically: + + .. code-block:: python + + with bootstrap('development.ini') as env: + request = env['request'] + # ... + See :ref:`writing_a_script` for more information about how to use this function. + + .. versionchanged:: 1.8 + + Added the ability to use the return value as a context manager. + """ app = get_app(config_uri, options=options) env = prepare(request) diff --git a/pyramid/router.py b/pyramid/router.py index 19773cf62..fd11925e9 100644 --- a/pyramid/router.py +++ b/pyramid/router.py @@ -34,8 +34,6 @@ from pyramid.traversal import ( ResourceTreeTraverser, ) -from pyramid.tweens import excview_tween_factory - @implementer(IRouter) class Router(object): @@ -51,11 +49,10 @@ class Router(object): self.routes_mapper = q(IRoutesMapper) self.request_factory = q(IRequestFactory, default=Request) self.request_extensions = q(IRequestExtensions) - tweens = q(ITweens) - if tweens is None: - tweens = excview_tween_factory self.orig_handle_request = self.handle_request - self.handle_request = tweens(self.handle_request, registry) + tweens = q(ITweens) + if tweens is not None: + self.handle_request = tweens(self.handle_request, registry) self.root_policy = self.root_factory # b/w compat self.registry = registry settings = registry.settings diff --git a/pyramid/scripting.py b/pyramid/scripting.py index d9587338f..7607d3ea3 100644 --- a/pyramid/scripting.py +++ b/pyramid/scripting.py @@ -56,12 +56,25 @@ def prepare(request=None, registry=None): ``root`` returned is the application's root resource object. The ``closer`` returned is a callable (accepting no arguments) that should be called when your scripting application is finished - using the root. ``registry`` is the registry object passed or - the last registry loaded into - :attr:`pyramid.config.global_registries` if no registry is passed. + using the root. ``registry`` is the resolved registry object. ``request`` is the request object passed or the constructed request if no request is passed. ``root_factory`` is the root factory used to construct the root. + + This function may be used as a context manager to call the ``closer`` + automatically: + + .. code-block:: python + + registry = config.registry + with prepare(registry) as env: + request = env['request'] + # ... + + .. versionchanged:: 1.8 + + Added the ability to use the return value as a context manager. + """ if registry is None: registry = getattr(request, 'registry', global_registries.last) @@ -85,8 +98,20 @@ def prepare(request=None, registry=None): root = root_factory(request) if getattr(request, 'context', None) is None: request.context = root - return {'root':root, 'closer':closer, 'registry':registry, - 'request':request, 'root_factory':root_factory} + return AppEnvironment( + root=root, + closer=closer, + registry=registry, + request=request, + root_factory=root_factory, + ) + +class AppEnvironment(dict): + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self['closer']() def _make_request(path, registry=None): """ Return a :meth:`pyramid.request.Request` object anchored at a diff --git a/pyramid/scripts/proutes.py b/pyramid/scripts/proutes.py index 19d91cc72..f75810c06 100644 --- a/pyramid/scripts/proutes.py +++ b/pyramid/scripts/proutes.py @@ -296,7 +296,7 @@ class PRoutesCommand(object): items = config.items('proutes') for k, v in items: if 'format' == k: - cols = re.split(r'[,|\s|\n]*', v) + cols = re.split(r'[,|\s\n]+', v) self.column_format = [x.strip() for x in cols] except configparser.NoSectionError: diff --git a/pyramid/testing.py b/pyramid/testing.py index ec06fe379..877b351db 100644 --- a/pyramid/testing.py +++ b/pyramid/testing.py @@ -478,6 +478,7 @@ def setUp(registry=None, request=None, hook_zca=True, autocommit=True, config.add_default_view_predicates() config.add_default_view_derivers() config.add_default_route_predicates() + config.add_default_tweens() config.commit() global have_zca try: diff --git a/pyramid/tests/test_authentication.py b/pyramid/tests/test_authentication.py index 0a22e5965..b9a4c6be4 100644 --- a/pyramid/tests/test_authentication.py +++ b/pyramid/tests/test_authentication.py @@ -1089,7 +1089,10 @@ class TestAuthTktCookieHelper(unittest.TestCase): helper = self._makeOne('secret') request = self._makeRequest() userid = object() - result = helper.remember(request, userid) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always', RuntimeWarning) + result = helper.remember(request, userid) + self.assertTrue(str(w[-1].message).startswith('userid is of type')) values = self._parseHeaders(result) self.assertEqual(len(result), 3) value = values[0] @@ -1476,6 +1479,79 @@ class TestBasicAuthAuthenticationPolicy(unittest.TestCase): self.assertEqual(policy.forget(None), [ ('WWW-Authenticate', 'Basic realm="SomeRealm"')]) + +class TestExtractHTTPBasicCredentials(unittest.TestCase): + def _get_func(self): + from pyramid.authentication import extract_http_basic_credentials + return extract_http_basic_credentials + + def test_no_auth_header(self): + request = testing.DummyRequest() + fn = self._get_func() + + self.assertIsNone(fn(request)) + + def test_invalid_payload(self): + import base64 + request = testing.DummyRequest() + request.headers['Authorization'] = 'Basic %s' % base64.b64encode( + bytes_('chrisrpassword')).decode('ascii') + fn = self._get_func() + self.assertIsNone(fn(request)) + + def test_not_a_basic_auth_scheme(self): + import base64 + request = testing.DummyRequest() + request.headers['Authorization'] = 'OtherScheme %s' % base64.b64encode( + bytes_('chrisr:password')).decode('ascii') + fn = self._get_func() + self.assertIsNone(fn(request)) + + def test_no_base64_encoding(self): + request = testing.DummyRequest() + request.headers['Authorization'] = 'Basic ...' + fn = self._get_func() + self.assertIsNone(fn(request)) + + def test_latin1_payload(self): + import base64 + request = testing.DummyRequest() + inputs = (b'm\xc3\xb6rk\xc3\xb6:' + b'm\xc3\xb6rk\xc3\xb6password').decode('utf-8') + request.headers['Authorization'] = 'Basic %s' % ( + base64.b64encode(inputs.encode('latin-1')).decode('latin-1')) + fn = self._get_func() + self.assertEqual(fn(request), ( + b'm\xc3\xb6rk\xc3\xb6'.decode('utf-8'), + b'm\xc3\xb6rk\xc3\xb6password'.decode('utf-8') + )) + + def test_utf8_payload(self): + import base64 + request = testing.DummyRequest() + inputs = (b'm\xc3\xb6rk\xc3\xb6:' + b'm\xc3\xb6rk\xc3\xb6password').decode('utf-8') + request.headers['Authorization'] = 'Basic %s' % ( + base64.b64encode(inputs.encode('utf-8')).decode('latin-1')) + fn = self._get_func() + self.assertEqual(fn(request), ( + b'm\xc3\xb6rk\xc3\xb6'.decode('utf-8'), + b'm\xc3\xb6rk\xc3\xb6password'.decode('utf-8') + )) + + def test_namedtuple_return(self): + import base64 + request = testing.DummyRequest() + request.headers['Authorization'] = 'Basic %s' % base64.b64encode( + bytes_('chrisr:pass')).decode('ascii') + fn = self._get_func() + result = fn(request) + + self.assertEqual(result.username, 'chrisr') + self.assertEqual(result.password, 'pass') + + + class TestSimpleSerializer(unittest.TestCase): def _makeOne(self): from pyramid.authentication import _SimpleSerializer diff --git a/pyramid/tests/test_config/test_init.py b/pyramid/tests/test_config/test_init.py index de199d079..7078d7e26 100644 --- a/pyramid/tests/test_config/test_init.py +++ b/pyramid/tests/test_config/test_init.py @@ -1545,6 +1545,31 @@ class TestActionState(unittest.TestCase): c.execute_actions() self.assertEqual(output, [('f', (1,), {}), ('g', (8,), {})]) + def test_reentrant_action_with_deferred_discriminator(self): + # see https://github.com/Pylons/pyramid/issues/2697 + from pyramid.registry import Deferred + output = [] + c = self._makeOne() + def f(*a, **k): + output.append(('f', a, k)) + c.actions.append((4, g, (4,), {}, (), None, 2)) + def g(*a, **k): + output.append(('g', a, k)) + def h(*a, **k): + output.append(('h', a, k)) + def discrim(): + self.assertEqual(output, [('f', (1,), {}), ('g', (2,), {})]) + return 3 + d = Deferred(discrim) + c.actions = [ + (d, h, (3,), {}, (), None, 1), # order 1 + (1, f, (1,)), # order 0 + (2, g, (2,)), # order 0 + ] + c.execute_actions() + self.assertEqual(output, [ + ('f', (1,), {}), ('g', (2,), {}), ('h', (3,), {}), ('g', (4,), {})]) + def test_reentrant_action_error(self): from pyramid.exceptions import ConfigurationError c = self._makeOne() @@ -1570,6 +1595,28 @@ class TestActionState(unittest.TestCase): (3, g, (8,)), ]) + def test_executing_conflicting_action_across_orders(self): + from pyramid.exceptions import ConfigurationConflictError + c = self._makeOne() + def f(*a, **k): pass + def g(*a, **k): pass + c.actions = [ + (1, f, (1,), {}, (), None, -1), + (1, g, (2,)), + ] + self.assertRaises(ConfigurationConflictError, c.execute_actions) + + def test_executing_conflicting_action_across_reentrant_orders(self): + from pyramid.exceptions import ConfigurationConflictError + c = self._makeOne() + def f(*a, **k): + c.actions.append((1, g, (8,))) + def g(*a, **k): pass + c.actions = [ + (1, f, (1,), {}, (), None, -1), + ] + self.assertRaises(ConfigurationConflictError, c.execute_actions) + class Test_reentrant_action_functional(unittest.TestCase): def _makeConfigurator(self, *arg, **kw): from pyramid.config import Configurator @@ -1597,6 +1644,21 @@ class Test_reentrant_action_functional(unittest.TestCase): self.assertEqual(route.name, 'foo') self.assertEqual(route.path, '/foo') + def test_deferred_discriminator(self): + # see https://github.com/Pylons/pyramid/issues/2697 + from pyramid.config import PHASE0_CONFIG + config = self._makeConfigurator() + def deriver(view, info): return view + deriver.options = ('foo',) + config.add_view_deriver(deriver, 'foo_view') + # add_view uses a deferred discriminator and will fail if executed + # prior to add_view_deriver executing its action + config.add_view(lambda r: r.response, name='', foo=1) + def dummy_action(): + # trigger a re-entrant action + config.action(None, lambda: None) + config.action(None, dummy_action, order=PHASE0_CONFIG) + config.commit() class Test_resolveConflicts(unittest.TestCase): def _callFUT(self, actions): @@ -1666,15 +1728,14 @@ class Test_resolveConflicts(unittest.TestCase): def test_it_success_dicts(self): from pyramid.tests.test_config import dummyfactory as f - from pyramid.config import expand_action result = self._callFUT([ - expand_action(None, f), - expand_action(1, f, (1,), {}, (), 'first'), - expand_action(1, f, (2,), {}, ('x',), 'second'), - expand_action(1, f, (3,), {}, ('y',), 'third'), - expand_action(4, f, (4,), {}, ('y',), 'should be last', 99999), - expand_action(3, f, (3,), {}, ('y',)), - expand_action(None, f, (5,), {}, ('y',)), + (None, f), + (1, f, (1,), {}, (), 'first'), + (1, f, (2,), {}, ('x',), 'second'), + (1, f, (3,), {}, ('y',), 'third'), + (4, f, (4,), {}, ('y',), 'should be last', 99999), + (3, f, (3,), {}, ('y',)), + (None, f, (5,), {}, ('y',)), ]) result = list(result) self.assertEqual( @@ -1740,17 +1801,16 @@ class Test_resolveConflicts(unittest.TestCase): def test_it_with_actions_grouped_by_order(self): from pyramid.tests.test_config import dummyfactory as f - from pyramid.config import expand_action result = self._callFUT([ - expand_action(None, f), # X - expand_action(1, f, (1,), {}, (), 'third', 10), # X - expand_action(1, f, (2,), {}, ('x',), 'fourth', 10), - expand_action(1, f, (3,), {}, ('y',), 'fifth', 10), - expand_action(2, f, (1,), {}, (), 'sixth', 10), # X - expand_action(3, f, (1,), {}, (), 'seventh', 10), # X - expand_action(5, f, (4,), {}, ('y',), 'eighth', 99999), # X - expand_action(4, f, (3,), {}, (), 'first', 5), # X - expand_action(4, f, (5,), {}, ('y',), 'second', 5), + (None, f), # X + (1, f, (1,), {}, (), 'third', 10), # X + (1, f, (2,), {}, ('x',), 'fourth', 10), + (1, f, (3,), {}, ('y',), 'fifth', 10), + (2, f, (1,), {}, (), 'sixth', 10), # X + (3, f, (1,), {}, (), 'seventh', 10), # X + (5, f, (4,), {}, ('y',), 'eighth', 99999), # X + (4, f, (3,), {}, (), 'first', 5), # X + (4, f, (5,), {}, ('y',), 'second', 5), ]) result = list(result) self.assertEqual(len(result), 6) @@ -1812,7 +1872,32 @@ class Test_resolveConflicts(unittest.TestCase): 'order': 99999} ] ) - + + def test_override_success_across_orders(self): + from pyramid.tests.test_config import dummyfactory as f + result = self._callFUT([ + (1, f, (2,), {}, ('x',), 'eek', 0), + (1, f, (3,), {}, ('x', 'y'), 'ack', 10), + ]) + result = list(result) + self.assertEqual(result, [ + {'info': 'eek', + 'args': (2,), + 'callable': f, + 'introspectables': (), + 'kw': {}, + 'discriminator': 1, + 'includepath': ('x',), + 'order': 0}, + ]) + + def test_conflicts_across_orders(self): + from pyramid.tests.test_config import dummyfactory as f + result = self._callFUT([ + (1, f, (2,), {}, ('x', 'y'), 'eek', 0), + (1, f, (3,), {}, ('x'), 'ack', 10), + ]) + self.assertRaises(ConfigurationConflictError, list, result) class TestGlobalRegistriesIntegration(unittest.TestCase): def setUp(self): diff --git a/pyramid/tests/test_scripting.py b/pyramid/tests/test_scripting.py index 1e952062b..00f738e02 100644 --- a/pyramid/tests/test_scripting.py +++ b/pyramid/tests/test_scripting.py @@ -134,6 +134,27 @@ class Test_prepare(unittest.TestCase): root, closer = info['root'], info['closer'] closer() + def test_it_is_a_context_manager(self): + request = DummyRequest({}) + registry = request.registry = self._makeRegistry() + closer_called = [False] + with self._callFUT(request=request) as info: + root, request = info['root'], info['request'] + pushed = self.manager.get() + self.assertEqual(pushed['request'], request) + self.assertEqual(pushed['registry'], registry) + self.assertEqual(pushed['request'].registry, registry) + self.assertEqual(root.a, (request,)) + orig_closer = info['closer'] + def closer(): + orig_closer() + closer_called[0] = True + info['closer'] = closer + self.assertTrue(closer_called[0]) + self.assertEqual(self.default, self.manager.get()) + self.assertEqual(request.context, root) + self.assertEqual(request.registry, registry) + class Test__make_request(unittest.TestCase): def _callFUT(self, path='/', registry=None): from pyramid.scripting import _make_request @@ -18,16 +18,15 @@ import sys from setuptools import setup, find_packages py_version = sys.version_info[:2] -is_pypy = '__pypy__' in sys.builtin_module_names PY3 = py_version[0] == 3 if PY3: - if py_version < (3, 3) and not is_pypy: # PyPy3 masquerades as Python 3.2... - raise RuntimeError('On Python 3, Pyramid requires Python 3.3 or better') + if py_version < (3, 4): + raise RuntimeError('On Python 3, Pyramid requires Python 3.4 or better') else: - if py_version < (2, 6): - raise RuntimeError('On Python 2, Pyramid requires Python 2.6 or better') + if py_version < (2, 7): + raise RuntimeError('On Python 2, Pyramid requires Python 2.7 or better') here = os.path.abspath(os.path.dirname(__file__)) try: @@ -81,7 +80,6 @@ setup(name='pyramid', "Programming Language :: Python", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: Implementation :: CPython", @@ -1,17 +1,18 @@ [tox] envlist = - py27,py33,py34,py35,pypy, + py27,py34,py35,py36,pypy, docs,pep8, {py2,py3}-cover,coverage, +skip-missing-interpreters = True [testenv] # Most of these are defaults but if you specify any you can't fall back # to defaults for others. basepython = py27: python2.7 - py33: python3.3 py34: python3.4 py35: python3.5 + py36: python3.6 pypy: pypy py2: python2.7 py3: python3.5 @@ -26,12 +27,6 @@ commands = python pyramid/scaffolds/tests.py deps = virtualenv -[testenv:py33-scaffolds] -basepython = python3.3 -commands = - python pyramid/scaffolds/tests.py -deps = virtualenv - [testenv:py34-scaffolds] basepython = python3.4 commands = |