add minimal pserve test

2 files changed, 303 insertions, 118 deletions

diff --git a/pyramid/scripts/pserve.py b/pyramid/scripts/pserve.py
index 561085119..b9d2d944f 100644
--- a/pyramid/scripts/pserve.py
+++ b/pyramid/scripts/pserve.py
@@ -8,16 +8,18 @@
 # Code taken also from QP: http://www.mems-exchange.org/software/qp/ From
 # lib/site.py
 
-import re
+import atexit
+import errno
+import logging
 import optparse
 import os
-import errno
-import sys
-import time
+import re
 import subprocess
+import sys
 import threading
-import atexit
-import logging
+import time
+import traceback
+
 from logging.config import fileConfig
 
 from pyramid.compat import configparser
@@ -26,16 +28,14 @@ from paste.deploy import loadapp, loadserver
 
 MAXFD = 1024
 
-jython = sys.platform.startswith('java')
-
 def main(argv=sys.argv):
-    command = ServeCommand(argv)
+    command = PServeCommand(argv)
     return command.run()
 
 class DaemonizeException(Exception):
     pass
 
-class ServeCommand(object):
+class PServeCommand(object):
 
     min_args = 0
     usage = 'CONFIG_FILE [start|stop|restart|status] [var=value]'
@@ -71,7 +71,8 @@ class ServeCommand(object):
         '--server-name',
         dest='server_name',
         metavar='SECTION_NAME',
-        help="Use the named server as defined in the configuration file (default: main)")
+        help=("Use the named server as defined in the configuration file "
+              "(default: main)"))
     if hasattr(os, 'fork'):
         parser.add_option(
             '--daemon',
@@ -82,7 +83,8 @@ class ServeCommand(object):
         '--pid-file',
         dest='pid_file',
         metavar='FILENAME',
-        help="Save PID to file (default to pyramid.pid if running in daemon mode)")
+        help=("Save PID to file (default to pyramid.pid if running in "
+              "daemon mode)"))
     parser.add_option(
         '--log-file',
         dest='log_file',
@@ -97,7 +99,8 @@ class ServeCommand(object):
         '--reload-interval',
         dest='reload_interval',
         default=1,
-        help="Seconds between checking files (low number can cause significant CPU usage)")
+        help=("Seconds between checking files (low number can cause "
+              "significant CPU usage)"))
     parser.add_option(
         '--monitor-restart',
         dest='monitor_restart',
@@ -108,6 +111,11 @@ class ServeCommand(object):
         action='store_true',
         dest='show_status',
         help="Show the status of the (presumably daemonized) server")
+    parser.add_option(
+        '-q', '--quiet',
+        action='store_true',
+        dest='quiet',
+        help='Produce little or no output')
 
     if hasattr(os, 'setuid'):
         # I don't think these are available on Windows
@@ -126,14 +134,8 @@ class ServeCommand(object):
         '--stop-daemon',
         dest='stop_daemon',
         action='store_true',
-        help='Stop a daemonized server (given a PID file, or default pyramid.pid file)')
-
-    if jython:
-        parser.add_option(
-            '--disable-jython-reloader',
-            action='store_true',
-            dest='disable_jython_reloader',
-            help="Disable the Jython reloader")
+        help=('Stop a daemonized server (given a PID file, or default '
+              'pyramid.pid file)'))
 
     _scheme_re = re.compile(r'^[a-z][a-z]+:', re.I)
 
@@ -147,6 +149,10 @@ class ServeCommand(object):
     def __init__(self, argv):
         self.options, self.args = self.parser.parse_args(argv[1:])
 
+    def out(self, msg):
+        if not self.options.quiet:
+            print(msg)
+
     def run(self):
         if self.options.stop_daemon:
             return self.stop_daemon()
@@ -154,13 +160,15 @@ class ServeCommand(object):
         if not hasattr(self.options, 'set_user'):
             # Windows case:
             self.options.set_user = self.options.set_group = None
+
         # @@: Is this the right stage to set the user at?
         self.change_user_group(
             self.options.set_user, self.options.set_group)
 
         if self.requires_config_file:
             if not self.args:
-                raise ValueError('You must give a config file')
+                self.out('You must give a config file')
+                return
             app_spec = self.args[0]
             if (len(self.args) > 1
                 and self.args[1] in self.possible_subcommands):
@@ -179,37 +187,20 @@ class ServeCommand(object):
                 cmd = None
                 restvars = self.args[:]
 
-        jython_monitor = False
         if self.options.reload:
-            if jython and not self.options.disable_jython_reloader:
-                # JythonMonitor raises the special SystemRestart
-                # exception that'll cause the Jython interpreter to
-                # reload in the existing Java process (avoiding
-                # subprocess startup time)
-                try:
-                    from paste.reloader import JythonMonitor
-                except ImportError:
-                    pass
-                else:
-                    jython_monitor = JythonMonitor(poll_interval=int(
-                            self.options.reload_interval))
-                    if self.requires_config_file:
-                        jython_monitor.watch_file(self.args[0])
-
-            if not jython_monitor:
-                if os.environ.get(self._reloader_environ_key):
-                    from paste import reloader
-                    if self.verbose > 1:
-                        print('Running reloading file monitor')
-                    reloader.install(int(self.options.reload_interval))
-                    if self.requires_config_file:
-                        reloader.watch_file(self.args[0])
-                else:
-                    return self.restart_with_reloader()
+            if os.environ.get(self._reloader_environ_key):
+                if self.verbose > 1:
+                    self.out('Running reloading file monitor')
+                install_reloader(int(self.options.reload_interval))
+                if self.requires_config_file:
+                    watch_file(self.args[0])
+            else:
+                return self.restart_with_reloader()
 
         if cmd not in (None, 'start', 'stop', 'restart', 'status'):
-            raise ValueError(
+            self.out(
                 'Error: must give start|stop|restart (not %s)' % cmd)
+            return
 
         if cmd == 'status' or self.options.show_status:
             return self.show_status()
@@ -218,9 +209,9 @@ class ServeCommand(object):
             result = self.stop_daemon()
             if result:
                 if cmd == 'restart':
-                    print("Could not stop daemon; aborting")
+                    self.out("Could not stop daemon; aborting")
                 else:
-                    print("Could not stop daemon")
+                    self.out("Could not stop daemon")
                 return result
             if cmd == 'stop':
                 return result
@@ -271,7 +262,7 @@ class ServeCommand(object):
                 self.daemonize()
             except DaemonizeException as ex:
                 if self.verbose > 0:
-                    print(str(ex))
+                    self.out(str(ex))
                 return
 
         if (self.options.monitor_restart
@@ -306,7 +297,7 @@ class ServeCommand(object):
                 msg = 'Starting server in PID %i.' % os.getpid()
             else:
                 msg = 'Starting server.'
-            print(msg)
+            self.out(msg)
 
         def serve():
             try:
@@ -318,15 +309,9 @@ class ServeCommand(object):
                     msg = ' '+str(e)
                 else:
                     msg = ''
-                print('Exiting%s (-v to see traceback)' % msg)
+                self.out('Exiting%s (-v to see traceback)' % msg)
 
-        if jython_monitor:
-            # JythonMonitor has to be ran from the main thread
-            threading.Thread(target=serve).start()
-            print('Starting Jython file monitor')
-            jython_monitor.periodic_reload()
-        else:
-            serve()
+        serve()
 
     def loadserver(self, server_spec, name, relative_to, **kw):
             return loadserver(
@@ -396,7 +381,7 @@ class ServeCommand(object):
                 % (pid, self.options.pid_file))
 
         if self.verbose > 0:
-            print('Entering daemon mode')
+            self.out('Entering daemon mode')
         pid = os.fork()
         if pid:
             # The forked process also has a handle on resources, so we
@@ -432,31 +417,69 @@ class ServeCommand(object):
         os.dup2(0, 1)  # standard output (1)
         os.dup2(0, 2)  # standard error (2)
 
+    def _remove_pid_file(self, written_pid, filename, verbosity):
+        current_pid = os.getpid()
+        if written_pid != current_pid:
+            # A forked process must be exiting, not the process that
+            # wrote the PID file
+            return
+        if not os.path.exists(filename):
+            return
+        f = open(filename)
+        content = f.read().strip()
+        f.close()
+        try:
+            pid_in_file = int(content)
+        except ValueError:
+            pass
+        else:
+            if pid_in_file != current_pid:
+                self.out("PID file %s contains %s, not expected PID %s" % (
+                    filename, pid_in_file, current_pid))
+                return
+        if verbosity > 0:
+            self.out("Removing PID file %s" % filename)
+        try:
+            os.unlink(filename)
+            return
+        except OSError as e:
+            # Record, but don't give traceback
+            self.out("Cannot remove PID file: %s" % e)
+        # well, at least lets not leave the invalid PID around...
+        try:
+            f = open(filename, 'w')
+            f.write('')
+            f.close()
+        except OSError as e:
+            self.out('Stale PID left in file: %s (%e)' % (filename, e))
+        else:
+            self.out('Stale PID removed')
+
     def record_pid(self, pid_file):
         pid = os.getpid()
         if self.verbose > 1:
-            print('Writing PID %s to %s' % (pid, pid_file))
+            self.out('Writing PID %s to %s' % (pid, pid_file))
         f = open(pid_file, 'w')
         f.write(str(pid))
         f.close()
-        atexit.register(_remove_pid_file, pid, pid_file, self.verbose)
+        atexit.register(self._remove_pid_file, pid, pid_file, self.verbose)
 
     def stop_daemon(self):
         pid_file = self.options.pid_file or 'pyramid.pid'
         if not os.path.exists(pid_file):
-            print('No PID file exists in %s' % pid_file)
+            self.out('No PID file exists in %s' % pid_file)
             return 1
         pid = read_pidfile(pid_file)
         if not pid:
-            print("Not a valid PID file in %s" % pid_file)
+            self.out("Not a valid PID file in %s" % pid_file)
             return 1
         pid = live_pidfile(pid_file)
         if not pid:
-            print("PID in %s is not valid (deleting)" % pid_file)
+            self.out("PID in %s is not valid (deleting)" % pid_file)
             try:
                 os.unlink(pid_file)
             except (OSError, IOError) as e:
-                print("Could not delete: %s" % e)
+                self.out("Could not delete: %s" % e)
                 return 2
             return 1
         for j in range(10):
@@ -466,7 +489,7 @@ class ServeCommand(object):
             os.kill(pid, signal.SIGTERM)
             time.sleep(1)
         else:
-            print("failed to kill web process %s" % pid)
+            self.out("failed to kill web process %s" % pid)
             return 3
         if os.path.exists(pid_file):
             os.unlink(pid_file)
@@ -475,17 +498,17 @@ class ServeCommand(object):
     def show_status(self):
         pid_file = self.options.pid_file or 'pyramid.pid'
         if not os.path.exists(pid_file):
-            print('No PID file %s' % pid_file)
+            self.out('No PID file %s' % pid_file)
             return 1
         pid = read_pidfile(pid_file)
         if not pid:
-            print('No PID in file %s' % pid_file)
+            self.out('No PID in file %s' % pid_file)
             return 1
         pid = live_pidfile(pid_file)
         if not pid:
-            print('PID %s in %s is not running' % (pid, pid_file))
+            self.out('PID %s in %s is not running' % (pid, pid_file))
             return 1
-        print('Server running in PID %s' % pid)
+        self.out('Server running in PID %s' % pid)
         return 0
 
     def restart_with_reloader(self):
@@ -494,9 +517,9 @@ class ServeCommand(object):
     def restart_with_monitor(self, reloader=False):
         if self.verbose > 0:
             if reloader:
-                print('Starting subprocess with file monitor')
+                self.out('Starting subprocess with file monitor')
             else:
-                print('Starting subprocess with monitor parent')
+                self.out('Starting subprocess with monitor parent')
         while 1:
             args = [self.quote_first_command_arg(sys.executable)] + sys.argv
             new_environ = os.environ.copy()
@@ -512,7 +535,7 @@ class ServeCommand(object):
                     exit_code = proc.wait()
                     proc = None
                 except KeyboardInterrupt:
-                    print('^C caught in monitor process')
+                    self.out('^C caught in monitor process')
                     if self.verbose > 1:
                         raise
                     return 1
@@ -531,7 +554,7 @@ class ServeCommand(object):
                 if exit_code != 3:
                     return exit_code
             if self.verbose > 0:
-                print('-'*20, 'Restarting', '-'*20)
+                self.out('%s %s %s' % ('-'*20, 'Restarting', '-'*20))
 
     def change_user_group(self, user, group):
         if not user and not group:
@@ -563,7 +586,7 @@ class ServeCommand(object):
                 gid = entry.pw_gid
             uid = entry.pw_uid
         if self.verbose > 0:
-            print('Changing user to %s:%s (%s:%s)' % (
+            self.out('Changing user to %s:%s (%s:%s)' % (
                 user, group or '(unknown)', uid, gid))
         if gid:
             os.setgid(gid)
@@ -634,45 +657,6 @@ def read_pidfile(filename):
     else:
         return None
 
-def _remove_pid_file(written_pid, filename, verbosity):
-    current_pid = os.getpid()
-    if written_pid != current_pid:
-        # A forked process must be exiting, not the process that
-        # wrote the PID file
-        return
-    if not os.path.exists(filename):
-        return
-    f = open(filename)
-    content = f.read().strip()
-    f.close()
-    try:
-        pid_in_file = int(content)
-    except ValueError:
-        pass
-    else:
-        if pid_in_file != current_pid:
-            print("PID file %s contains %s, not expected PID %s" % (
-                filename, pid_in_file, current_pid))
-            return
-    if verbosity > 0:
-        print("Removing PID file %s" % filename)
-    try:
-        os.unlink(filename)
-        return
-    except OSError as e:
-        # Record, but don't give traceback
-        print("Cannot remove PID file: %s" % e)
-    # well, at least lets not leave the invalid PID around...
-    try:
-        f = open(filename, 'w')
-        f.write('')
-        f.close()
-    except OSError as e:
-        print('Stale PID left in file: %s (%e)' % (filename, e))
-    else:
-        print('Stale PID removed')
-
-
 def ensure_port_cleanup(bound_addresses, maxtries=30, sleeptime=2):
     """
     This makes sure any open ports are closed.
@@ -680,7 +664,6 @@ def ensure_port_cleanup(bound_addresses, maxtries=30, sleeptime=2):
     Does this by connecting to them until they give connection
     refused.  Servers should call like::
 
-        import paste.script
         ensure_port_cleanup([80, 443])
     """
     atexit.register(_cleanup_ports, bound_addresses, maxtries=maxtries,
@@ -717,6 +700,181 @@ def _turn_sigterm_into_systemexit():
         raise SystemExit
     signal.signal(signal.SIGTERM, handle_term)
 
+def install_reloader(poll_interval=1):
+    """
+    Install the reloading monitor.
+
+    On some platforms server threads may not terminate when the main
+    thread does, causing ports to remain open/locked.  The
+    ``raise_keyboard_interrupt`` option creates a unignorable signal
+    which causes the whole application to shut-down (rudely).
+    """
+    mon = Monitor(poll_interval=poll_interval)
+    t = threading.Thread(target=mon.periodic_reload)
+    t.setDaemon(True)
+    t.start()
+
+class classinstancemethod(object):
+    """
+    Acts like a class method when called from a class, like an
+    instance method when called by an instance.  The method should
+    take two arguments, 'self' and 'cls'; one of these will be None
+    depending on how the method was called.
+    """
+
+    def __init__(self, func):
+        self.func = func
+        self.__doc__ = func.__doc__
+
+    def __get__(self, obj, type=None):
+        return _methodwrapper(self.func, obj=obj, type=type)
+
+class _methodwrapper(object):
+
+    def __init__(self, func, obj, type):
+        self.func = func
+        self.obj = obj
+        self.type = type
+
+    def __call__(self, *args, **kw):
+        assert not kw.has_key('self') and not kw.has_key('cls'), (
+            "You cannot use 'self' or 'cls' arguments to a "
+            "classinstancemethod")
+        return self.func(*((self.obj, self.type) + args), **kw)
+
+    def __repr__(self):
+        if self.obj is None:
+            return ('<bound class method %s.%s>'
+                    % (self.type.__name__, self.func.func_name))
+        else:
+            return ('<bound method %s.%s of %r>'
+                    % (self.type.__name__, self.func.func_name, self.obj))
+
+
+class Monitor(object):
+    """
+    A file monitor and server restarter.
+
+    Use this like:
+
+    ..code-block:: Python
+
+        install_reloader()
+
+    Then make sure your server is installed with a shell script like::
+
+        err=3
+        while test "$err" -eq 3 ; do
+            python server.py
+            err="$?"
+        done
+
+    or is run from this .bat file (if you use Windows)::
+
+        @echo off
+        :repeat
+            python server.py
+        if %errorlevel% == 3 goto repeat
+
+    or run a monitoring process in Python (``pserve --reload`` does
+    this).  
+
+    Use the ``watch_file(filename)`` function to cause a reload/restart for
+    other other non-Python files (e.g., configuration files).  If you have
+    a dynamic set of files that grows over time you can use something like::
+
+        def watch_config_files():
+            return CONFIG_FILE_CACHE.keys()
+        add_file_callback(watch_config_files)
+
+    Then every time the reloader polls files it will call
+    ``watch_config_files`` and check all the filenames it returns.
+    """
+    instances = []
+    global_extra_files = []
+    global_file_callbacks = []
+
+    def __init__(self, poll_interval):
+        self.module_mtimes = {}
+        self.keep_running = True
+        self.poll_interval = poll_interval
+        self.extra_files = list(self.global_extra_files)
+        self.instances.append(self)
+        self.file_callbacks = list(self.global_file_callbacks)
+
+    def periodic_reload(self):
+        while True:
+            if not self.check_reload():
+                # use os._exit() here and not sys.exit() since within a
+                # thread sys.exit() just closes the given thread and
+                # won't kill the process; note os._exit does not call
+                # any atexit callbacks, nor does it do finally blocks,
+                # flush open files, etc.  In otherwords, it is rude.
+                os._exit(3)
+                break
+            time.sleep(self.poll_interval)
+
+    def check_reload(self):
+        filenames = list(self.extra_files)
+        for file_callback in self.file_callbacks:
+            try:
+                filenames.extend(file_callback())
+            except:
+                print(
+                    "Error calling reloader callback %r:" % file_callback)
+                traceback.print_exc()
+        for module in sys.modules.values():
+            try:
+                filename = module.__file__
+            except (AttributeError, ImportError):
+                continue
+            if filename is not None:
+                filenames.append(filename)
+        for filename in filenames:
+            try:
+                stat = os.stat(filename)
+                if stat:
+                    mtime = stat.st_mtime
+                else:
+                    mtime = 0
+            except (OSError, IOError):
+                continue
+            if filename.endswith('.pyc') and os.path.exists(filename[:-1]):
+                mtime = max(os.stat(filename[:-1]).st_mtime, mtime)
+            if not self.module_mtimes.has_key(filename):
+                self.module_mtimes[filename] = mtime
+            elif self.module_mtimes[filename] < mtime:
+                print("%s changed; reloading..." % filename)
+                return False
+        return True
+
+    def watch_file(self, cls, filename):
+        """Watch the named file for changes"""
+        filename = os.path.abspath(filename)
+        if self is None:
+            for instance in cls.instances:
+                instance.watch_file(filename)
+            cls.global_extra_files.append(filename)
+        else:
+            self.extra_files.append(filename)
+
+    watch_file = classinstancemethod(watch_file)
+
+    def add_file_callback(self, cls, callback):
+        """Add a callback -- a function that takes no parameters -- that will
+        return a list of filenames to watch for changes."""
+        if self is None:
+            for instance in cls.instances:
+                instance.add_file_callback(callback)
+            cls.global_file_callbacks.append(callback)
+        else:
+            self.file_callbacks.append(callback)
+
+    add_file_callback = classinstancemethod(add_file_callback)
+
+watch_file = Monitor.watch_file
+add_file_callback = Monitor.add_file_callback
+
 # For paste.deploy server instantiation (egg:pyramid#wsgiref)
 def wsgiref_server_runner(wsgi_app, global_conf, **kw):
     from wsgiref.simple_server import make_server
diff --git a/pyramid/tests/test_scripts/test_pserve.py b/pyramid/tests/test_scripts/test_pserve.py
new file mode 100644
index 000000000..bc2e9aac8
--- /dev/null
+++ b/pyramid/tests/test_scripts/test_pserve.py
@@ -0,0 +1,27 @@
+import unittest
+
+class TestPServeCommand(unittest.TestCase):
+    def setUp(self):
+        from pyramid.compat import NativeIO
+        self.out_ = NativeIO()
+
+    def out(self, msg):
+        self.out_.write(msg)
+        
+    def _getTargetClass(self):
+        from pyramid.scripts.pserve import PServeCommand
+        return PServeCommand
+
+    def _makeOne(self, *args):
+        effargs = ['pserve']
+        effargs.extend(args)
+        cmd = self._getTargetClass()(effargs)
+        cmd.out = self.out
+        return cmd
+
+    def test_no_args(self):
+        inst = self._makeOne()
+        result = inst.run()
+        self.assertEqual(result, None)
+        self.assertEqual(self.out_.getvalue(), 'You must give a config file')
+