From 2e6d63ce873dd866f93215310c633a6d75dc0e13 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Thu, 6 Oct 2011 01:01:52 -0400 Subject: add minimal pserve test --- pyramid/scripts/pserve.py | 394 +++++++++++++++++++++--------- pyramid/tests/test_scripts/test_pserve.py | 27 ++ 2 files changed, 303 insertions(+), 118 deletions(-) create mode 100644 pyramid/tests/test_scripts/test_pserve.py diff --git a/pyramid/scripts/pserve.py b/pyramid/scripts/pserve.py index 561085119..b9d2d944f 100644 --- a/pyramid/scripts/pserve.py +++ b/pyramid/scripts/pserve.py @@ -8,16 +8,18 @@ # Code taken also from QP: http://www.mems-exchange.org/software/qp/ From # lib/site.py -import re +import atexit +import errno +import logging import optparse import os -import errno -import sys -import time +import re import subprocess +import sys import threading -import atexit -import logging +import time +import traceback + from logging.config import fileConfig from pyramid.compat import configparser @@ -26,16 +28,14 @@ from paste.deploy import loadapp, loadserver MAXFD = 1024 -jython = sys.platform.startswith('java') - def main(argv=sys.argv): - command = ServeCommand(argv) + command = PServeCommand(argv) return command.run() class DaemonizeException(Exception): pass -class ServeCommand(object): +class PServeCommand(object): min_args = 0 usage = 'CONFIG_FILE [start|stop|restart|status] [var=value]' @@ -71,7 +71,8 @@ class ServeCommand(object): '--server-name', dest='server_name', metavar='SECTION_NAME', - help="Use the named server as defined in the configuration file (default: main)") + help=("Use the named server as defined in the configuration file " + "(default: main)")) if hasattr(os, 'fork'): parser.add_option( '--daemon', @@ -82,7 +83,8 @@ class ServeCommand(object): '--pid-file', dest='pid_file', metavar='FILENAME', - help="Save PID to file (default to pyramid.pid if running in daemon mode)") + help=("Save PID to file (default to pyramid.pid if running in " + "daemon mode)")) parser.add_option( '--log-file', dest='log_file', @@ -97,7 +99,8 @@ class ServeCommand(object): '--reload-interval', dest='reload_interval', default=1, - help="Seconds between checking files (low number can cause significant CPU usage)") + help=("Seconds between checking files (low number can cause " + "significant CPU usage)")) parser.add_option( '--monitor-restart', dest='monitor_restart', @@ -108,6 +111,11 @@ class ServeCommand(object): action='store_true', dest='show_status', help="Show the status of the (presumably daemonized) server") + parser.add_option( + '-q', '--quiet', + action='store_true', + dest='quiet', + help='Produce little or no output') if hasattr(os, 'setuid'): # I don't think these are available on Windows @@ -126,14 +134,8 @@ class ServeCommand(object): '--stop-daemon', dest='stop_daemon', action='store_true', - help='Stop a daemonized server (given a PID file, or default pyramid.pid file)') - - if jython: - parser.add_option( - '--disable-jython-reloader', - action='store_true', - dest='disable_jython_reloader', - help="Disable the Jython reloader") + help=('Stop a daemonized server (given a PID file, or default ' + 'pyramid.pid file)')) _scheme_re = re.compile(r'^[a-z][a-z]+:', re.I) @@ -147,6 +149,10 @@ class ServeCommand(object): def __init__(self, argv): self.options, self.args = self.parser.parse_args(argv[1:]) + def out(self, msg): + if not self.options.quiet: + print(msg) + def run(self): if self.options.stop_daemon: return self.stop_daemon() @@ -154,13 +160,15 @@ class ServeCommand(object): if not hasattr(self.options, 'set_user'): # Windows case: self.options.set_user = self.options.set_group = None + # @@: Is this the right stage to set the user at? self.change_user_group( self.options.set_user, self.options.set_group) if self.requires_config_file: if not self.args: - raise ValueError('You must give a config file') + self.out('You must give a config file') + return app_spec = self.args[0] if (len(self.args) > 1 and self.args[1] in self.possible_subcommands): @@ -179,37 +187,20 @@ class ServeCommand(object): cmd = None restvars = self.args[:] - jython_monitor = False if self.options.reload: - if jython and not self.options.disable_jython_reloader: - # JythonMonitor raises the special SystemRestart - # exception that'll cause the Jython interpreter to - # reload in the existing Java process (avoiding - # subprocess startup time) - try: - from paste.reloader import JythonMonitor - except ImportError: - pass - else: - jython_monitor = JythonMonitor(poll_interval=int( - self.options.reload_interval)) - if self.requires_config_file: - jython_monitor.watch_file(self.args[0]) - - if not jython_monitor: - if os.environ.get(self._reloader_environ_key): - from paste import reloader - if self.verbose > 1: - print('Running reloading file monitor') - reloader.install(int(self.options.reload_interval)) - if self.requires_config_file: - reloader.watch_file(self.args[0]) - else: - return self.restart_with_reloader() + if os.environ.get(self._reloader_environ_key): + if self.verbose > 1: + self.out('Running reloading file monitor') + install_reloader(int(self.options.reload_interval)) + if self.requires_config_file: + watch_file(self.args[0]) + else: + return self.restart_with_reloader() if cmd not in (None, 'start', 'stop', 'restart', 'status'): - raise ValueError( + self.out( 'Error: must give start|stop|restart (not %s)' % cmd) + return if cmd == 'status' or self.options.show_status: return self.show_status() @@ -218,9 +209,9 @@ class ServeCommand(object): result = self.stop_daemon() if result: if cmd == 'restart': - print("Could not stop daemon; aborting") + self.out("Could not stop daemon; aborting") else: - print("Could not stop daemon") + self.out("Could not stop daemon") return result if cmd == 'stop': return result @@ -271,7 +262,7 @@ class ServeCommand(object): self.daemonize() except DaemonizeException as ex: if self.verbose > 0: - print(str(ex)) + self.out(str(ex)) return if (self.options.monitor_restart @@ -306,7 +297,7 @@ class ServeCommand(object): msg = 'Starting server in PID %i.' % os.getpid() else: msg = 'Starting server.' - print(msg) + self.out(msg) def serve(): try: @@ -318,15 +309,9 @@ class ServeCommand(object): msg = ' '+str(e) else: msg = '' - print('Exiting%s (-v to see traceback)' % msg) + self.out('Exiting%s (-v to see traceback)' % msg) - if jython_monitor: - # JythonMonitor has to be ran from the main thread - threading.Thread(target=serve).start() - print('Starting Jython file monitor') - jython_monitor.periodic_reload() - else: - serve() + serve() def loadserver(self, server_spec, name, relative_to, **kw): return loadserver( @@ -396,7 +381,7 @@ class ServeCommand(object): % (pid, self.options.pid_file)) if self.verbose > 0: - print('Entering daemon mode') + self.out('Entering daemon mode') pid = os.fork() if pid: # The forked process also has a handle on resources, so we @@ -432,31 +417,69 @@ class ServeCommand(object): os.dup2(0, 1) # standard output (1) os.dup2(0, 2) # standard error (2) + def _remove_pid_file(self, written_pid, filename, verbosity): + current_pid = os.getpid() + if written_pid != current_pid: + # A forked process must be exiting, not the process that + # wrote the PID file + return + if not os.path.exists(filename): + return + f = open(filename) + content = f.read().strip() + f.close() + try: + pid_in_file = int(content) + except ValueError: + pass + else: + if pid_in_file != current_pid: + self.out("PID file %s contains %s, not expected PID %s" % ( + filename, pid_in_file, current_pid)) + return + if verbosity > 0: + self.out("Removing PID file %s" % filename) + try: + os.unlink(filename) + return + except OSError as e: + # Record, but don't give traceback + self.out("Cannot remove PID file: %s" % e) + # well, at least lets not leave the invalid PID around... + try: + f = open(filename, 'w') + f.write('') + f.close() + except OSError as e: + self.out('Stale PID left in file: %s (%e)' % (filename, e)) + else: + self.out('Stale PID removed') + def record_pid(self, pid_file): pid = os.getpid() if self.verbose > 1: - print('Writing PID %s to %s' % (pid, pid_file)) + self.out('Writing PID %s to %s' % (pid, pid_file)) f = open(pid_file, 'w') f.write(str(pid)) f.close() - atexit.register(_remove_pid_file, pid, pid_file, self.verbose) + atexit.register(self._remove_pid_file, pid, pid_file, self.verbose) def stop_daemon(self): pid_file = self.options.pid_file or 'pyramid.pid' if not os.path.exists(pid_file): - print('No PID file exists in %s' % pid_file) + self.out('No PID file exists in %s' % pid_file) return 1 pid = read_pidfile(pid_file) if not pid: - print("Not a valid PID file in %s" % pid_file) + self.out("Not a valid PID file in %s" % pid_file) return 1 pid = live_pidfile(pid_file) if not pid: - print("PID in %s is not valid (deleting)" % pid_file) + self.out("PID in %s is not valid (deleting)" % pid_file) try: os.unlink(pid_file) except (OSError, IOError) as e: - print("Could not delete: %s" % e) + self.out("Could not delete: %s" % e) return 2 return 1 for j in range(10): @@ -466,7 +489,7 @@ class ServeCommand(object): os.kill(pid, signal.SIGTERM) time.sleep(1) else: - print("failed to kill web process %s" % pid) + self.out("failed to kill web process %s" % pid) return 3 if os.path.exists(pid_file): os.unlink(pid_file) @@ -475,17 +498,17 @@ class ServeCommand(object): def show_status(self): pid_file = self.options.pid_file or 'pyramid.pid' if not os.path.exists(pid_file): - print('No PID file %s' % pid_file) + self.out('No PID file %s' % pid_file) return 1 pid = read_pidfile(pid_file) if not pid: - print('No PID in file %s' % pid_file) + self.out('No PID in file %s' % pid_file) return 1 pid = live_pidfile(pid_file) if not pid: - print('PID %s in %s is not running' % (pid, pid_file)) + self.out('PID %s in %s is not running' % (pid, pid_file)) return 1 - print('Server running in PID %s' % pid) + self.out('Server running in PID %s' % pid) return 0 def restart_with_reloader(self): @@ -494,9 +517,9 @@ class ServeCommand(object): def restart_with_monitor(self, reloader=False): if self.verbose > 0: if reloader: - print('Starting subprocess with file monitor') + self.out('Starting subprocess with file monitor') else: - print('Starting subprocess with monitor parent') + self.out('Starting subprocess with monitor parent') while 1: args = [self.quote_first_command_arg(sys.executable)] + sys.argv new_environ = os.environ.copy() @@ -512,7 +535,7 @@ class ServeCommand(object): exit_code = proc.wait() proc = None except KeyboardInterrupt: - print('^C caught in monitor process') + self.out('^C caught in monitor process') if self.verbose > 1: raise return 1 @@ -531,7 +554,7 @@ class ServeCommand(object): if exit_code != 3: return exit_code if self.verbose > 0: - print('-'*20, 'Restarting', '-'*20) + self.out('%s %s %s' % ('-'*20, 'Restarting', '-'*20)) def change_user_group(self, user, group): if not user and not group: @@ -563,7 +586,7 @@ class ServeCommand(object): gid = entry.pw_gid uid = entry.pw_uid if self.verbose > 0: - print('Changing user to %s:%s (%s:%s)' % ( + self.out('Changing user to %s:%s (%s:%s)' % ( user, group or '(unknown)', uid, gid)) if gid: os.setgid(gid) @@ -634,45 +657,6 @@ def read_pidfile(filename): else: return None -def _remove_pid_file(written_pid, filename, verbosity): - current_pid = os.getpid() - if written_pid != current_pid: - # A forked process must be exiting, not the process that - # wrote the PID file - return - if not os.path.exists(filename): - return - f = open(filename) - content = f.read().strip() - f.close() - try: - pid_in_file = int(content) - except ValueError: - pass - else: - if pid_in_file != current_pid: - print("PID file %s contains %s, not expected PID %s" % ( - filename, pid_in_file, current_pid)) - return - if verbosity > 0: - print("Removing PID file %s" % filename) - try: - os.unlink(filename) - return - except OSError as e: - # Record, but don't give traceback - print("Cannot remove PID file: %s" % e) - # well, at least lets not leave the invalid PID around... - try: - f = open(filename, 'w') - f.write('') - f.close() - except OSError as e: - print('Stale PID left in file: %s (%e)' % (filename, e)) - else: - print('Stale PID removed') - - def ensure_port_cleanup(bound_addresses, maxtries=30, sleeptime=2): """ This makes sure any open ports are closed. @@ -680,7 +664,6 @@ def ensure_port_cleanup(bound_addresses, maxtries=30, sleeptime=2): Does this by connecting to them until they give connection refused. Servers should call like:: - import paste.script ensure_port_cleanup([80, 443]) """ atexit.register(_cleanup_ports, bound_addresses, maxtries=maxtries, @@ -717,6 +700,181 @@ def _turn_sigterm_into_systemexit(): raise SystemExit signal.signal(signal.SIGTERM, handle_term) +def install_reloader(poll_interval=1): + """ + Install the reloading monitor. + + On some platforms server threads may not terminate when the main + thread does, causing ports to remain open/locked. The + ``raise_keyboard_interrupt`` option creates a unignorable signal + which causes the whole application to shut-down (rudely). + """ + mon = Monitor(poll_interval=poll_interval) + t = threading.Thread(target=mon.periodic_reload) + t.setDaemon(True) + t.start() + +class classinstancemethod(object): + """ + Acts like a class method when called from a class, like an + instance method when called by an instance. The method should + take two arguments, 'self' and 'cls'; one of these will be None + depending on how the method was called. + """ + + def __init__(self, func): + self.func = func + self.__doc__ = func.__doc__ + + def __get__(self, obj, type=None): + return _methodwrapper(self.func, obj=obj, type=type) + +class _methodwrapper(object): + + def __init__(self, func, obj, type): + self.func = func + self.obj = obj + self.type = type + + def __call__(self, *args, **kw): + assert not kw.has_key('self') and not kw.has_key('cls'), ( + "You cannot use 'self' or 'cls' arguments to a " + "classinstancemethod") + return self.func(*((self.obj, self.type) + args), **kw) + + def __repr__(self): + if self.obj is None: + return ('' + % (self.type.__name__, self.func.func_name)) + else: + return ('' + % (self.type.__name__, self.func.func_name, self.obj)) + + +class Monitor(object): + """ + A file monitor and server restarter. + + Use this like: + + ..code-block:: Python + + install_reloader() + + Then make sure your server is installed with a shell script like:: + + err=3 + while test "$err" -eq 3 ; do + python server.py + err="$?" + done + + or is run from this .bat file (if you use Windows):: + + @echo off + :repeat + python server.py + if %errorlevel% == 3 goto repeat + + or run a monitoring process in Python (``pserve --reload`` does + this). + + Use the ``watch_file(filename)`` function to cause a reload/restart for + other other non-Python files (e.g., configuration files). If you have + a dynamic set of files that grows over time you can use something like:: + + def watch_config_files(): + return CONFIG_FILE_CACHE.keys() + add_file_callback(watch_config_files) + + Then every time the reloader polls files it will call + ``watch_config_files`` and check all the filenames it returns. + """ + instances = [] + global_extra_files = [] + global_file_callbacks = [] + + def __init__(self, poll_interval): + self.module_mtimes = {} + self.keep_running = True + self.poll_interval = poll_interval + self.extra_files = list(self.global_extra_files) + self.instances.append(self) + self.file_callbacks = list(self.global_file_callbacks) + + def periodic_reload(self): + while True: + if not self.check_reload(): + # use os._exit() here and not sys.exit() since within a + # thread sys.exit() just closes the given thread and + # won't kill the process; note os._exit does not call + # any atexit callbacks, nor does it do finally blocks, + # flush open files, etc. In otherwords, it is rude. + os._exit(3) + break + time.sleep(self.poll_interval) + + def check_reload(self): + filenames = list(self.extra_files) + for file_callback in self.file_callbacks: + try: + filenames.extend(file_callback()) + except: + print( + "Error calling reloader callback %r:" % file_callback) + traceback.print_exc() + for module in sys.modules.values(): + try: + filename = module.__file__ + except (AttributeError, ImportError): + continue + if filename is not None: + filenames.append(filename) + for filename in filenames: + try: + stat = os.stat(filename) + if stat: + mtime = stat.st_mtime + else: + mtime = 0 + except (OSError, IOError): + continue + if filename.endswith('.pyc') and os.path.exists(filename[:-1]): + mtime = max(os.stat(filename[:-1]).st_mtime, mtime) + if not self.module_mtimes.has_key(filename): + self.module_mtimes[filename] = mtime + elif self.module_mtimes[filename] < mtime: + print("%s changed; reloading..." % filename) + return False + return True + + def watch_file(self, cls, filename): + """Watch the named file for changes""" + filename = os.path.abspath(filename) + if self is None: + for instance in cls.instances: + instance.watch_file(filename) + cls.global_extra_files.append(filename) + else: + self.extra_files.append(filename) + + watch_file = classinstancemethod(watch_file) + + def add_file_callback(self, cls, callback): + """Add a callback -- a function that takes no parameters -- that will + return a list of filenames to watch for changes.""" + if self is None: + for instance in cls.instances: + instance.add_file_callback(callback) + cls.global_file_callbacks.append(callback) + else: + self.file_callbacks.append(callback) + + add_file_callback = classinstancemethod(add_file_callback) + +watch_file = Monitor.watch_file +add_file_callback = Monitor.add_file_callback + # For paste.deploy server instantiation (egg:pyramid#wsgiref) def wsgiref_server_runner(wsgi_app, global_conf, **kw): from wsgiref.simple_server import make_server diff --git a/pyramid/tests/test_scripts/test_pserve.py b/pyramid/tests/test_scripts/test_pserve.py new file mode 100644 index 000000000..bc2e9aac8 --- /dev/null +++ b/pyramid/tests/test_scripts/test_pserve.py @@ -0,0 +1,27 @@ +import unittest + +class TestPServeCommand(unittest.TestCase): + def setUp(self): + from pyramid.compat import NativeIO + self.out_ = NativeIO() + + def out(self, msg): + self.out_.write(msg) + + def _getTargetClass(self): + from pyramid.scripts.pserve import PServeCommand + return PServeCommand + + def _makeOne(self, *args): + effargs = ['pserve'] + effargs.extend(args) + cmd = self._getTargetClass()(effargs) + cmd.out = self.out + return cmd + + def test_no_args(self): + inst = self._makeOne() + result = inst.run() + self.assertEqual(result, None) + self.assertEqual(self.out_.getvalue(), 'You must give a config file') + -- cgit v1.2.3