[Supervisor-checkins] r860 - in supervisor/trunk: . src/medusa src/medusa/debian src/medusa/demo src/medusa/docs src/medusa/test src/medusa/thread src/supervisor src/supervisor/tests
Mike Naberezny
mike at maintainable.com
Fri May 22 23:51:55 EDT 2009
Author: Mike Naberezny <mike at maintainable.com>
Date: Fri May 22 23:51:54 2009
New Revision: 860
Log:
- We now bundle our own patched version of Medusa contributed by Jason
Kirtland to allow Supervisor to run on Python 2.6. This was done
because Python 2.6 introduced backwards incompatible changes to
asyncore and asynchat in the stdlib.
Added:
supervisor/trunk/src/medusa/
supervisor/trunk/src/medusa/CHANGES.txt
supervisor/trunk/src/medusa/INSTALL.txt
supervisor/trunk/src/medusa/LICENSE.txt
supervisor/trunk/src/medusa/MANIFEST
supervisor/trunk/src/medusa/Makefile
supervisor/trunk/src/medusa/README.txt
supervisor/trunk/src/medusa/TODO.txt
supervisor/trunk/src/medusa/__init__.py
supervisor/trunk/src/medusa/asynchat_25.py
supervisor/trunk/src/medusa/asyncore_25.py
supervisor/trunk/src/medusa/auth_handler.py
supervisor/trunk/src/medusa/chat_server.py
supervisor/trunk/src/medusa/counter.py
supervisor/trunk/src/medusa/debian/
supervisor/trunk/src/medusa/debian/changelog
supervisor/trunk/src/medusa/debian/control
supervisor/trunk/src/medusa/debian/copyright
supervisor/trunk/src/medusa/debian/postinst (contents, props changed)
supervisor/trunk/src/medusa/debian/prerm
supervisor/trunk/src/medusa/debian/rules (contents, props changed)
supervisor/trunk/src/medusa/default_handler.py
supervisor/trunk/src/medusa/demo/
supervisor/trunk/src/medusa/demo/publish.py
supervisor/trunk/src/medusa/demo/script_server.py
supervisor/trunk/src/medusa/demo/simple_anon_ftpd.py
supervisor/trunk/src/medusa/demo/start_medusa.py
supervisor/trunk/src/medusa/demo/winFTPserver.py
supervisor/trunk/src/medusa/docs/
supervisor/trunk/src/medusa/docs/README.html
supervisor/trunk/src/medusa/docs/async_blurbs.txt
supervisor/trunk/src/medusa/docs/composing_producers.gif (contents, props changed)
supervisor/trunk/src/medusa/docs/data_flow.gif (contents, props changed)
supervisor/trunk/src/medusa/docs/data_flow.html
supervisor/trunk/src/medusa/docs/debugging.txt
supervisor/trunk/src/medusa/docs/producers.gif (contents, props changed)
supervisor/trunk/src/medusa/docs/programming.html
supervisor/trunk/src/medusa/docs/proxy_notes.txt
supervisor/trunk/src/medusa/docs/threads.txt
supervisor/trunk/src/medusa/docs/tkinter.txt
supervisor/trunk/src/medusa/event_loop.py
supervisor/trunk/src/medusa/filesys.py
supervisor/trunk/src/medusa/ftp_server.py
supervisor/trunk/src/medusa/http_date.py
supervisor/trunk/src/medusa/http_server.py
supervisor/trunk/src/medusa/logger.py
supervisor/trunk/src/medusa/m_syslog.py
supervisor/trunk/src/medusa/medusa_gif.py
supervisor/trunk/src/medusa/monitor.py
supervisor/trunk/src/medusa/monitor_client.py
supervisor/trunk/src/medusa/monitor_client_win32.py
supervisor/trunk/src/medusa/producers.py
supervisor/trunk/src/medusa/put_handler.py
supervisor/trunk/src/medusa/redirecting_handler.py
supervisor/trunk/src/medusa/resolver.py
supervisor/trunk/src/medusa/rpc_client.py
supervisor/trunk/src/medusa/rpc_server.py
supervisor/trunk/src/medusa/script_handler.py
supervisor/trunk/src/medusa/setup.py
supervisor/trunk/src/medusa/status_handler.py
supervisor/trunk/src/medusa/test/
supervisor/trunk/src/medusa/test/asyn_http_bench.py (contents, props changed)
supervisor/trunk/src/medusa/test/bench.py
supervisor/trunk/src/medusa/test/max_sockets.py
supervisor/trunk/src/medusa/test/test_11.py
supervisor/trunk/src/medusa/test/test_lb.py
supervisor/trunk/src/medusa/test/test_medusa.py
supervisor/trunk/src/medusa/test/test_producers.py
supervisor/trunk/src/medusa/test/test_single_11.py
supervisor/trunk/src/medusa/test/tests.txt
supervisor/trunk/src/medusa/thread/
supervisor/trunk/src/medusa/thread/pi_module.py
supervisor/trunk/src/medusa/thread/select_trigger.py
supervisor/trunk/src/medusa/thread/test_module.py
supervisor/trunk/src/medusa/thread/thread_channel.py
supervisor/trunk/src/medusa/thread/thread_handler.py
supervisor/trunk/src/medusa/unix_user_handler.py
supervisor/trunk/src/medusa/virtual_handler.py
supervisor/trunk/src/medusa/xmlrpc_handler.py
Modified:
supervisor/trunk/CHANGES.txt
supervisor/trunk/setup.py
supervisor/trunk/src/supervisor/http.py
supervisor/trunk/src/supervisor/http_client.py
supervisor/trunk/src/supervisor/options.py
supervisor/trunk/src/supervisor/process.py
supervisor/trunk/src/supervisor/supervisorctl.py
supervisor/trunk/src/supervisor/supervisord.py
supervisor/trunk/src/supervisor/tests/test_http.py
supervisor/trunk/src/supervisor/tests/test_supervisord.py
Modified: supervisor/trunk/CHANGES.txt
==============================================================================
--- supervisor/trunk/CHANGES.txt (original)
+++ supervisor/trunk/CHANGES.txt Fri May 22 23:51:54 2009
@@ -1,4 +1,9 @@
Next Release
+
+ - We now bundle our own patched version of Medusa contributed by Jason
+ Kirtland to allow Supervisor to run on Python 2.6. This was done
+ because Python 2.6 introduced backwards incompatible changes to
+ asyncore and asynchat in the stdlib.
- The console script ``memmon``, introduced in Supervisor 3.0a4, has
been moved to Superlance (http://pypi.python.org/pypi/superlance).
Modified: supervisor/trunk/setup.py
==============================================================================
--- supervisor/trunk/setup.py (original)
+++ supervisor/trunk/setup.py Fri May 22 23:51:54 2009
@@ -81,10 +81,9 @@
'COPYRIGHT.txt'
]
)],
- install_requires = ['medusa >= 0.5.4', 'meld3 >= 0.6.4',
- 'elementtree >= 1.2.6,<1.2.7'],
+ install_requires = ['meld3 >= 0.6.4', 'elementtree >= 1.2.6,<1.2.7'],
extras_require = {'iterparse':['cElementTree >= 1.0.2']},
- tests_require = ['meld3 >= 0.6.3', 'medusa >= 0.5.4'],
+ tests_require = ['meld3 >= 0.6.3'],
include_package_data = True,
zip_safe = False,
namespace_packages = ['supervisor'],
Added: supervisor/trunk/src/medusa/CHANGES.txt
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/CHANGES.txt Fri May 22 23:51:54 2009
@@ -0,0 +1,81 @@
+PATCHES MADE ONLY TO THIS MEDUSA PACKAGE BUNDLED WITH SUPERVISOR
+* Re-added asyncore.py as asyncore_25.py and asynchat.py as asynchat_25.py
+ and updated Medusa throughout to use these. The Python 2.6 stdlib version
+ of these modules introduced backward-incompatible changes.
+
+Version 0.5.5:
+
+* [Patch #855389] ADD RNFR & RNTO commands to FTP server (Robin Becker)
+* [Patch #852089] In status_handler, catch any exception raised by the status()
+ method.
+* [Patch #855695] Bugfix for filesys.msdos_date
+* [Patch from Jason Sibre] Improve performance of xmlrpc_handler
+ by avoiding string concatentation
+* [Patch from Jason Sibre] Add interface to http_request for multiple
+ headers with the same name.
+
+
+Version 0.5.4:
+
+* Open syslog using datagram sockets, and only try streams if that fails
+ (Alxy Sav)
+* Fix bug in http_server.crack_request() (Contributed by Canis Lupus)
+* Incorporate bugfixes to thread/select_trigger.py from ZEO's version
+ (Zope Corporation)
+* Add demo/winFTPserver.py, an FTP server that uses Windows
+ authorization to determine who can access the server. (Contributed
+ by John Abel)
+* Add control files for creating a Debian package of Medusa (python2.2-medusa).
+
+
+Version 0.5.3:
+
+* Delete the broken and rather boring dual_server and simple_httpd
+ demo scripts. start_medusa.py should be sufficient as an example.
+* Fix indentation bug in demo/script_server.py noted by Richard Philips
+* Fix bug in producers.composite_producer, spotted and fixed by Daniel Krech
+* Added test suite for producers.py
+* Fix timestamps in http_server logs
+* Fix unix_user_handler bug, spotted and fixed by Sergio Fernández.
+* Fix auth_handler bug, spotted and fixed by Sergio Fernández.
+* Delete unused http_server.fifo class and fifo.py module.
+
+Version 0.5.2:
+
+* Fix syntax error and missing import in default_handler.py
+* Fix various scripts in demo/
+
+Version 0.5.1:
+
+* Apply cleanup patch from Donovan Baarda
+* Fix bug reported by Van Gale: counter.py and auth_handler.py did
+ long(...)[:-1] to chop off a trailing L generated in earlier
+ versions of Python.
+* Fix bug in ftp_server.py that I introduced in 0.5
+* Remove some duplicated producer classes
+* Removed work_in_progress/ directory and the 'continuation' module
+* Remove MIME type table code and use the stdlib's mimelib module
+
+Version 0.5:
+
+* Added a setup.py installation script, which will install all the code
+ under the package name 'medusa'.
+* Added README.txt and CHANGES.txt.
+* Fixed NameError in util/convert_mime_type_table.py
+* Fixed TypeError in test/test_medusa.py
+* Fixed several problems detected by PyChecker
+* Changed demos to use 'from medusa import ...'
+* Rearranged files to reduce the number of subdirectories.
+* Removed or updated uses of the obsolete regsub module
+* Removed asyncore.py and asynchat.py; these modules were added to Python's
+ standard library with version 1.5.2, and Medusa now assumes that they're
+ present.
+* Removed many obsolete files:
+ poll/pollmodule.c, as Python's select module now supports poll()
+ patches/posixmodule.c, as that patch was incorporated in Python
+ old/*, script_handler_demo/*, sendfile/*
+ The old ANNOUNCE files
+* Reindented all files to use four-space indents
+
+
+The last version of Medusa released by Sam Rushing was medusa-20010416.
Added: supervisor/trunk/src/medusa/INSTALL.txt
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/INSTALL.txt Fri May 22 23:51:54 2009
@@ -0,0 +1,126 @@
+
+ Medusa Installation.
+---------------------------------------------------------------------------
+
+1. INSTALL PYTHON
+
+Medusa is distributed as Python source code. Before using Medusa, you
+will need to install Python on your machine.
+
+The Python interpreter, source, documentation, etc... may be obtained
+from
+
+ http://www.python.org/
+
+Versions for many different operating systems are available, including
+Unix, 32-bit Windows (Win95 & NT), Macintosh, VMS, etc... Medusa has
+been tested on Unix and Windows, though it may very well work on other
+operating systems.
+
+You don't need to learn Python in order to use Medusa. However, if
+you are interested in extending Medusa, you should spend the hour or
+so that it will take you to go through the Python Tutorial:
+
+ http://www.python.org/doc/tut/
+
+Python is remarkably easy to learn, and I guarantee that it will be
+worth your while. After only about thirty minutes, you should know
+enough about Python to be able to start customizing and extending
+Medusa.
+
+
+2. INSTALL MEDUSA
+
+The core Medusa code consists of a single package named 'medusa'. To
+install it in the site-packages directory of your Python installation,
+run the command "python setup.py install".
+
+After running this command, you should be able to run the Python
+interpreter and import things from the 'medusa' package:
+
+bash-2.05$ python
+>>> from medusa import ftp_server
+>>>
+
+
+3. WRITE A MEDUSA STARTUP SCRIPT
+
+Once you have installed Python, you are ready to configure Medusa.
+Medusa does not use configuration files per se, or even command-line
+arguments. It is configured via a 'startup script', written in
+Python. A sample is provided in 'demo/start_medusa.py'. You should make
+a copy of this.
+
+The sample startup script is heavily commented. Many (though not all)
+of Medusa's features are made available in the startup script. You may
+modify this script by commenting out portions, adding or changing
+parameters, etc...
+
+Here is a section from the front of 'demo/start_medusa.py'
+
+| if len(sys.argv) > 1:
+| # process a few convenient arguments
+| [HOSTNAME, IP_ADDRESS, PUBLISHING_ROOT] = sys.argv[1:]
+| else:
+| HOSTNAME = 'www.nightmare.com'
+| # This is the IP address of the network interface you want
+| # your servers to be visible from. This can be changed to ''
+| # to listen on all interfaces.
+| IP_ADDRESS = '205.160.176.5'
+|
+| # Root of the http and ftp server's published filesystems.
+| PUBLISHING_ROOT = '/home/www'
+|
+| HTTP_PORT = 8080 # The standard port is 80
+| FTP_PORT = 8021 # The standard port is 21
+| CHAT_PORT = 8888
+| MONITOR_PORT = 9999
+
+If you are familiar with the process of configuring a web or ftp
+server, then these parameters should be fairly obvious: You will
+need to change the hostname, IP address, and port numbers for the
+server that you wish to run.
+
+A Medusa configuration does not need to be this complex -
+demo/start_medusa.py is bloated somewhat by its attempt to include
+most of the available features. Another example startup script,
+demo/publish.py, is also available for you to look at.
+
+Once you have made your own startup script, you may simply invoke
+the Python interpreter on it:
+
+[Unix]
+$ python start_medusa.py &
+[Win32]
+d:\medusa\> start python start_medusa.py
+
+Medusa (V3.8) started at Sat Jan 24 01:43:21 1998
+ Hostname: ziggurat.nightmare.com
+ Port:8080
+<Unix User Directory Handler at 080e9c08 [~user/public_html, 0 filesystems loaded]>
+FTP server started at Sat Jan 24 01:43:21 1998
+ Authorizer:<test_authorizer instance at 80e8938>
+ Hostname: ziggurat.nightmare.com
+ Port: 21
+192.168.200.40:1450 - - [24/Jan/1998:07:43:23 -0500] "GET /status HTTP/1.0" 200 1638
+192.168.200.40:1451 - - [24/Jan/1998:07:43:23 -0500] "GET /status/medusa.gif HTTP/1.0" 200 1084
+
+Documentation for specific Medusa servers is somewhat lacking, mostly
+because development continues to move rapidly. The best place to go
+to understand Medusa and how it works is to dive into the source code.
+Many of the more interesting features, especially the latest, are
+described only in the source code.
+
+Some notes on data flow in Medusa are available in
+'docs/data_flow.html'.
+
+I encourage you to examine and experiment with Medusa. You may
+develop your own extensions, handlers, etc... I appreciate feedback
+from users and developers on desired features, and of course
+descriptions of your most splendid hacks.
+
+Medusa's design is somewhat novel compared to most other network
+servers. In fact, the asynchronous i/o capability seems to have
+attracted the majority of paying customers, who are often more
+interested in harnessing the i/o framework than the actual web and ftp
+servers.
Added: supervisor/trunk/src/medusa/LICENSE.txt
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/LICENSE.txt Fri May 22 23:51:54 2009
@@ -0,0 +1,30 @@
+Medusa was once distributed under a 'free for non-commercial use'
+license, but in May of 2000 Sam Rushing changed the license to be
+identical to the standard Python license at the time. The standard
+Python license has always applied to the core components of Medusa,
+this change just frees up the rest of the system, including the http
+server, ftp server, utilities, etc. Medusa is therefore under the
+following license:
+
+==============================
+Permission to use, copy, modify, and distribute this software and
+its documentation for any purpose and without fee is hereby granted,
+provided that the above copyright notice appear in all copies and
+that both that copyright notice and this permission notice appear in
+supporting documentation, and that the name of Sam Rushing not be
+used in advertising or publicity pertaining to distribution of the
+software without specific, written prior permission.
+
+SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+==============================
+
+Sam would like to take this opportunity to thank all of the folks who
+supported Medusa over the years by purchasing commercial licenses.
+
+
Added: supervisor/trunk/src/medusa/MANIFEST
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/MANIFEST Fri May 22 23:51:54 2009
@@ -0,0 +1,73 @@
+asynchat_25.py
+asyncore_25.py
+auth_handler.py
+CHANGES.txt
+chat_server.py
+counter.py
+default_handler.py
+demo/publish.py
+demo/script_server.py
+demo/simple_anon_ftpd.py
+demo/start_medusa.py
+demo/winFTPserver.py
+docs/async_blurbs.txt
+docs/composing_producers.gif
+docs/data_flow.gif
+docs/data_flow.html
+docs/debugging.txt
+docs/producers.gif
+docs/programming.html
+docs/proxy_notes.txt
+docs/README.html
+docs/threads.txt
+docs/tkinter.txt
+event_loop.py
+filesys.py
+ftp_server.py
+http_date.py
+http_server.py
+__init__.py
+INSTALL.txt
+LICENSE.txt
+logger.py
+Makefile
+MANIFEST
+medusa_gif.py
+monitor_client.py
+monitor_client_win32.py
+monitor.py
+m_syslog.py
+producers.py
+put_handler.py
+README.txt
+redirecting_handler.py
+resolver.py
+rpc_client.py
+rpc_server.py
+script_handler.py
+setup.py
+status_handler.py
+test/asyn_http_bench.py
+test/max_sockets.py
+test/test_11.py
+test/test_lb.py
+test/test_medusa.py
+test/test_single_11.py
+test/tests.txt
+thread/pi_module.py
+thread/select_trigger.py
+thread/test_module.py
+thread/thread_channel.py
+thread/thread_handler.py
+TODO.txt
+unix_user_handler.py
+test/test_producers.py
+test/bench.py
+virtual_handler.py
+xmlrpc_handler.py
+debian/changelog
+debian/control
+debian/copyright
+debian/postinst
+debian/prerm
+debian/rules
Added: supervisor/trunk/src/medusa/Makefile
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/Makefile Fri May 22 23:51:54 2009
@@ -0,0 +1,9 @@
+# -*- Mode: Makefile -*-
+
+clean:
+ find ./ -name '*.pyc' -exec rm {} \;
+ find ./ -name '*~' -exec rm {} \;
+
+dist_debian:
+ dpkg-buildpackage -rfakeroot
+
Added: supervisor/trunk/src/medusa/README.txt
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/README.txt Fri May 22 23:51:54 2009
@@ -0,0 +1,39 @@
+Medusa is a 'server platform' -- it provides a framework for
+implementing asynchronous socket-based servers (TCP/IP and on Unix,
+Unix domain, sockets).
+
+An asynchronous socket server is a server that can communicate with many
+other clients simultaneously by multiplexing I/O within a single
+process/thread. In the context of an HTTP server, this means a single
+process can serve hundreds or even thousands of clients, depending only on
+the operating system's configuration and limitations.
+
+There are several advantages to this approach:
+
+ o performance - no fork() or thread() start-up costs per hit.
+
+ o scalability - the overhead per client can be kept rather small,
+ on the order of several kilobytes of memory.
+
+ o persistence - a single-process server can easily coordinate the
+ actions of several different connections. This makes things like
+ proxy servers and gateways easy to implement. It also makes it
+ possible to share resources like database handles.
+
+Medusa includes HTTP, FTP, and 'monitor' (remote python interpreter)
+servers. Medusa can simultaneously support several instances of
+either the same or different server types - for example you could
+start up two HTTP servers, an FTP server, and a monitor server. Then
+you could connect to the monitor server to control and manipulate
+medusa while it is running.
+
+Other servers and clients have been written (SMTP, POP3, NNTP), and
+several are in the planning stages.
+
+Medusa was originally written by Sam Rushing <rushing at nightmare.com>,
+and its original Web page is at <http://www.nightmare.com/medusa/>. After
+Sam moved on to other things, A.M. Kuchling <akuchlin at mems-exchange.org>
+took over maintenance of the Medusa package.
+
+--amk
+
Added: supervisor/trunk/src/medusa/TODO.txt
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/TODO.txt Fri May 22 23:51:54 2009
@@ -0,0 +1,13 @@
+Things to do
+============
+
+Bring remaining code up to current standards
+Translate docs to RST
+Write README, INSTALL, docs
+What should __init__ import? Anything? Every single class?
+Use syslog module in m_syslog for the constants?
+Add abo's support for blocking producers
+Get all the producers into the producers module and write tests for them
+
+Test suites for protocols: how could that be implemented?
+
Added: supervisor/trunk/src/medusa/__init__.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/__init__.py Fri May 22 23:51:54 2009
@@ -0,0 +1,6 @@
+"""medusa.__init__
+"""
+
+# created 2002/03/19, AMK
+
+__revision__ = "$Id: __init__.py,v 1.2 2002/03/19 22:49:34 amk Exp $"
Added: supervisor/trunk/src/medusa/asynchat_25.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/asynchat_25.py Fri May 22 23:51:54 2009
@@ -0,0 +1,295 @@
+# -*- Mode: Python; tab-width: 4 -*-
+# Id: asynchat.py,v 2.26 2000/09/07 22:29:26 rushing Exp
+# Author: Sam Rushing <rushing at nightmare.com>
+
+# ======================================================================
+# Copyright 1996 by Sam Rushing
+#
+# All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of Sam
+# Rushing not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+# ======================================================================
+
+r"""A class supporting chat-style (command/response) protocols.
+
+This class adds support for 'chat' style protocols - where one side
+sends a 'command', and the other sends a response (examples would be
+the common internet protocols - smtp, nntp, ftp, etc..).
+
+The handle_read() method looks at the input stream for the current
+'terminator' (usually '\r\n' for single-line responses, '\r\n.\r\n'
+for multi-line output), calling self.found_terminator() on its
+receipt.
+
+for example:
+Say you build an async nntp client using this class. At the start
+of the connection, you'll have self.terminator set to '\r\n', in
+order to process the single-line greeting. Just before issuing a
+'LIST' command you'll set it to '\r\n.\r\n'. The output of the LIST
+command will be accumulated (using your own 'collect_incoming_data'
+method) up to the terminator, and then control will be returned to
+you - by calling your self.found_terminator() method.
+"""
+
+import socket
+from medusa import asyncore_25 as asyncore
+from collections import deque
+
+class async_chat (asyncore.dispatcher):
+ """This is an abstract class. You must derive from this class, and add
+ the two methods collect_incoming_data() and found_terminator()"""
+
+ # these are overridable defaults
+
+ ac_in_buffer_size = 4096
+ ac_out_buffer_size = 4096
+
+ def __init__ (self, conn=None):
+ self.ac_in_buffer = ''
+ self.ac_out_buffer = ''
+ self.producer_fifo = fifo()
+ asyncore.dispatcher.__init__ (self, conn)
+
+ def collect_incoming_data(self, data):
+ raise NotImplementedError, "must be implemented in subclass"
+
+ def found_terminator(self):
+ raise NotImplementedError, "must be implemented in subclass"
+
+ def set_terminator (self, term):
+ "Set the input delimiter. Can be a fixed string of any length, an integer, or None"
+ self.terminator = term
+
+ def get_terminator (self):
+ return self.terminator
+
+ # grab some more data from the socket,
+ # throw it to the collector method,
+ # check for the terminator,
+ # if found, transition to the next state.
+
+ def handle_read (self):
+
+ try:
+ data = self.recv (self.ac_in_buffer_size)
+ except socket.error, why:
+ self.handle_error()
+ return
+
+ self.ac_in_buffer = self.ac_in_buffer + data
+
+ # Continue to search for self.terminator in self.ac_in_buffer,
+ # while calling self.collect_incoming_data. The while loop
+ # is necessary because we might read several data+terminator
+ # combos with a single recv(1024).
+
+ while self.ac_in_buffer:
+ lb = len(self.ac_in_buffer)
+ terminator = self.get_terminator()
+ if not terminator:
+ # no terminator, collect it all
+ self.collect_incoming_data (self.ac_in_buffer)
+ self.ac_in_buffer = ''
+ elif isinstance(terminator, int) or isinstance(terminator, long):
+ # numeric terminator
+ n = terminator
+ if lb < n:
+ self.collect_incoming_data (self.ac_in_buffer)
+ self.ac_in_buffer = ''
+ self.terminator = self.terminator - lb
+ else:
+ self.collect_incoming_data (self.ac_in_buffer[:n])
+ self.ac_in_buffer = self.ac_in_buffer[n:]
+ self.terminator = 0
+ self.found_terminator()
+ else:
+ # 3 cases:
+ # 1) end of buffer matches terminator exactly:
+ # collect data, transition
+ # 2) end of buffer matches some prefix:
+ # collect data to the prefix
+ # 3) end of buffer does not match any prefix:
+ # collect data
+ terminator_len = len(terminator)
+ index = self.ac_in_buffer.find(terminator)
+ if index != -1:
+ # we found the terminator
+ if index > 0:
+ # don't bother reporting the empty string (source of subtle bugs)
+ self.collect_incoming_data (self.ac_in_buffer[:index])
+ self.ac_in_buffer = self.ac_in_buffer[index+terminator_len:]
+ # This does the Right Thing if the terminator is changed here.
+ self.found_terminator()
+ else:
+ # check for a prefix of the terminator
+ index = find_prefix_at_end (self.ac_in_buffer, terminator)
+ if index:
+ if index != lb:
+ # we found a prefix, collect up to the prefix
+ self.collect_incoming_data (self.ac_in_buffer[:-index])
+ self.ac_in_buffer = self.ac_in_buffer[-index:]
+ break
+ else:
+ # no prefix, collect it all
+ self.collect_incoming_data (self.ac_in_buffer)
+ self.ac_in_buffer = ''
+
+ def handle_write (self):
+ self.initiate_send ()
+
+ def handle_close (self):
+ self.close()
+
+ def push (self, data):
+ self.producer_fifo.push (simple_producer (data))
+ self.initiate_send()
+
+ def push_with_producer (self, producer):
+ self.producer_fifo.push (producer)
+ self.initiate_send()
+
+ def readable (self):
+ "predicate for inclusion in the readable for select()"
+ return (len(self.ac_in_buffer) <= self.ac_in_buffer_size)
+
+ def writable (self):
+ "predicate for inclusion in the writable for select()"
+ # return len(self.ac_out_buffer) or len(self.producer_fifo) or (not self.connected)
+ # this is about twice as fast, though not as clear.
+ return not (
+ (self.ac_out_buffer == '') and
+ self.producer_fifo.is_empty() and
+ self.connected
+ )
+
+ def close_when_done (self):
+ "automatically close this channel once the outgoing queue is empty"
+ self.producer_fifo.push (None)
+
+ # refill the outgoing buffer by calling the more() method
+ # of the first producer in the queue
+ def refill_buffer (self):
+ while 1:
+ if len(self.producer_fifo):
+ p = self.producer_fifo.first()
+ # a 'None' in the producer fifo is a sentinel,
+ # telling us to close the channel.
+ if p is None:
+ if not self.ac_out_buffer:
+ self.producer_fifo.pop()
+ self.close()
+ return
+ elif isinstance(p, str):
+ self.producer_fifo.pop()
+ self.ac_out_buffer = self.ac_out_buffer + p
+ return
+ data = p.more()
+ if data:
+ self.ac_out_buffer = self.ac_out_buffer + data
+ return
+ else:
+ self.producer_fifo.pop()
+ else:
+ return
+
+ def initiate_send (self):
+ obs = self.ac_out_buffer_size
+ # try to refill the buffer
+ if (len (self.ac_out_buffer) < obs):
+ self.refill_buffer()
+
+ if self.ac_out_buffer and self.connected:
+ # try to send the buffer
+ try:
+ num_sent = self.send (self.ac_out_buffer[:obs])
+ if num_sent:
+ self.ac_out_buffer = self.ac_out_buffer[num_sent:]
+
+ except socket.error, why:
+ self.handle_error()
+ return
+
+ def discard_buffers (self):
+ # Emergencies only!
+ self.ac_in_buffer = ''
+ self.ac_out_buffer = ''
+ while self.producer_fifo:
+ self.producer_fifo.pop()
+
+
+class simple_producer:
+
+ def __init__ (self, data, buffer_size=512):
+ self.data = data
+ self.buffer_size = buffer_size
+
+ def more (self):
+ if len (self.data) > self.buffer_size:
+ result = self.data[:self.buffer_size]
+ self.data = self.data[self.buffer_size:]
+ return result
+ else:
+ result = self.data
+ self.data = ''
+ return result
+
+class fifo:
+ def __init__ (self, list=None):
+ if not list:
+ self.list = deque()
+ else:
+ self.list = deque(list)
+
+ def __len__ (self):
+ return len(self.list)
+
+ def is_empty (self):
+ return not self.list
+
+ def first (self):
+ return self.list[0]
+
+ def push (self, data):
+ self.list.append(data)
+
+ def pop (self):
+ if self.list:
+ return (1, self.list.popleft())
+ else:
+ return (0, None)
+
+# Given 'haystack', see if any prefix of 'needle' is at its end. This
+# assumes an exact match has already been checked. Return the number of
+# characters matched.
+# for example:
+# f_p_a_e ("qwerty\r", "\r\n") => 1
+# f_p_a_e ("qwertydkjf", "\r\n") => 0
+# f_p_a_e ("qwerty\r\n", "\r\n") => <undefined>
+
+# this could maybe be made faster with a computed regex?
+# [answer: no; circa Python-2.0, Jan 2001]
+# new python: 28961/s
+# old python: 18307/s
+# re: 12820/s
+# regex: 14035/s
+
+def find_prefix_at_end (haystack, needle):
+ l = len(needle) - 1
+ while l and not haystack.endswith(needle[:l]):
+ l -= 1
+ return l
Added: supervisor/trunk/src/medusa/asyncore_25.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/asyncore_25.py Fri May 22 23:51:54 2009
@@ -0,0 +1,551 @@
+# -*- Mode: Python -*-
+# Id: asyncore.py,v 2.51 2000/09/07 22:29:26 rushing Exp
+# Author: Sam Rushing <rushing at nightmare.com>
+
+# ======================================================================
+# Copyright 1996 by Sam Rushing
+#
+# All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of Sam
+# Rushing not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+# ======================================================================
+
+"""Basic infrastructure for asynchronous socket service clients and servers.
+
+There are only two ways to have a program on a single processor do "more
+than one thing at a time". Multi-threaded programming is the simplest and
+most popular way to do it, but there is another very different technique,
+that lets you have nearly all the advantages of multi-threading, without
+actually using multiple threads. it's really only practical if your program
+is largely I/O bound. If your program is CPU bound, then pre-emptive
+scheduled threads are probably what you really need. Network servers are
+rarely CPU-bound, however.
+
+If your operating system supports the select() system call in its I/O
+library (and nearly all do), then you can use it to juggle multiple
+communication channels at once; doing other work while your I/O is taking
+place in the "background." Although this strategy can seem strange and
+complex, especially at first, it is in many ways easier to understand and
+control than multi-threaded programming. The module documented here solves
+many of the difficult problems for you, making the task of building
+sophisticated high-performance network servers and clients a snap.
+"""
+
+import select
+import socket
+import sys
+import time
+
+import os
+from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, \
+ ENOTCONN, ESHUTDOWN, EINTR, EISCONN, errorcode
+
+try:
+ socket_map
+except NameError:
+ socket_map = {}
+
+class ExitNow(Exception):
+ pass
+
+def read(obj):
+ try:
+ obj.handle_read_event()
+ except ExitNow:
+ raise
+ except:
+ obj.handle_error()
+
+def write(obj):
+ try:
+ obj.handle_write_event()
+ except ExitNow:
+ raise
+ except:
+ obj.handle_error()
+
+def _exception (obj):
+ try:
+ obj.handle_expt_event()
+ except ExitNow:
+ raise
+ except:
+ obj.handle_error()
+
+def readwrite(obj, flags):
+ try:
+ if flags & (select.POLLIN | select.POLLPRI):
+ obj.handle_read_event()
+ if flags & select.POLLOUT:
+ obj.handle_write_event()
+ if flags & (select.POLLERR | select.POLLHUP | select.POLLNVAL):
+ obj.handle_expt_event()
+ except ExitNow:
+ raise
+ except:
+ obj.handle_error()
+
+def poll(timeout=0.0, map=None):
+ if map is None:
+ map = socket_map
+ if map:
+ r = []; w = []; e = []
+ for fd, obj in map.items():
+ is_r = obj.readable()
+ is_w = obj.writable()
+ if is_r:
+ r.append(fd)
+ if is_w:
+ w.append(fd)
+ if is_r or is_w:
+ e.append(fd)
+ if [] == r == w == e:
+ time.sleep(timeout)
+ else:
+ try:
+ r, w, e = select.select(r, w, e, timeout)
+ except select.error, err:
+ if err[0] != EINTR:
+ raise
+ else:
+ return
+
+ for fd in r:
+ obj = map.get(fd)
+ if obj is None:
+ continue
+ read(obj)
+
+ for fd in w:
+ obj = map.get(fd)
+ if obj is None:
+ continue
+ write(obj)
+
+ for fd in e:
+ obj = map.get(fd)
+ if obj is None:
+ continue
+ _exception(obj)
+
+def poll2(timeout=0.0, map=None):
+ # Use the poll() support added to the select module in Python 2.0
+ if map is None:
+ map = socket_map
+ if timeout is not None:
+ # timeout is in milliseconds
+ timeout = int(timeout*1000)
+ pollster = select.poll()
+ if map:
+ for fd, obj in map.items():
+ flags = 0
+ if obj.readable():
+ flags |= select.POLLIN | select.POLLPRI
+ if obj.writable():
+ flags |= select.POLLOUT
+ if flags:
+ # Only check for exceptions if object was either readable
+ # or writable.
+ flags |= select.POLLERR | select.POLLHUP | select.POLLNVAL
+ pollster.register(fd, flags)
+ try:
+ r = pollster.poll(timeout)
+ except select.error, err:
+ if err[0] != EINTR:
+ raise
+ r = []
+ for fd, flags in r:
+ obj = map.get(fd)
+ if obj is None:
+ continue
+ readwrite(obj, flags)
+
+poll3 = poll2 # Alias for backward compatibility
+
+def loop(timeout=30.0, use_poll=False, map=None, count=None):
+ if map is None:
+ map = socket_map
+
+ if use_poll and hasattr(select, 'poll'):
+ poll_fun = poll2
+ else:
+ poll_fun = poll
+
+ if count is None:
+ while map:
+ poll_fun(timeout, map)
+
+ else:
+ while map and count > 0:
+ poll_fun(timeout, map)
+ count = count - 1
+
+class dispatcher:
+
+ debug = False
+ connected = False
+ accepting = False
+ closing = False
+ addr = None
+
+ def __init__(self, sock=None, map=None):
+ if map is None:
+ self._map = socket_map
+ else:
+ self._map = map
+
+ if sock:
+ self.set_socket(sock, map)
+ # I think it should inherit this anyway
+ self.socket.setblocking(0)
+ self.connected = True
+ # XXX Does the constructor require that the socket passed
+ # be connected?
+ try:
+ self.addr = sock.getpeername()
+ except socket.error:
+ # The addr isn't crucial
+ pass
+ else:
+ self.socket = None
+
+ def __repr__(self):
+ status = [self.__class__.__module__+"."+self.__class__.__name__]
+ if self.accepting and self.addr:
+ status.append('listening')
+ elif self.connected:
+ status.append('connected')
+ if self.addr is not None:
+ try:
+ status.append('%s:%d' % self.addr)
+ except TypeError:
+ status.append(repr(self.addr))
+ return '<%s at %#x>' % (' '.join(status), id(self))
+
+ def add_channel(self, map=None):
+ #self.log_info('adding channel %s' % self)
+ if map is None:
+ map = self._map
+ map[self._fileno] = self
+
+ def del_channel(self, map=None):
+ fd = self._fileno
+ if map is None:
+ map = self._map
+ if map.has_key(fd):
+ #self.log_info('closing channel %d:%s' % (fd, self))
+ del map[fd]
+ self._fileno = None
+
+ def create_socket(self, family, type):
+ self.family_and_type = family, type
+ self.socket = socket.socket(family, type)
+ self.socket.setblocking(0)
+ self._fileno = self.socket.fileno()
+ self.add_channel()
+
+ def set_socket(self, sock, map=None):
+ self.socket = sock
+## self.__dict__['socket'] = sock
+ self._fileno = sock.fileno()
+ self.add_channel(map)
+
+ def set_reuse_addr(self):
+ # try to re-use a server port if possible
+ try:
+ self.socket.setsockopt(
+ socket.SOL_SOCKET, socket.SO_REUSEADDR,
+ self.socket.getsockopt(socket.SOL_SOCKET,
+ socket.SO_REUSEADDR) | 1
+ )
+ except socket.error:
+ pass
+
+ # ==================================================
+ # predicates for select()
+ # these are used as filters for the lists of sockets
+ # to pass to select().
+ # ==================================================
+
+ def readable(self):
+ return True
+
+ def writable(self):
+ return True
+
+ # ==================================================
+ # socket object methods.
+ # ==================================================
+
+ def listen(self, num):
+ self.accepting = True
+ if os.name == 'nt' and num > 5:
+ num = 1
+ return self.socket.listen(num)
+
+ def bind(self, addr):
+ self.addr = addr
+ return self.socket.bind(addr)
+
+ def connect(self, address):
+ self.connected = False
+ err = self.socket.connect_ex(address)
+ # XXX Should interpret Winsock return values
+ if err in (EINPROGRESS, EALREADY, EWOULDBLOCK):
+ return
+ if err in (0, EISCONN):
+ self.addr = address
+ self.connected = True
+ self.handle_connect()
+ else:
+ raise socket.error, (err, errorcode[err])
+
+ def accept(self):
+ # XXX can return either an address pair or None
+ try:
+ conn, addr = self.socket.accept()
+ return conn, addr
+ except socket.error, why:
+ if why[0] == EWOULDBLOCK:
+ pass
+ else:
+ raise
+
+ def send(self, data):
+ try:
+ result = self.socket.send(data)
+ return result
+ except socket.error, why:
+ if why[0] == EWOULDBLOCK:
+ return 0
+ else:
+ raise
+ return 0
+
+ def recv(self, buffer_size):
+ try:
+ data = self.socket.recv(buffer_size)
+ if not data:
+ # a closed connection is indicated by signaling
+ # a read condition, and having recv() return 0.
+ self.handle_close()
+ return ''
+ else:
+ return data
+ except socket.error, why:
+ # winsock sometimes throws ENOTCONN
+ if why[0] in [ECONNRESET, ENOTCONN, ESHUTDOWN]:
+ self.handle_close()
+ return ''
+ else:
+ raise
+
+ def close(self):
+ self.del_channel()
+ self.socket.close()
+
+ # cheap inheritance, used to pass all other attribute
+ # references to the underlying socket object.
+ def __getattr__(self, attr):
+ return getattr(self.socket, attr)
+
+ # log and log_info may be overridden to provide more sophisticated
+ # logging and warning methods. In general, log is for 'hit' logging
+ # and 'log_info' is for informational, warning and error logging.
+
+ def log(self, message):
+ sys.stderr.write('log: %s\n' % str(message))
+
+ def log_info(self, message, type='info'):
+ if __debug__ or type != 'info':
+ print '%s: %s' % (type, message)
+
+ def handle_read_event(self):
+ if self.accepting:
+ # for an accepting socket, getting a read implies
+ # that we are connected
+ if not self.connected:
+ self.connected = True
+ self.handle_accept()
+ elif not self.connected:
+ self.handle_connect()
+ self.connected = True
+ self.handle_read()
+ else:
+ self.handle_read()
+
+ def handle_write_event(self):
+ # getting a write implies that we are connected
+ if not self.connected:
+ self.handle_connect()
+ self.connected = True
+ self.handle_write()
+
+ def handle_expt_event(self):
+ self.handle_expt()
+
+ def handle_error(self):
+ nil, t, v, tbinfo = compact_traceback()
+
+ # sometimes a user repr method will crash.
+ try:
+ self_repr = repr(self)
+ except:
+ self_repr = '<__repr__(self) failed for object at %0x>' % id(self)
+
+ self.log_info(
+ 'uncaptured python exception, closing channel %s (%s:%s %s)' % (
+ self_repr,
+ t,
+ v,
+ tbinfo
+ ),
+ 'error'
+ )
+ self.close()
+
+ def handle_expt(self):
+ self.log_info('unhandled exception', 'warning')
+
+ def handle_read(self):
+ self.log_info('unhandled read event', 'warning')
+
+ def handle_write(self):
+ self.log_info('unhandled write event', 'warning')
+
+ def handle_connect(self):
+ self.log_info('unhandled connect event', 'warning')
+
+ def handle_accept(self):
+ self.log_info('unhandled accept event', 'warning')
+
+ def handle_close(self):
+ self.log_info('unhandled close event', 'warning')
+ self.close()
+
+# ---------------------------------------------------------------------------
+# adds simple buffered output capability, useful for simple clients.
+# [for more sophisticated usage use asynchat.async_chat]
+# ---------------------------------------------------------------------------
+
+class dispatcher_with_send(dispatcher):
+
+ def __init__(self, sock=None, map=None):
+ dispatcher.__init__(self, sock, map)
+ self.out_buffer = ''
+
+ def initiate_send(self):
+ num_sent = 0
+ num_sent = dispatcher.send(self, self.out_buffer[:512])
+ self.out_buffer = self.out_buffer[num_sent:]
+
+ def handle_write(self):
+ self.initiate_send()
+
+ def writable(self):
+ return (not self.connected) or len(self.out_buffer)
+
+ def send(self, data):
+ if self.debug:
+ self.log_info('sending %s' % repr(data))
+ self.out_buffer = self.out_buffer + data
+ self.initiate_send()
+
+# ---------------------------------------------------------------------------
+# used for debugging.
+# ---------------------------------------------------------------------------
+
+def compact_traceback():
+ t, v, tb = sys.exc_info()
+ tbinfo = []
+ assert tb # Must have a traceback
+ while tb:
+ tbinfo.append((
+ tb.tb_frame.f_code.co_filename,
+ tb.tb_frame.f_code.co_name,
+ str(tb.tb_lineno)
+ ))
+ tb = tb.tb_next
+
+ # just to be safe
+ del tb
+
+ file, function, line = tbinfo[-1]
+ info = ' '.join(['[%s|%s|%s]' % x for x in tbinfo])
+ return (file, function, line), t, v, info
+
+def close_all(map=None):
+ if map is None:
+ map = socket_map
+ for x in map.values():
+ x.socket.close()
+ map.clear()
+
+# Asynchronous File I/O:
+#
+# After a little research (reading man pages on various unixen, and
+# digging through the linux kernel), I've determined that select()
+# isn't meant for doing asynchronous file i/o.
+# Heartening, though - reading linux/mm/filemap.c shows that linux
+# supports asynchronous read-ahead. So _MOST_ of the time, the data
+# will be sitting in memory for us already when we go to read it.
+#
+# What other OS's (besides NT) support async file i/o? [VMS?]
+#
+# Regardless, this is useful for pipes, and stdin/stdout...
+
+if os.name == 'posix':
+ import fcntl
+
+ class file_wrapper:
+ # here we override just enough to make a file
+ # look like a socket for the purposes of asyncore.
+
+ def __init__(self, fd):
+ self.fd = fd
+
+ def recv(self, *args):
+ return os.read(self.fd, *args)
+
+ def send(self, *args):
+ return os.write(self.fd, *args)
+
+ read = recv
+ write = send
+
+ def close(self):
+ os.close(self.fd)
+
+ def fileno(self):
+ return self.fd
+
+ class file_dispatcher(dispatcher):
+
+ def __init__(self, fd, map=None):
+ dispatcher.__init__(self, None, map)
+ self.connected = True
+ self.set_file(fd)
+ # set it to non-blocking mode
+ flags = fcntl.fcntl(fd, fcntl.F_GETFL, 0)
+ flags = flags | os.O_NONBLOCK
+ fcntl.fcntl(fd, fcntl.F_SETFL, flags)
+
+ def set_file(self, fd):
+ self._fileno = fd
+ self.socket = file_wrapper(fd)
+ self.add_channel()
Added: supervisor/trunk/src/medusa/auth_handler.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/auth_handler.py Fri May 22 23:51:54 2009
@@ -0,0 +1,139 @@
+# -*- Mode: Python -*-
+#
+# Author: Sam Rushing <rushing at nightmare.com>
+# Copyright 1996-2000 by Sam Rushing
+# All Rights Reserved.
+#
+
+RCS_ID = '$Id: auth_handler.py,v 1.6 2002/11/25 19:40:23 akuchling Exp $'
+
+# support for 'basic' authenticaion.
+
+import base64
+try:
+ from hashlib import md5
+except ImportError:
+ from md5 import new as md5
+import re
+import string
+import time
+import counter
+
+import default_handler
+
+get_header = default_handler.get_header
+
+import producers
+
+# This is a 'handler' that wraps an authorization method
+# around access to the resources normally served up by
+# another handler.
+
+# does anyone support digest authentication? (rfc2069)
+
+class auth_handler:
+ def __init__ (self, dict, handler, realm='default'):
+ self.authorizer = dictionary_authorizer (dict)
+ self.handler = handler
+ self.realm = realm
+ self.pass_count = counter.counter()
+ self.fail_count = counter.counter()
+
+ def match (self, request):
+ # by default, use the given handler's matcher
+ return self.handler.match (request)
+
+ def handle_request (self, request):
+ # authorize a request before handling it...
+ scheme = get_header (AUTHORIZATION, request.header)
+
+ if scheme:
+ scheme = string.lower (scheme)
+ if scheme == 'basic':
+ cookie = get_header (AUTHORIZATION, request.header, 2)
+ try:
+ decoded = base64.decodestring (cookie)
+ except:
+ print 'malformed authorization info <%s>' % cookie
+ request.error (400)
+ return
+ auth_info = string.split (decoded, ':')
+ if self.authorizer.authorize (auth_info):
+ self.pass_count.increment()
+ request.auth_info = auth_info
+ self.handler.handle_request (request)
+ else:
+ self.handle_unauthorized (request)
+ #elif scheme == 'digest':
+ # print 'digest: ',AUTHORIZATION.group(2)
+ else:
+ print 'unknown/unsupported auth method: %s' % scheme
+ self.handle_unauthorized(request)
+ else:
+ # list both? prefer one or the other?
+ # you could also use a 'nonce' here. [see below]
+ #auth = 'Basic realm="%s" Digest realm="%s"' % (self.realm, self.realm)
+ #nonce = self.make_nonce (request)
+ #auth = 'Digest realm="%s" nonce="%s"' % (self.realm, nonce)
+ #request['WWW-Authenticate'] = auth
+ #print 'sending header: %s' % request['WWW-Authenticate']
+ self.handle_unauthorized (request)
+
+ def handle_unauthorized (self, request):
+ # We are now going to receive data that we want to ignore.
+ # to ignore the file data we're not interested in.
+ self.fail_count.increment()
+ request.channel.set_terminator (None)
+ request['Connection'] = 'close'
+ request['WWW-Authenticate'] = 'Basic realm="%s"' % self.realm
+ request.error (401)
+
+ def make_nonce (self, request):
+ "A digest-authentication <nonce>, constructed as suggested in RFC 2069"
+ ip = request.channel.server.ip
+ now = str(long(time.time()))
+ if now[-1:] == 'L':
+ now = now[:-1]
+ private_key = str (id (self))
+ nonce = string.join ([ip, now, private_key], ':')
+ return self.apply_hash (nonce)
+
+ def apply_hash (self, s):
+ "Apply MD5 to a string <s>, then wrap it in base64 encoding."
+ m = md5()
+ m.update (s)
+ d = m.digest()
+ # base64.encodestring tacks on an extra linefeed.
+ return base64.encodestring (d)[:-1]
+
+ def status (self):
+ # Thanks to mwm at contessa.phone.net (Mike Meyer)
+ r = [
+ producers.simple_producer (
+ '<li>Authorization Extension : '
+ '<b>Unauthorized requests:</b> %s<ul>' % self.fail_count
+ )
+ ]
+ if hasattr (self.handler, 'status'):
+ r.append (self.handler.status())
+ r.append (
+ producers.simple_producer ('</ul>')
+ )
+ return producers.composite_producer(r)
+
+class dictionary_authorizer:
+ def __init__ (self, dict):
+ self.dict = dict
+
+ def authorize (self, auth_info):
+ [username, password] = auth_info
+ if (self.dict.has_key (username)) and (self.dict[username] == password):
+ return 1
+ else:
+ return 0
+
+AUTHORIZATION = re.compile (
+ # scheme challenge
+ 'Authorization: ([^ ]+) (.*)',
+ re.IGNORECASE
+ )
Added: supervisor/trunk/src/medusa/chat_server.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/chat_server.py Fri May 22 23:51:54 2009
@@ -0,0 +1,151 @@
+# -*- Mode: Python -*-
+#
+# Author: Sam Rushing <rushing at nightmare.com>
+# Copyright 1997-2000 by Sam Rushing
+# All Rights Reserved.
+#
+
+RCS_ID = '$Id: chat_server.py,v 1.4 2002/03/20 17:37:48 amk Exp $'
+
+import string
+
+VERSION = string.split(RCS_ID)[2]
+
+import socket
+import asyncore_25 as asyncore
+import asynchat_25 as asynchat
+import status_handler
+
+class chat_channel (asynchat.async_chat):
+
+ def __init__ (self, server, sock, addr):
+ asynchat.async_chat.__init__ (self, sock)
+ self.server = server
+ self.addr = addr
+ self.set_terminator ('\r\n')
+ self.data = ''
+ self.nick = None
+ self.push ('nickname?: ')
+
+ def collect_incoming_data (self, data):
+ self.data = self.data + data
+
+ def found_terminator (self):
+ line = self.data
+ self.data = ''
+ if self.nick is None:
+ self.nick = string.split (line)[0]
+ if not self.nick:
+ self.nick = None
+ self.push ('huh? gimmee a nickname: ')
+ else:
+ self.greet()
+ else:
+ if not line:
+ pass
+ elif line[0] != '/':
+ self.server.push_line (self, line)
+ else:
+ self.handle_command (line)
+
+ def greet (self):
+ self.push ('Hello, %s\r\n' % self.nick)
+ num_channels = len(self.server.channels)-1
+ if num_channels == 0:
+ self.push ('[Kinda lonely in here... you\'re the only caller!]\r\n')
+ else:
+ self.push ('[There are %d other callers]\r\n' % (len(self.server.channels)-1))
+ nicks = map (lambda x: x.get_nick(), self.server.channels.keys())
+ self.push (string.join (nicks, '\r\n ') + '\r\n')
+ self.server.push_line (self, '[joined]')
+
+ def handle_command (self, command):
+ import types
+ command_line = string.split(command)
+ name = 'cmd_%s' % command_line[0][1:]
+ if hasattr (self, name):
+ # make sure it's a method...
+ method = getattr (self, name)
+ if type(method) == type(self.handle_command):
+ method (command_line[1:])
+ else:
+ self.push ('unknown command: %s' % command_line[0])
+
+ def cmd_quit (self, args):
+ self.server.push_line (self, '[left]')
+ self.push ('Goodbye!\r\n')
+ self.close_when_done()
+
+ # alias for '/quit' - '/q'
+ cmd_q = cmd_quit
+
+ def push_line (self, nick, line):
+ self.push ('%s: %s\r\n' % (nick, line))
+
+ def handle_close (self):
+ self.close()
+
+ def close (self):
+ del self.server.channels[self]
+ asynchat.async_chat.close (self)
+
+ def get_nick (self):
+ if self.nick is not None:
+ return self.nick
+ else:
+ return 'Unknown'
+
+class chat_server (asyncore.dispatcher):
+
+ SERVER_IDENT = 'Chat Server (V%s)' % VERSION
+
+ channel_class = chat_channel
+
+ spy = 1
+
+ def __init__ (self, ip='', port=8518):
+ asyncore.dispatcher.__init__(self)
+ self.port = port
+ self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
+ self.bind ((ip, port))
+ print '%s started on port %d' % (self.SERVER_IDENT, port)
+ self.listen (5)
+ self.channels = {}
+ self.count = 0
+
+ def handle_accept (self):
+ conn, addr = self.accept()
+ self.count = self.count + 1
+ print 'client #%d - %s:%d' % (self.count, addr[0], addr[1])
+ self.channels[self.channel_class (self, conn, addr)] = 1
+
+ def push_line (self, from_channel, line):
+ nick = from_channel.get_nick()
+ if self.spy:
+ print '%s: %s' % (nick, line)
+ for c in self.channels.keys():
+ if c is not from_channel:
+ c.push ('%s: %s\r\n' % (nick, line))
+
+ def status (self):
+ lines = [
+ '<h2>%s</h2>' % self.SERVER_IDENT,
+ '<br>Listening on Port: %d' % self.port,
+ '<br><b>Total Sessions:</b> %d' % self.count,
+ '<br><b>Current Sessions:</b> %d' % (len(self.channels))
+ ]
+ return status_handler.lines_producer (lines)
+
+ def writable (self):
+ return 0
+
+if __name__ == '__main__':
+ import sys
+
+ if len(sys.argv) > 1:
+ port = string.atoi (sys.argv[1])
+ else:
+ port = 8518
+
+ s = chat_server ('', port)
+ asyncore.loop()
Added: supervisor/trunk/src/medusa/counter.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/counter.py Fri May 22 23:51:54 2009
@@ -0,0 +1,51 @@
+# -*- Mode: Python -*-
+
+# It is tempting to add an __int__ method to this class, but it's not
+# a good idea. This class tries to gracefully handle integer
+# overflow, and to hide this detail from both the programmer and the
+# user. Note that the __str__ method can be relied on for printing out
+# the value of a counter:
+#
+# >>> print 'Total Client: %s' % self.total_clients
+#
+# If you need to do arithmetic with the value, then use the 'as_long'
+# method, the use of long arithmetic is a reminder that the counter
+# will overflow.
+
+class counter:
+ "general-purpose counter"
+
+ def __init__ (self, initial_value=0):
+ self.value = initial_value
+
+ def increment (self, delta=1):
+ result = self.value
+ try:
+ self.value = self.value + delta
+ except OverflowError:
+ self.value = long(self.value) + delta
+ return result
+
+ def decrement (self, delta=1):
+ result = self.value
+ try:
+ self.value = self.value - delta
+ except OverflowError:
+ self.value = long(self.value) - delta
+ return result
+
+ def as_long (self):
+ return long(self.value)
+
+ def __nonzero__ (self):
+ return self.value != 0
+
+ def __repr__ (self):
+ return '<counter value=%s at %x>' % (self.value, id(self))
+
+ def __str__ (self):
+ s = str(long(self.value))
+ if s[-1:] == 'L':
+ s = s[:-1]
+ return s
+
Added: supervisor/trunk/src/medusa/debian/changelog
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/debian/changelog Fri May 22 23:51:54 2009
@@ -0,0 +1,12 @@
+python2.3-medusa (0.5.4-2) unstable; urgency=low
+
+ * Switch to Python 2.3.
+
+ -- A.M. Kuchling <amk at amk.ca> Thu, 9 Oct 2003 08:22:00 -0400
+
+python2.2-medusa (0.5.4-1) unstable; urgency=low
+
+ * Initial Release.
+
+ -- A.M. Kuchling <amk at amk.ca> Fri, 22 Aug 2003 08:54:11 -0400
+
Added: supervisor/trunk/src/medusa/debian/control
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/debian/control Fri May 22 23:51:54 2009
@@ -0,0 +1,14 @@
+Source: python2.3-medusa
+Priority: optional
+Maintainer: A.M. Kuchling <amk at amk.ca>
+Build-Depends: debhelper (>> 3.0.0)
+Standards-Version: 3.5.8
+Section: net
+
+Package: python2.3-medusa
+Section: net
+Architecture: all
+Depends: python2.3
+Description: Medusa is a 'server platform' -- it provides a framework for
+ implementing asynchronous socket-based servers (TCP/IP and on Unix,
+ Unix domain, sockets).
Added: supervisor/trunk/src/medusa/debian/copyright
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/debian/copyright Fri May 22 23:51:54 2009
@@ -0,0 +1,24 @@
+This package was debianized by A.M. Kuchling <amk at amk.ca> on
+Fri, 22 Aug 2003 08:54:11 -0400.
+
+It was downloaded from www.amk.ca/python/code/medusa.html
+
+Upstream Author: A.M. Kuchling <amk at amk.ca>
+
+Copyright:
+
+Permission to use, copy, modify, and distribute this software and
+its documentation for any purpose and without fee is hereby granted,
+provided that the above copyright notice appear in all copies and
+that both that copyright notice and this permission notice appear in
+supporting documentation, and that the name of Sam Rushing not be
+used in advertising or publicity pertaining to distribution of the
+software without specific, written prior permission.
+
+SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
Added: supervisor/trunk/src/medusa/debian/postinst
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/debian/postinst Fri May 22 23:51:54 2009
@@ -0,0 +1,49 @@
+#! /bin/sh
+# postinst script for medusa
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+# * <postinst> `configure' <most-recently-configured-version>
+# * <old-postinst> `abort-upgrade' <new version>
+# * <conflictor's-postinst> `abort-remove' `in-favour' <package>
+# <new-version>
+# * <deconfigured's-postinst> `abort-deconfigure' `in-favour'
+# <failed-install-package> <version> `removing'
+# <conflicting-package> <version>
+# for details, see http://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+#
+# quoting from the policy:
+# Any necessary prompting should almost always be confined to the
+# post-installation script, and should be protected with a conditional
+# so that unnecessary prompting doesn't happen if a package's
+# installation fails and the `postinst' is called with `abort-upgrade',
+# `abort-remove' or `abort-deconfigure'.
+
+PACKAGE=python2.3-medusa
+VERSION=2.3
+LIB="/usr/lib/python$VERSION"
+DIRLIST="$LIB/site-packages/medusa"
+
+case "$1" in
+ configure|abort-upgrade|abort-remove|abort-deconfigure)
+ for i in $DIRLIST ; do
+ /usr/bin/python$VERSION -O $LIB/compileall.py -q $i
+ /usr/bin/python$VERSION $LIB/compileall.py -q $i
+ done
+ ;;
+
+ *)
+ echo "postinst called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+
+
+exit 0
+
+
Added: supervisor/trunk/src/medusa/debian/prerm
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/debian/prerm Fri May 22 23:51:54 2009
@@ -0,0 +1,26 @@
+#! /bin/sh
+# prerm script for medusa
+
+set -e
+
+PACKAGE=python2.3-medusa
+VERSION=2.3
+LIB="/usr/lib/python$VERSION"
+DIRLIST="$LIB/site-packages/medusa"
+
+case "$1" in
+ remove|upgrade|failed-upgrade)
+ for i in $DIRLIST ; do
+ find $i -name '*.py[co]' -exec rm \{\} \;
+ done
+ ;;
+
+ *)
+ echo "prerm called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+
+
+exit 0
Added: supervisor/trunk/src/medusa/debian/rules
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/debian/rules Fri May 22 23:51:54 2009
@@ -0,0 +1,49 @@
+#!/usr/bin/make -f
+# Sample debian/rules that uses debhelper.
+# GNU copyright 1997 to 1999 by Joey Hess.
+
+# Uncomment this to turn on verbose mode.
+#export DH_VERBOSE=1
+
+# This is the debhelper compatibility version to use.
+export DH_COMPAT=4
+
+
+
+build: build-stamp
+ /usr/bin/python setup.py build
+build-stamp:
+ touch build-stamp
+
+configure:
+ # Do nothing
+
+clean:
+ dh_testdir
+ dh_testroot
+ rm -f build-stamp
+
+ -rm -rf build
+
+ dh_clean
+
+install: build
+ dh_testdir
+ dh_testroot
+ dh_clean -k
+ /usr/bin/python setup.py install --no-compile --prefix=$(CURDIR)/debian/python2.3-medusa/usr
+
+# Build architecture-independent files here.
+binary-indep: install
+ dh_testdir
+ dh_testroot
+
+ dh_installdocs
+ dh_installdeb
+ dh_gencontrol
+ dh_md5sums
+ dh_builddeb
+# We have nothing to do by default.
+
+binary: binary-indep
+.PHONY: build clean binary-indep binary install
Added: supervisor/trunk/src/medusa/default_handler.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/default_handler.py Fri May 22 23:51:54 2009
@@ -0,0 +1,215 @@
+# -*- Mode: Python -*-
+#
+# Author: Sam Rushing <rushing at nightmare.com>
+# Copyright 1997 by Sam Rushing
+# All Rights Reserved.
+#
+
+RCS_ID = '$Id: default_handler.py,v 1.8 2002/08/01 18:15:45 akuchling Exp $'
+
+# standard python modules
+import mimetypes
+import re
+import stat
+import string
+
+# medusa modules
+import http_date
+import http_server
+import status_handler
+import producers
+
+unquote = http_server.unquote
+
+# This is the 'default' handler. it implements the base set of
+# features expected of a simple file-delivering HTTP server. file
+# services are provided through a 'filesystem' object, the very same
+# one used by the FTP server.
+#
+# You can replace or modify this handler if you want a non-standard
+# HTTP server. You can also derive your own handler classes from
+# it.
+#
+# support for handling POST requests is available in the derived
+# class <default_with_post_handler>, defined below.
+#
+
+from counter import counter
+
+class default_handler:
+
+ valid_commands = ['GET', 'HEAD']
+
+ IDENT = 'Default HTTP Request Handler'
+
+ # Pathnames that are tried when a URI resolves to a directory name
+ directory_defaults = [
+ 'index.html',
+ 'default.html'
+ ]
+
+ default_file_producer = producers.file_producer
+
+ def __init__ (self, filesystem):
+ self.filesystem = filesystem
+ # count total hits
+ self.hit_counter = counter()
+ # count file deliveries
+ self.file_counter = counter()
+ # count cache hits
+ self.cache_counter = counter()
+
+ hit_counter = 0
+
+ def __repr__ (self):
+ return '<%s (%s hits) at %x>' % (
+ self.IDENT,
+ self.hit_counter,
+ id (self)
+ )
+
+ # always match, since this is a default
+ def match (self, request):
+ return 1
+
+ # handle a file request, with caching.
+
+ def handle_request (self, request):
+
+ if request.command not in self.valid_commands:
+ request.error (400) # bad request
+ return
+
+ self.hit_counter.increment()
+
+ path, params, query, fragment = request.split_uri()
+
+ if '%' in path:
+ path = unquote (path)
+
+ # strip off all leading slashes
+ while path and path[0] == '/':
+ path = path[1:]
+
+ if self.filesystem.isdir (path):
+ if path and path[-1] != '/':
+ request['Location'] = 'http://%s/%s/' % (
+ request.channel.server.server_name,
+ path
+ )
+ request.error (301)
+ return
+
+ # we could also generate a directory listing here,
+ # may want to move this into another method for that
+ # purpose
+ found = 0
+ if path and path[-1] != '/':
+ path = path + '/'
+ for default in self.directory_defaults:
+ p = path + default
+ if self.filesystem.isfile (p):
+ path = p
+ found = 1
+ break
+ if not found:
+ request.error (404) # Not Found
+ return
+
+ elif not self.filesystem.isfile (path):
+ request.error (404) # Not Found
+ return
+
+ file_length = self.filesystem.stat (path)[stat.ST_SIZE]
+
+ ims = get_header_match (IF_MODIFIED_SINCE, request.header)
+
+ length_match = 1
+ if ims:
+ length = ims.group (4)
+ if length:
+ try:
+ length = string.atoi (length)
+ if length != file_length:
+ length_match = 0
+ except:
+ pass
+
+ ims_date = 0
+
+ if ims:
+ ims_date = http_date.parse_http_date (ims.group (1))
+
+ try:
+ mtime = self.filesystem.stat (path)[stat.ST_MTIME]
+ except:
+ request.error (404)
+ return
+
+ if length_match and ims_date:
+ if mtime <= ims_date:
+ request.reply_code = 304
+ request.done()
+ self.cache_counter.increment()
+ return
+ try:
+ file = self.filesystem.open (path, 'rb')
+ except IOError:
+ request.error (404)
+ return
+
+ request['Last-Modified'] = http_date.build_http_date (mtime)
+ request['Content-Length'] = file_length
+ self.set_content_type (path, request)
+
+ if request.command == 'GET':
+ request.push (self.default_file_producer (file))
+
+ self.file_counter.increment()
+ request.done()
+
+ def set_content_type (self, path, request):
+ ext = string.lower (get_extension (path))
+ typ, encoding = mimetypes.guess_type(path)
+ if typ is not None:
+ request['Content-Type'] = typ
+ else:
+ # TODO: test a chunk off the front of the file for 8-bit
+ # characters, and use application/octet-stream instead.
+ request['Content-Type'] = 'text/plain'
+
+ def status (self):
+ return producers.simple_producer (
+ '<li>%s' % status_handler.html_repr (self)
+ + '<ul>'
+ + ' <li><b>Total Hits:</b> %s' % self.hit_counter
+ + ' <li><b>Files Delivered:</b> %s' % self.file_counter
+ + ' <li><b>Cache Hits:</b> %s' % self.cache_counter
+ + '</ul>'
+ )
+
+# HTTP/1.0 doesn't say anything about the "; length=nnnn" addition
+# to this header. I suppose its purpose is to avoid the overhead
+# of parsing dates...
+IF_MODIFIED_SINCE = re.compile (
+ 'If-Modified-Since: ([^;]+)((; length=([0-9]+)$)|$)',
+ re.IGNORECASE
+ )
+
+USER_AGENT = re.compile ('User-Agent: (.*)', re.IGNORECASE)
+
+CONTENT_TYPE = re.compile (
+ r'Content-Type: ([^;]+)((; boundary=([A-Za-z0-9\'\(\)+_,./:=?-]+)$)|$)',
+ re.IGNORECASE
+ )
+
+get_header = http_server.get_header
+get_header_match = http_server.get_header_match
+
+def get_extension (path):
+ dirsep = string.rfind (path, '/')
+ dotsep = string.rfind (path, '.')
+ if dotsep > dirsep:
+ return path[dotsep+1:]
+ else:
+ return ''
Added: supervisor/trunk/src/medusa/demo/publish.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/demo/publish.py Fri May 22 23:51:54 2009
@@ -0,0 +1,49 @@
+# -*- Mode: Python -*-
+
+# Demonstrates use of the auth and put handlers to support publishing
+# web pages via HTTP.
+
+# It is also possible to set up the ftp server to do essentially the
+# same thing.
+
+# Security Note: Using HTTP with the 'Basic' authentication scheme is
+# only slightly more secure than using FTP: both techniques involve
+# sending a unencrypted password of the network (http basic auth
+# base64-encodes the username and password). The 'Digest' scheme is
+# much more secure, but not widely supported yet. <sigh>
+
+from medusa import asyncore_25 as asyncore
+from medusa import default_handler
+from medusa import http_server
+from medusa import put_handler
+from medusa import auth_handler
+from medusa import filesys
+
+# For this demo, we'll just use a dictionary of usernames/passwords.
+# You can of course use anything that supports the mapping interface,
+# and it would be pretty easy to set this up to use the crypt module
+# on unix.
+
+users = { 'mozart' : 'jupiter', 'beethoven' : 'pastoral' }
+
+# The filesystem we will be giving access to
+fs = filesys.os_filesystem('/home/medusa')
+
+# The 'default' handler - delivers files for the HTTP GET method.
+dh = default_handler.default_handler(fs)
+
+# Supports the HTTP PUT method...
+ph = put_handler.put_handler(fs, '/.*')
+
+# ... but be sure to wrap it with an auth handler:
+ah = auth_handler.auth_handler(users, ph)
+
+# Create a Web Server
+hs = http_server.http_server(ip='', port=8080)
+
+# install the handlers we created:
+
+hs.install_handler(dh) # for GET
+hs.install_handler(ah) # for PUT
+
+asyncore.loop()
Added: supervisor/trunk/src/medusa/demo/script_server.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/demo/script_server.py Fri May 22 23:51:54 2009
@@ -0,0 +1,42 @@
+# -*- Mode: Python -*-
+
+import re, sys
+from medusa import asyncore_25 as asyncore
+from medusa import http_server
+from medusa import default_handler
+from medusa import logger
+from medusa import script_handler
+from medusa import filesys
+
+PUBLISHING_ROOT='/home/medusa'
+CONTENT_LENGTH = re.compile ('Content-Length: ([0-9]+)', re.IGNORECASE)
+
+class sample_input_collector:
+ def __init__ (self, request, length):
+ self.request = request
+ self.length = length
+
+ def collect_incoming_data (self, data):
+ print 'data from %s: <%s>' % (self.request, repr(data))
+
+class post_script_handler (script_handler.script_handler):
+
+ def handle_request (self, request):
+ if request.command == 'post':
+ cl = default_handler.get_header(CONTENT_LENGTH, request.header)
+ ic = sample_input_collector(request, cl)
+ request.collector = ic
+ print request.header
+
+ return script_handler.script_handler.handle_request (self, request)
+
+lg = logger.file_logger (sys.stdout)
+fs = filesys.os_filesystem (PUBLISHING_ROOT)
+dh = default_handler.default_handler (fs)
+ph = post_script_handler (fs)
+hs = http_server.http_server ('', 8081, logger_object = lg)
+
+hs.install_handler (dh)
+hs.install_handler (ph)
+
+asyncore.loop()
Added: supervisor/trunk/src/medusa/demo/simple_anon_ftpd.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/demo/simple_anon_ftpd.py Fri May 22 23:51:54 2009
@@ -0,0 +1,27 @@
+# -*- Mode: Python -*-
+
+from medusa import asyncore_25 as asyncore
+from medusa import ftp_server
+
+# create a 'dummy' authorizer (one that lets everyone in) that returns
+# a read-only filesystem rooted at '/home/ftp'
+
+authorizer = ftp_server.dummy_authorizer('/home/ftp')
+
+# Create an ftp server using this authorizer, running on port 8021
+# [the standard port is 21, but you are probably already running
+# a server there]
+
+fs = ftp_server.ftp_server(authorizer, port=8021)
+
+# Run the async main loop
+asyncore.loop()
+
+# to test this server, try
+# $ ftp myhost 8021
+# when using the standard bsd ftp client,
+# $ ncftp -p 8021 myhost
+# when using ncftp, and
+# ftp://myhost:8021/
+# from a web browser.
+
Added: supervisor/trunk/src/medusa/demo/start_medusa.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/demo/start_medusa.py Fri May 22 23:51:54 2009
@@ -0,0 +1,196 @@
+# -*- Mode: Python -*-
+
+#
+# Sample/Template Medusa Startup Script.
+#
+# This file acts as a configuration file and startup script for Medusa.
+#
+# You should make a copy of this file, then add, change or comment out
+# appropriately. Then you can start up the server by simply typing
+#
+# $ python start_medusa.py
+#
+
+import os
+import sys
+
+from medusa import asyncore_25 as asyncore
+from medusa import http_server
+from medusa import ftp_server
+from medusa import chat_server
+from medusa import monitor
+from medusa import filesys
+from medusa import default_handler
+from medusa import status_handler
+from medusa import resolver
+from medusa import logger
+
+if len(sys.argv) > 1:
+ # process a few convenient arguments
+ [HOSTNAME, IP_ADDRESS, PUBLISHING_ROOT] = sys.argv[1:]
+else:
+ HOSTNAME = 'www.nightmare.com'
+ # This is the IP address of the network interface you want
+ # your servers to be visible from. This can be changed to ''
+ # to listen on all interfaces.
+ IP_ADDRESS = '205.160.176.5'
+
+ # Root of the http and ftp server's published filesystems.
+ PUBLISHING_ROOT = '/home/www'
+
+HTTP_PORT = 8080 # The standard port is 80
+FTP_PORT = 8021 # The standard port is 21
+CHAT_PORT = 8888
+MONITOR_PORT = 9999
+
+# ===========================================================================
+# Caching DNS Resolver
+# ===========================================================================
+# The resolver is used to resolve incoming IP address (for logging),
+# and also to resolve hostnames for HTTP Proxy requests. I recommend
+# using a nameserver running on the local machine, but you can also
+# use a remote nameserver.
+
+rs = resolver.caching_resolver ('127.0.0.1')
+
+# ===========================================================================
+# Logging.
+# ===========================================================================
+
+# There are several types of logging objects. Multiple loggers may be combined,
+# See 'logger.py' for more details.
+
+# This will log to stdout:
+lg = logger.file_logger (sys.stdout)
+
+# This will log to syslog:
+#lg = logger.syslog_logger ('/dev/log')
+
+# This will wrap the logger so that it will
+# 1) keep track of the last 500 entries
+# 2) display an entry in the status report with a hyperlink
+# to view these log entries.
+#
+# If you decide to comment this out, be sure to remove the
+# logger object from the list of status objects below.
+#
+
+lg = status_handler.logger_for_status (lg)
+
+# ===========================================================================
+# Filesystem Object.
+# ===========================================================================
+# An abstraction for the file system. Filesystem objects can be
+# combined and implemented in interesting ways. The default type
+# simply remaps a directory to root.
+
+fs = filesys.os_filesystem (PUBLISHING_ROOT)
+
+# ===========================================================================
+# Default HTTP handler
+# ===========================================================================
+
+# The 'default' handler for the HTTP server is one that delivers
+# files normally - this is the expected behavior of a web server.
+# Note that you needn't use it: Your web server might not want to
+# deliver files!
+
+# This default handler uses the filesystem object we just constructed.
+
+dh = default_handler.default_handler (fs)
+
+# ===========================================================================
+# HTTP Server
+# ===========================================================================
+hs = http_server.http_server (IP_ADDRESS, HTTP_PORT, rs, lg)
+
+# Here we install the default handler created above.
+hs.install_handler (dh)
+
+# ===========================================================================
+# Unix user `public_html' directory support
+# ===========================================================================
+if os.name == 'posix':
+ from medusa import unix_user_handler
+ uh = unix_user_handler.unix_user_handler ('public_html')
+ hs.install_handler (uh)
+
+# ===========================================================================
+# FTP Server
+# ===========================================================================
+
+# Here we create an 'anonymous' ftp server.
+# Note: the ftp server is read-only by default. [in this mode, all
+# 'write-capable' commands are unavailable]
+
+ftp = ftp_server.ftp_server (
+ ftp_server.anon_authorizer (
+ PUBLISHING_ROOT
+ ),
+ ip=IP_ADDRESS,
+ port=FTP_PORT,
+ resolver=rs,
+ logger_object=lg
+ )
+
+# ===========================================================================
+# Monitor Server:
+# ===========================================================================
+
+# This creates a secure monitor server, binding to the loopback
+# address on port 9999, with password 'fnord'. The monitor server
+# can be used to examine and control the server while it is running.
+# If you wish to access the server from another machine, you will
+# need to use '' or some other IP instead of '127.0.0.1'.
+ms = monitor.secure_monitor_server ('fnord', '127.0.0.1', MONITOR_PORT)
+
+# ===========================================================================
+# Chat Server
+# ===========================================================================
+
+# The chat server is a simple IRC-like server: It is meant as a
+# demonstration of how to write new servers and plug them into medusa.
+# It's a very simple server (it took about 2 hours to write), but it
+# could be easily extended. For example, it could be integrated with
+# the web server, perhaps providing navigational tools to browse
+# through a series of discussion groups, listing the number of current
+# users, authentication, etc...
+
+cs = chat_server.chat_server (IP_ADDRESS, CHAT_PORT)
+
+# ===========================================================================
+# Status Handler
+# ===========================================================================
+
+# These are objects that can report their status via the HTTP server.
+# You may comment out any of these, or add more of your own. The only
+# requirement for a 'status-reporting' object is that it have a method
+# 'status' that will return a producer, which will generate an HTML
+# description of the status of the object.
+
+status_objects = [
+ hs,
+ ftp,
+ ms,
+ cs,
+ rs,
+ lg
+ ]
+
+# Create a status handler. By default it binds to the URI '/status'...
+sh = status_handler.status_extension(status_objects)
+# ... and install it on the web server.
+hs.install_handler (sh)
+
+# become 'nobody'
+if os.name == 'posix':
+ if hasattr (os, 'seteuid'):
+ import pwd
+ [uid, gid] = pwd.getpwnam ('nobody')[2:4]
+ os.setegid (gid)
+ os.seteuid (uid)
+
+# Finally, start up the server loop! This loop will not exit until
+# all clients and servers are closed. You may cleanly shut the system
+# down by sending SIGINT (a.k.a. KeyboardInterrupt).
+asyncore.loop()
Added: supervisor/trunk/src/medusa/demo/winFTPserver.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/demo/winFTPserver.py Fri May 22 23:51:54 2009
@@ -0,0 +1,51 @@
+#
+# winFTPServer.py -- FTP server that uses Win32 user API
+#
+# Contributed by John Abel
+#
+# For it to authenticate users correctly, the user running the
+# script must be added to the security policy "Act As Part Of The OS".
+# This is needed for the LogonUser to work. A pain, but something that MS
+# forgot to mention in the API.
+
+
+import win32security, win32con, win32api, win32net
+import ntsecuritycon, pywintypes
+from medusa import asyncore_25 as asyncore
+from medusa import ftp_server, filesys
+
+class Win32Authorizer:
+
+
+ def authorize (self, channel, userName, passWord):
+ self.AdjustPrivilege( ntsecuritycon.SE_CHANGE_NOTIFY_NAME )
+ self.AdjustPrivilege( ntsecuritycon.SE_ASSIGNPRIMARYTOKEN_NAME )
+ self.AdjustPrivilege( ntsecuritycon.SE_TCB_NAME )
+ try:
+ logonHandle = win32security.LogonUser( userName,
+ None,
+ passWord,
+ win32con.LOGON32_LOGON_INTERACTIVE,
+ win32con.LOGON32_PROVIDER_DEFAULT )
+ except pywintypes.error, ErrorMsg:
+ return 0, ErrorMsg[ 2 ], None
+
+ userInfo = win32net.NetUserGetInfo( None, userName, 1 )
+
+ return 1, 'Login successful', filesys.os_filesystem( userInfo[ 'home_dir' ] )
+
+ def AdjustPrivilege( self, priv ):
+ flags = ntsecuritycon.TOKEN_ADJUST_PRIVILEGES | ntsecuritycon.TOKEN_QUERY
+ htoken = win32security.OpenProcessToken(win32api.GetCurrentProcess(), flags)
+ id = win32security.LookupPrivilegeValue(None, priv)
+ newPrivileges = [(id, ntsecuritycon.SE_PRIVILEGE_ENABLED)]
+ win32security.AdjustTokenPrivileges(htoken, 0, newPrivileges)
+
+def start_Server():
+# ftpServ = ftp_server.ftp_server( ftp_server.anon_authorizer( "D:\MyDocuments\MyDownloads"), port=21 )
+ ftpServ = ftp_server.ftp_server( Win32Authorizer(), port=21 )
+ asyncore.loop()
+
+if __name__ == "__main__":
+ print "Starting FTP Server"
+ start_Server()
Added: supervisor/trunk/src/medusa/docs/README.html
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/docs/README.html Fri May 22 23:51:54 2009
@@ -0,0 +1,207 @@
+<html>
+<body>
+
+<h1> What is Medusa? </h1>
+<hr>
+
+<p>
+Medusa is an architecture for very-high-performance TCP/IP servers
+(like HTTP, FTP, and NNTP). Medusa is different from most other
+servers because it runs as a single process, multiplexing I/O with its
+various client and server connections within a single process/thread.
+
+<p>
+It is capable of smoother and higher performance than most other
+servers, while placing a dramatically reduced load on the server
+machine. The single-process, single-thread model simplifies design
+and enables some new persistence capabilities that are otherwise
+difficult or impossible to implement.
+
+<p>
+Medusa is supported on any platform that can run Python and includes a
+functional implementation of the <socket> and <select>
+modules. This includes the majority of Unix implementations.
+
+<p>
+During development, it is constantly tested on Linux and Win32
+[Win95/WinNT], but the core asynchronous capability has been shown to
+work on several other platforms, including the Macintosh. It might
+even work on VMS.
+
+
+<h2>The Power of Python</h2>
+
+<p>
+A distinguishing feature of Medusa is that it is written entirely in
+Python. Python (<a href="http://www.python.org/">http://www.python.org/</a>) is a
+'very-high-level' object-oriented language developed by Guido van
+Rossum (currently at CNRI). It is easy to learn, and includes many
+modern programming features such as storage management, dynamic
+typing, and an extremely flexible object system. It also provides
+convenient interfaces to C and C++.
+
+<p>
+The rapid prototyping and delivery capabilities are hard to exaggerate;
+for example
+<ul>
+
+ <li>It took me longer to read the documentation for persistent HTTP
+ connections (the 'Keep-Alive' connection token) than to add the
+ feature to Medusa.
+
+ <li>A simple IRC-like chat server system was written in about 90 minutes.
+
+</ul>
+
+<p> I've heard similar stories from alpha test sites, and other users of
+the core async library.
+
+<h2>Server Notes</h2>
+
+<p>Both the FTP and HTTP servers use an abstracted 'filesystem object' to
+gain access to a given directory tree. One possible server extension
+technique would be to build behavior into this filesystem object,
+rather than directly into the server: Then the extension could be
+shared with both the FTP and HTTP servers.
+
+<h3>HTTP</h3>
+
+<p>The core HTTP server itself is quite simple - all functionality is
+provided through 'extensions'. Extensions can be plugged in
+dynamically. [i.e., you could log in to the server via the monitor
+service and add or remove an extension on the fly]. The basic
+file-delivery service is provided by a 'default' extension, which
+matches all URI's. You can build more complex behavior by replacing
+or extending this class.
+
+
+<p>The default extension includes support for the 'Connection: Keep-Alive'
+token, and will re-use a client channel when requested by the client.
+
+<h3>FTP</h3>
+
+<p>On Unix, the ftp server includes support for 'real' users, so that it
+may be used as a drop-in replacement for the normal ftp server. Since
+most ftp servers on Unix use the 'forking' model, each child process
+changes its user/group persona after a successful login. This is a
+appears to be a secure design.
+
+
+<p>Medusa takes a different approach - whenever Medusa performs an
+operation for a particular user [listing a directory, opening a file],
+it temporarily switches to that user's persona _only_ for the duration
+of the operation. [and each such operation is protected by a
+try/finally exception handler].
+
+
+<p>To do this Medusa MUST run with super-user privileges. This is a
+HIGHLY experimental approach, and although it has been thoroughly
+tested on Linux, security problems may still exist. If you are
+concerned about the security of your server machine, AND YOU SHOULD
+BE, I suggest running Medusa's ftp server in anonymous-only mode,
+under an account with limited privileges ('nobody' is usually used for
+this purpose).
+
+
+<p>I am very interested in any feedback on this feature, most
+especially information on how the server behaves on different
+implementations of Unix, and of course any security problems that are
+found.
+
+<hr>
+
+<h3>Monitor</h3>
+
+<p>The monitor server gives you remote, 'back-door' access to your server
+while it is running. It implements a remote python interpreter. Once
+connected to the monitor, you can do just about anything you can do from
+the normal python interpreter. You can examine data structures, servers,
+connection objects. You can enable or disable extensions, restart the server,
+reload modules, etc...
+
+<p>The monitor server is protected with an MD5-based authentication
+similar to that proposed in RFC1725 for the POP3 protocol. The server
+sends the client a timestamp, which is then appended to a secret
+password. The resulting md5 digest is sent back to the server, which
+then compares this to the expected result. Failed login attempts are
+logged and immediately disconnected. The password itself is not sent
+over the network (unless you have foolishly transmitted it yourself
+through an insecure telnet or X11 session. 8^)
+
+<p>For this reason telnet cannot be used to connect to the monitor
+server when it is in a secure mode (the default). A client program is
+provided for this purpose. You will be prompted for a password when
+starting up the server, and by the monitor client.
+
+<p>For extra added security on Unix, the monitor server will
+eventually be able to use a Unix-domain socket, which can be protected
+behind a 'firewall' directory (similar to the InterNet News server).
+
+<hr>
+<h2>Performance Notes</h2>
+
+<h3>The <code>select()</code> function</h3>
+
+<p>At the heart of Medusa is a single <code>select()</code> loop.
+This loop handles all open socket connections, both servers and
+clients. It is in effect constantly asking the system: 'which of
+these sockets has activity?'. Performance of this system call can
+vary widely between operating systems.
+
+<p>There are also often builtin limitations to the number of sockets
+('file descriptors') that a single process, or a whole system, can
+manipulate at the same time. Early versions of Linux placed draconian
+limits (256) that have since been raised. Windows 95 has a limit of
+64, while OSF/1 seems to allow up to 4096.
+
+<p>These limits don't affect only Medusa, you will find them described
+in the documentation for other web and ftp servers, too.
+
+<p>The documentation for the Apache web server has some excellent
+notes on tweaking performance for various Unix implementations. See
+<a href="http://www.apache.org/docs/misc/perf.html">
+http://www.apache.org/docs/misc/perf.html</a>
+for more information.
+
+<h3>Buffer sizes</h3>
+
+<p>
+The default buffer sizes used by Medusa are set with a bias toward
+Internet-based servers: They are relatively small, so that the buffer
+overhead for each connection is low. The assumption is that Medusa
+will be talking to a large number of low-bandwidth connections, rather
+than a smaller number of high bandwidth.
+
+<p>This choice trades run-time memory use for efficiency - the down
+side of this is that high-speed local connections (i.e., over a local
+ethernet) will transfer data at a slower rate than necessary.
+
+<p>This parameter can easily be tweaked by the site designer, and can
+in fact be adjusted on a per-server or even per-client basis. For
+example, you could have the FTP server use larger buffer sizes for
+connections from certain domains.
+
+<p>If there's enough interest, I have some rough ideas for how to make
+these buffer sizes automatically adjust to an optimal setting. Send
+email if you'd like to see this feature.
+
+<hr>
+
+<p>See <a href="medusa.html">./medusa.html</a> for a brief overview of
+some of the ideas behind Medusa's design, and for a description of
+current and upcoming features.
+
+<p><h3>Enjoy!</h3>
+
+<hr>
+<br>-Sam Rushing
+<br><a href="mailto:rushing at nightmare.com">rushing at nightmare.com</a>
+
+<!--
+ Local Variables:
+ indent-use-tabs: nil
+ end:
+-->
+
+</body>
+</html>
Added: supervisor/trunk/src/medusa/docs/async_blurbs.txt
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/docs/async_blurbs.txt Fri May 22 23:51:54 2009
@@ -0,0 +1,48 @@
+
+[from the win32 sdk named pipe documentation]
+
+==================================================
+The simplest server process can use the CreateNamedPipe function to
+create a single instance of a pipe, connect to a single client,
+communicate with the client, disconnect the pipe, close the pipe
+handle, and terminate. Typically, however, a server process must
+communicate with multiple client processes. A server process can use a
+single pipe instance by connecting to and disconnecting from each
+client in sequence, but performance would be poor. To handle multiple
+clients simultaneously, the server process must create multiple pipe
+instances.
+
+There are three basic strategies for servicing multiple pipe instances.
+
+· Create multiple threads (and/or processes) with a separate thread
+for each instance of the pipe. For an example of a multithreaded
+server process, see Multithreaded Server.
+
+· Overlap operations by specifying an OVERLAPPED structure in the
+ReadFile, WriteFile, and ConnectNamedPipe functions. For an example of
+a server process that uses overlapped operations, see Server Using
+Overlapped Input and Output.
+
+· Overlap operations by using the ReadFileEx and WriteFileEx
+functions, which specify a completion routine to be executed when the
+operation is complete. For an example of a server process that uses
+completion routines, see Server Using Completion Routines.
+
+
+The multithreaded server strategy is easy to write, because the thread
+for each instance handles communications for only a single client. The
+system allocates processor time to each thread as needed. But each
+thread uses system resources, which is a potential disadvantage for a
+server that handles a large number of clients. Other complications
+occur if the actions of one client necessitate communications with
+other clients (as for a network game program, where a move by one
+player must be communicated to the other players).
+
+With a single-threaded server, it is easier to coordinate operations
+that affect multiple clients, and it is easier to protect shared
+resources (for example, a database file) from simultaneous access by
+multiple clients. The challenge of a single-threaded server is that it
+requires coordination of overlapped operations in order to allocate
+processor time for handling the simultaneous needs of the clients.
+
+==================================================
Added: supervisor/trunk/src/medusa/docs/composing_producers.gif
==============================================================================
Binary file. No diff available.
Added: supervisor/trunk/src/medusa/docs/data_flow.gif
==============================================================================
Binary file. No diff available.
Added: supervisor/trunk/src/medusa/docs/data_flow.html
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/docs/data_flow.html Fri May 22 23:51:54 2009
@@ -0,0 +1,83 @@
+
+<h1>Data Flow in Medusa</h1>
+
+<img src="data_flow.gif">
+
+<p>Data flow, both input and output, is asynchronous. This is
+signified by the <i>request</i> and <i>reply</i> queues in the above
+diagram. This means that both requests and replies can get 'backed
+up', and are still handled correctly. For instance, HTTP/1.1 supports
+the concept of <i>pipelined requests</i>, where a series of requests
+are sent immediately to a server, and the replies are sent as they are
+processed. With a <i>synchronous</i> request, the client would have
+to wait for a reply to each request before sending the next.</p>
+
+<p>The input data is partitioned into requests by looking for a
+<i>terminator</i>. A terminator is simply a protocol-specific
+delimiter - often simply CRLF (carriage-return line-feed), though it
+can be longer (for example, MIME multi-part boundaries can be
+specified as terminators). The protocol handler is notified whenever
+a complete request has been received.</p>
+
+<p>The protocol handler then generates a reply, which is enqueued for
+output back to the client. Sometimes, instead of queuing the actual
+data, an object that will generate this data is used, called a
+<i>producer</i>.</p>
+
+<img src="producers.gif">
+
+<p>The use of <code>producers</code> gives the programmer
+extraordinary control over how output is generated and inserted into
+the output queue. Though they are simple objects (requiring only a
+single method, <i>more()</i>, to be defined), they can be
+<i>composed</i> - simple producers can be wrapped around each other to
+create arbitrarily complex behaviors. [now would be a good time to
+browse through some of the producer classes in
+<code>producers.py</code>.]</p>
+
+<p>The HTTP/1.1 producers make an excellent example. HTTP allows
+replies to be encoded in various ways - for example a reply consisting
+of dynamically-generated output might use the 'chunked' transfer
+encoding to send data that is compressed on-the-fly.</p>
+
+<img src="composing_producers.gif">
+
+<p>In the diagram, green producers actually generate output, and grey
+ones transform it in some manner. This producer might generate output
+looking like this:
+
+<pre>
+ HTTP/1.1 200 OK
+ Content-Encoding: gzip
+ Transfer-Encoding: chunked
+ Header ==> Date: Mon, 04 Aug 1997 21:31:44 GMT
+ Content-Type: text/html
+ Server: Medusa/3.0
+
+ Chunking ==> 0x200
+ Compression ==> <512 bytes of compressed html>
+ 0x200
+ <512 bytes of compressed html>
+ ...
+ 0
+
+</pre>
+
+<p>Still more can be done with this output stream: For the purpose of
+efficiency, it makes sense to send output in large, fixed-size chunks:
+This transformation can be applied by wrapping a 'globbing' producer
+around the whole thing.</p>
+
+<p>An important feature of Medusa's producers is that they are
+actually rather small objects that do not expand into actual output
+data until the moment they are needed: The <code>async_chat</code>
+class will only call on a producer for output when the outgoing socket
+has indicated that it is ready for data. Thus Medusa is extremely
+efficient when faced with network delays, 'hiccups', and low bandwidth
+clients.
+
+<p>One final note: The mechanisms described above are completely
+general - although the examples given demonstrate application to the
+<code>http</code> protocol, Medusa's asynchronous core has been
+applied to many different protocols, including <code>smtp</code>,
+<code>pop3</code>, <code>ftp</code>, and even <code>dns</code>.
Added: supervisor/trunk/src/medusa/docs/debugging.txt
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/docs/debugging.txt Fri May 22 23:51:54 2009
@@ -0,0 +1,59 @@
+===========================================================================
+ The Monitor Server
+===========================================================================
+The monitor server gives the developer a way to get into a server while
+it's running. Here's a quick demonstration of how to get to your server
+objects while in the monitor:
+
+[rushing at gnome medusa]$ python monitor_client.py 127.0.0.1 9999
+Enter Password:
+Python 1.5 (#47, Jul 27 1998, 00:59:35) [GCC egcs-2.90.29 980515 (egcs-1.0.3 release)]
+Copyright 1991-1995 Stichting Mathematisch Centrum, Amsterdam
+Welcome to <secure_monitor_channel connected 127.0.0.1:10775 at 81dbdd8>
+>>> from __main__ import *
+>>> dir()
+['CHAT_PORT', 'FTP_PORT', 'HOSTNAME', 'HTTP_PORT', 'IP_ADDRESS', 'MONITOR_PORT', 'PUBLISHING_ROOT', '__builtins__', 'asyncore', 'chat_server', 'cs', 'debug_mode', 'default_handler', 'dh', 'filesys', 'fs', 'ftp', 'ftp_server', 'hs', 'http_server', 'lg', 'logger', 'monitor', 'ms', 'os', 'resolver', 'rs', 'sh', 'status_handler', 'status_objects', 'sys', 'uh', 'unix_user_handler']
+>>> ms
+<secure_monitor_server listening :9999 at 812a6f8>
+>>> rs
+<caching_resolver at 8133890>
+>>> hs
+<http_server listening :8080 at 81237a0>
+>>> dir(hs)
+['_fileno', 'accept', 'accepting', 'addr', 'bind', 'bytes_in', 'bytes_out', 'close', 'connect', 'connect_ex', 'dup', 'exceptions', 'family_and_type', 'fileno', 'getpeername', 'getsockname', 'getsockopt', 'handlers', 'ip', 'listen', 'logger', 'makefile', 'port', 'recv', 'recvfrom', 'send', 'sendto', 'server_name', 'server_port', 'setblocking', 'setsockopt', 'shutdown', 'socket', 'total_clients', 'total_requests']
+>>> hs.total_clients
+<counter value=0 at 8122468>
+>>> cs
+<chat_server listening :8888 at 8125bf0>
+>>> cs.close()
+log: closing channel 9:<chat_server listening :8888 at 8125bf0>
+>>>
+
+Use 'exit' or 'Ctrl-D' to close a monitor session.
+
+===========================================================================
+ Emergency Debug Mode
+===========================================================================
+
+Bugs and memory leaks in long-running servers may take weeks to show
+up. A useful debugging technique is to leave a crippled or bloated
+server running, but to move the servers' sockets to different ports so
+that you can start up another fresh copy in the meanwhile.
+
+This allows you to debug the servers without forcing you to leave your
+site broken while you do it.
+
+Here's how:
+
+Uncomment the support for '/status/emergency_debug' in <status_handler.py>
+
+When your server goes ballistic, simply hit this URL. This will shut
+down the servers, and restart them on new ports. Add 10000 to the old
+port number, so if you were running a web server on port 80, it will
+be moved to 10080.
+
+Start up another copy of your system, which will run on the original
+ports.
+
+Now you can debug the errant servers in isolation. Beats using GDB!
+
Added: supervisor/trunk/src/medusa/docs/producers.gif
==============================================================================
Binary file. No diff available.
Added: supervisor/trunk/src/medusa/docs/programming.html
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/docs/programming.html Fri May 22 23:51:54 2009
@@ -0,0 +1,646 @@
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN">
+<html>
+ <head>
+ <title>Programming in Python with Medusa and the Async Sockets Library</title>
+ </head>
+
+ <body>
+ <h1>Programming in Python with Medusa and the Async Sockets Library</h1>
+
+ <h2>Introduction</h2>
+ <h3>Why Asynchronous?</h3>
+
+ <p>
+ There are only two ways to have a program on a single processor do
+ 'more than one thing at a time'. Multi-threaded programming is
+ the simplest and most popular way to do it, but there is another
+ very different technique, that lets you have nearly all the
+ advantages of multi-threading, without actually using multiple
+ threads. It's really only practical if your program is <i>I/O
+ bound</i> (I/O is the principle bottleneck). If your program is
+ CPU bound, then pre-emptive scheduled threads are probably what
+ you really need. Network servers are rarely CPU-bound, however.
+ </p>
+
+ <p>
+ If your operating system supports the <code>select()</code>
+ system call in its I/O library (and nearly all do), then you can
+ use it to juggle multiple communication channels at once; doing
+ other work while your I/O is taking place in the "background".
+ Although this strategy can seem strange and complex (especially
+ at first), it is in many ways easier to understand and control
+ than multi-threaded programming. The library documented here
+ solves many of the difficult problems for you, making the task
+ of building sophisticated high-performance network servers and
+ clients a snap.
+ </p>
+
+ <h3>Select-based multiplexing in the real world</h3>
+
+ <p>
+ Several well-known Web servers (and other programs) are written using
+ exactly this technique:
+ the <a href="http://www.acme.com/software/thttpd/">thttpd</a>
+ and <a href="http://www.zeus.co.uk/">Zeus</a>,
+ and <a href="http://squid.nlanr.net/">Squid Internet Object Cache</a> servers
+ are excellent examples..
+ <a href="http://www.isc.org/inn.html">The InterNet News server (INN)</a> used
+ this technique for several years before the web exploded.
+ </p>
+
+ <p>
+ An interesting web server comparison chart is available at the
+ <a href="http://www.acme.com/software/thttpd/benchmarks.html">thttpd web site</a>
+ <p>
+
+ <h3>Variations on a Theme: poll() and WaitForMultipleObjects</h3>
+ <p>
+ Of similar (but better) design is the <code>poll()</code> system
+ call. The main advantage of <code>poll()</code> (for our
+ purposes) is that it does not used fixed-size file-descriptor
+ tables, and is thus more easily scalable than
+ <code>select()</code>. <code>poll()</code> is only recently becoming
+ widely available, so you need to check for availability on your particular
+ operating system.
+ </p>
+
+ <p>
+ In the Windows world, the Win32 API provides a bewildering array
+ of features for multiplexing. Although slightly different in
+ semantics, the combination of Event objects and the
+ <code>WaitForMultipleObjects()</code> interface gives
+ essentially the same power as <code>select()</code> on Unix. A
+ version of this library specific to Win32 has not been written
+ yet, mostly because Win32 also provides <code>select()</code>
+ (at least for sockets). If such an interface were written, it
+ would have the advantage of allowing us to multiplex on other
+ objects types, like named pipes and files.
+ </p>
+
+ <h3>select()</h3>
+
+ <p>
+ Here's what <code>select()</code> does: you pass in a set of
+ file descriptors, in effect asking the operating system, "let me
+ know when anything happens to any of these descriptors". (A
+ <i>descriptor</i> is simply a numeric handle used by the
+ operating system to keep track of a file, socket, pipe, or other
+ I/O object. It is usually an index into a system table of some
+ kind). You can also use a timeout, so that if <i>nothing</i>
+ happens in the allotted period, <code>select()</code> will return
+ control to your program.
+ </p>
+
+ <p>
+ <code>select()</code> takes three <code>fd_set</code> arguments;
+ one for each of the following possible states/events:
+ readability, writability, and exceptional conditions. The last set
+ is less useful than it sounds; in the context of TCP/IP it refers
+ to the presence of out-of-band (OOB) data. OOB is a relatively unportable
+ and poorly used feature that you can (and should) ignore unless you really
+ need it.
+ </p>
+
+ <p>
+ So that leaves only two types of events to build our programs
+ around; <i>read</i> events and <i>write</i> events. As it turns
+ out, this is actually enough to get by with, because other types
+ of events can be implied by the sequencing of these two. It
+ also keeps the low-level interface as simple as possible -
+ always a good thing in my book.
+ </p>
+
+ <h3>The polling loop</h3>
+ <p>
+ Now that you know what <code>select()</code> does, you're ready
+ for the final piece of the puzzle: the main polling loop. This
+ is nothing more than a simple while loop that continually calls
+ <code>select()</code> with a timeout (I usually use a 30-second
+ timeout). Such a program will use virtually no CPU if your
+ server is idle; it spends most of its time letting the operating
+ system do the waiting for it. This is much more efficient than a
+ <a href="http://wombat.doc.ic.ac.uk/foldoc/foldoc.cgi?query=busy-wait">busy-wait</a>
+ loop.
+ </p>
+ <p> Here is a pseudo-code example of a polling loop:
+ <pre>
+while (any_descriptors_left):
+ events = select (descriptors, timeout)
+ for event in events:
+ handle_event (event)
+</pre>
+ <p>
+ If you take a look at the code used by the library, it looks
+ very similar to this. (see the file <code>asyncore.py</code>,
+ the functions poll() and loop()). Now, on to the magic that must
+ take place to handle the events...
+ </p>
+
+ <h2>The Code</h2>
+ <h3>Blocking vs. Non-Blocking</h3>
+ <p>
+ File descriptors can be in either blocking or non-blocking mode.
+ A descriptor in blocking mode will stop (or 'block') your entire
+ program until the requested event takes place. For example, if
+ you ask to read 64 bytes from a descriptor attached to a socket
+ which is ultimately connected to a modem deep in the backwaters
+ of the Internet, you may wait a while for those 64 bytes.
+ </p>
+ <p>
+ If you put the descriptor in non-blocking mode, then one of two
+ things might happen: if the data is sitting in a local buffer,
+ it will be returned to you immediately; otherwise you will get
+ back a code (usually <code>EWOULDBLOCK</code>) telling you that
+ the read is in progress, and you should check back later to see
+ if it's done.
+ <p>
+
+ <h3>sockets vs. other kinds of descriptors</h3>
+
+ <p>
+ Although most of our discussion will be about TCP/IP sockets, on
+ Unix you can use <code>select()</code> to multiplex other kinds
+ of communications objects, like pipes and ttys. (Unfortunately,
+ select() cannot be used to do non-blocking file I/O. Please
+ correct me if you have information to the contrary!)
+ </p>
+
+ <h3>The socket_map</h3>
+ <p>
+ We use a global dictionary (<code>asyncore.socket_map</code>) to
+ keep track of all the active socket objects. The keys for this
+ dictionary are the objects themselves. Nothing is stored in the
+ value slot. Each time through the loop, this dictionary is scanned.
+ Each object is asked which <code>fd_sets</code> it wants to be in.
+ These sets are then passed on to <code>select()</code>.
+ </p>
+
+ <h3>asyncore.dispatcher</h3>
+ <p>
+ The first class we'll introduce you to is the
+ <code>dispatcher</code> class. This is a thin wrapper around a
+ low-level socket object. We have attached a few methods for
+ event-handling to it. Otherwise, it can be treated as a normal
+ non-blocking socket object.
+ </p>
+ <p>
+ The direct interface between the select loop and the socket object
+ are the <code>handle_read_event</code> and <code>handle_write_event</code>
+ methods. These are called whenever an object 'fires' that event.
+ </p>
+ <p>
+ The firing of these low-level events can tell us whether certain
+ higher-level events have taken place, depending on the timing
+ and state of the connection. For example, if we have asked for
+ a socket to connect to another host, we know that the connection
+ has been made when the socket fires a write event (at this point
+ you know that you may write to it with the expectation of
+ success).
+ <br>
+ The implied events are
+ <ul>
+ <li>handle_connect.
+ <br>implied by a write event.
+ <li>handle_close
+ <br>implied by a read event with no data available.
+ <li>handle_accept
+ <br>implied by a read event on a listening socket.
+ </ul>
+ </p>
+ <p>
+ Thus, the set of user-level events is a little larger than simply
+ <code>readable</code> and <code>writeable</code>. The full set of
+ events your code may handle are:
+ <ul>
+ <li>handle_read
+ <li>handle_write
+ <li>handle_expt (OOB data)
+ <li>handle_connect
+ <li>handle_close
+ <li>handle_accept
+ </ul>
+
+ <p>
+ A quick terminology note: In order to distinguish between
+ low-level socket objects and those based on the async library
+ classes, I call these higher-level objects <i>channels</i>.
+
+ </p>
+ <h3>Enough Gibberish, let's write some code</h3>
+ <p>
+ Ok, that's enough abstract talk. Let's do something useful and
+ concrete with this stuff. We'll write a simple HTTP client that
+ demonstrates how easy it is to build a powerful tool in only a few
+ lines of code.
+ </p>
+
+<pre>
+<font color="800000"># -*- Mode: Python; tab-width: 4 -*-</font>
+
+<font color="808000">import</font> asyncore
+<font color="808000">import</font> socket
+<font color="808000">import</font> string
+
+<font color="808000">class</font><font color="000080"> http_client</font> (asyncore.dispatcher):
+
+ <font color="808000">def</font><font color="000080"> __init__</font> (self, host, path):
+ asyncore.dispatcher.__init__ (self)
+ self.path = path
+ self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
+ self.connect ((host, 80))
+
+ <font color="808000">def</font><font color="000080"> handle_connect</font> (self):
+ self.send (<font color="008000">'GET %s HTTP/1.0\r\n\r\n'</font> % self.path)
+
+ <font color="808000">def</font><font color="000080"> handle_read</font> (self):
+ data = self.recv (8192)
+ <font color="808000">print</font> data
+
+ <font color="808000">def</font><font color="000080"> handle_write</font> (self):
+ <font color="808000">pass</font>
+
+<font color="808000">if</font> __name__ == <font color="008000">'__main__'</font>:
+ <font color="808000">import</font> sys
+ <font color="808000">import</font> urlparse
+ <font color="808000">for</font> url <font color="808000">in</font> sys.argv[1:]:
+ parts = urlparse.urlparse (url)
+ <font color="808000">if</font> parts[0] != <font color="008000">'http'</font>:
+ <font color="808000">raise</font> ValueError, <font color="008000">"HTTP URL's only, please"</font>
+ <font color="808000">else</font>:
+ host = parts[1]
+ path = parts[2]
+ http_client (host, path)
+ asyncore.loop()
+
+</pre>
+
+
+<!-- Thanks to Just van Rossum for PyFontify.py! -->
+</pre>
+
+ <p>
+ HTTP is (in theory, at least) a very simple protocol. You connect to the
+ web server, send the string <code>"GET /some/path HTTP/1.0"</code>, and the
+ server will send a short header, followed by the file you asked for. It will
+ then close the connection.
+ <p>
+ We have defined a single new class, <code>http_client</code>, derived
+ from the abstract class <code>asyncore.dispatcher</code>. There are three
+ event handlers defined.<ul>
+ <li><code>handle_connect</code>
+ <br>Once we have made the connection, we send the request string.
+ <li><code>handle_read</code>
+ <br>As the server sends data back to us, we simply print it out.
+ <li><code>handle_write</code>
+ <br>Ignore this for the moment, I'm brushing over a technical detail
+ we'll clean up in a moment.
+ </ul>
+
+ <p>Go ahead and run this demo - giving a single URL as an argument, like this:
+ <p><font color="006000"><code>$ python asynhttp.py http://www.nightmare.com/</code></font>
+ <p>You should see something like this:
+ <p>
+
+<pre>
+[rushing at gnome demo]$ python asynhttp.py http://www.nightmare.com/
+log: adding channel <http_client at 80ef3e8>
+HTTP/1.0 200 OK
+Server: Medusa/3.19
+Content-Type: text/html
+Content-Length: 1649
+Last-Modified: Sun, 26 Jul 1998 23:57:51 GMT
+Date: Sat, 16 Jan 1999 13:04:30 GMT
+
+[... body of the file ...]
+
+log: unhandled close event
+log: closing channel 4:<http_client connected at 80ef3e8>
+
+</pre>
+
+ <p>
+ The 'log' messages are there to help, they are useful when
+ debugging but you will want to disable them later. The first log message
+ tells you that a new <code>http_client</code> object has been added to the
+ socket map. At the end, you'll notice there's a warning that you haven't
+ bothered to handle the <code>close</code> event. No big deal, for now.
+
+ <p>
+ Now at this point we haven't seen anything <i>revolutionary</i>, but that's
+ because we've only looked at one URL. Go ahead and add a few other URL's
+ to the argument list; as many as you like - and make sure they're on different
+ hosts...
+
+ <p>
+ <i>Now</i> you begin to see why <code>select()</code> is so powerful. Depending
+ on your operating system (and its configuration), <code>select()</code> can be
+ fed hundreds, or even thousands of descriptors like this. (I've recently tested
+ <code>select()</code> on a FreeBSD box with over 10,000 descriptors).
+
+ <p>
+ A really good way to understand <code>select()</code> is to put a print statement
+ into the asyncore.poll() function:
+<pre>
+ [...]
+ (r,w,e) = select.select (r,w,e, timeout)
+ print '---'
+ print 'read', r
+ print 'write', w
+ [...]
+</pre>
+
+ <p>
+ Each time through the loop you will see which channels have fired
+ which events. If you haven't skipped ahead, you'll also notice a pointless
+ barrage of events, with all your http_client objects in the 'writable' set.
+ This is because we were a bit lazy earlier; sweeping some ugliness under
+ the rug. Let's fix that now.
+
+ <h3>Buffered Output</h3>
+ <p>
+ In our <code>handle_connect</code>, we cheated a bit by calling
+ <code>send</code> without examining its return code. In truth,
+ since we are using a non-blocking socket, it's (theoretically)
+ possible that our data didn't get sent. To do this <i>correctly</i>,
+ we actually need to set up a buffer of outgoing data, and then send
+ as much of the buffer as we can whenever we see a <code>write</code>
+ event:
+
+<pre>
+
+<font color="808000">class</font><font color="000080"> http_client</font> (asyncore.dispatcher):
+
+ <font color="808000">def</font><font color="000080"> __init__</font> (self, host, path):
+ asyncore.dispatcher.__init__ (self)
+ self.path = path
+ self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
+ self.connect ((host, 80))
+ self.buffer = <font color="008000">'GET %s HTTP/1.0\r\n\r\n'</font> % self.path
+
+ <font color="808000">def</font><font color="000080"> handle_connect</font> (self):
+ <font color="808000">pass</font>
+
+ <font color="808000">def</font><font color="000080"> handle_read</font> (self):
+ data = self.recv (8192)
+ <font color="808000">print</font> data
+
+ <font color="808000">def</font><font color="000080"> writable</font> (self):
+ <font color="808000">return</font> (len(self.buffer) > 0)
+
+ <font color="808000">def</font><font color="000080"> handle_write</font> (self):
+ sent = self.send (self.buffer)
+ self.buffer = self.buffer[sent:]
+</pre>
+
+ <p>
+ The <code>handle_connect</code> method no longer assumes it can
+ send its request string successfully. We move its work over to
+ <code>handle_write</code>; which trims <code>self.buffer</code>
+ as pieces of it are sent succesfully.
+
+ <p>
+ We also introduce the <code>writable</code> method. Each time
+ through the loop, the set of sockets is scanned, the
+ <code>readable</code> and <code>writable</code> methods of each
+ object are called to see if are interested in those events. The
+ default methods simply return 1, indicating that by default all
+ channels will be in both sets. In this case, however, we are only
+ interested in writing as long as we have something to write. So
+ we override this method, making its behavior dependent on the length
+ of <code>self.buffer</code>.
+
+ <p>
+ If you try the client now (with the print statements in
+ <code>asyncore.poll()</code>), you'll see that
+ <code>select</code> is firing more efficiently.
+
+ <h3>asynchat.py</h3>
+ <p>
+ The dispatcher class is useful, but somewhat limited in
+ capability. As you might guess, managing input and output
+ buffers manually can get complex, especially if you're working
+ with a protocol more complicated than HTTP.
+
+ <p>
+ The <code>async_chat</code> class does a lot of the heavy
+ lifting for you. It automatically handles the buffering of both
+ input and output, and provides a "line terminator" facility that
+ partitions an input stream into logical lines for you. It is
+ also carefully designed to support <i>pipelining</i> - a nice
+ feature that we'll explain later.
+
+ <p>
+ There are four new methods to introduce:
+ <ul>
+
+ <li><code>set_terminator (self, <eol-string>)</code>
+ <br>Set the string used to identify <i>end-of-line</i>. For most
+ Internet protocols, this is the string <code>\r\n</code>, that is;
+ a carriage return followed by a line feed. To turn off input scanning,
+ use <code>None</code>
+
+ <li><code>collect_incoming_data (self, data)</code>
+ <br>Called whenever data is available from
+ a socket. Usually, your implementation will accumulate this
+ data into a buffer of some kind.
+
+ <li><code>found_terminator (self)</code>
+ <br>Called whenever an end-of-line marker has been seen. Typically
+ your code will process and clear the input buffer.
+
+ <li><code>push (data)</code>
+ <br>This is a buffered version of <code>send</code>. It will place
+ the data in an outgoing buffer.
+
+ </ul>
+ <p>
+ These methods build on the underlying capabilities of
+ <code>dispatcher</code> by providing implementations of
+ <code>handle_read</code> <code>handle_write</code>, etc...
+ <code>handle_read</code> collects data into an input buffer, which
+ is continually scanned for the terminator string. Data in between
+ terminators is feed to your <code>collect_incoming_data</code> method.
+
+ <p>
+ The implementation of <code>handle_write</code> and <code>writable</code>
+ examine an outgoing-data queue, and automatically send data whenever
+ possible.
+
+ <h3>A Proxy Server</h3>
+ In order to demonstrate the <code>async_chat</code> class, we will
+ put together a simple proxy server. A proxy server combines a server
+ and a client together, in effect sitting between the real server and
+ client. You can use this to monitor or debug protocol traffic.
+
+<pre>
+
+<font color="800000"># -*- Mode: Python; tab-width: 4 -*-</font>
+
+<font color="808000">import</font> asynchat
+<font color="808000">import</font> asyncore
+<font color="808000">import</font> socket
+<font color="808000">import</font> string
+
+<font color="808000">class</font><font color="000080"> proxy_server</font> (asyncore.dispatcher):
+
+ <font color="808000">def</font><font color="000080"> __init__</font> (self, host, port):
+ asyncore.dispatcher.__init__ (self)
+ self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
+ self.set_reuse_addr()
+ self.there = (host, port)
+ here = (<font color="008000">''</font>, port + 8000)
+ self.bind (here)
+ self.listen (5)
+
+ <font color="808000">def</font><font color="000080"> handle_accept</font> (self):
+ proxy_receiver (self, self.accept())
+
+<font color="808000">class</font><font color="000080"> proxy_sender</font> (asynchat.async_chat):
+
+ <font color="808000">def</font><font color="000080"> __init__</font> (self, receiver, address):
+ asynchat.async_chat.__init__ (self)
+ self.receiver = receiver
+ self.set_terminator (None)
+ self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
+ self.buffer = <font color="008000">''</font>
+ self.set_terminator (<font color="008000">'\n'</font>)
+ self.connect (address)
+
+ <font color="808000">def</font><font color="000080"> handle_connect</font> (self):
+ <font color="808000">print</font> <font color="008000">'Connected'</font>
+
+ <font color="808000">def</font><font color="000080"> collect_incoming_data</font> (self, data):
+ self.buffer = self.buffer + data
+
+ <font color="808000">def</font><font color="000080"> found_terminator</font> (self):
+ data = self.buffer
+ self.buffer = <font color="008000">''</font>
+ <font color="808000">print</font> <font color="008000">'==> (%d) %s'</font> % (self.id, repr(data))
+ self.receiver.push (data + <font color="008000">'\n'</font>)
+
+ <font color="808000">def</font><font color="000080"> handle_close</font> (self):
+ self.receiver.close()
+ self.close()
+
+<font color="808000">class</font><font color="000080"> proxy_receiver</font> (asynchat.async_chat):
+
+ channel_counter = 0
+
+ <font color="808000">def</font><font color="000080"> __init__</font> (self, server, (conn, addr)):
+ asynchat.async_chat.__init__ (self, conn)
+ self.set_terminator (<font color="008000">'\n'</font>)
+ self.server = server
+ self.id = self.channel_counter
+ self.channel_counter = self.channel_counter + 1
+ self.sender = proxy_sender (self, server.there)
+ self.sender.id = self.id
+ self.buffer = <font color="008000">''</font>
+
+ <font color="808000">def</font><font color="000080"> collect_incoming_data</font> (self, data):
+ self.buffer = self.buffer + data
+
+ <font color="808000">def</font><font color="000080"> found_terminator</font> (self):
+ data = self.buffer
+ self.buffer = <font color="008000">''</font>
+ <font color="808000">print</font> <font color="008000">'<== (%d) %s'</font> % (self.id, repr(data))
+ self.sender.push (data + <font color="008000">'\n'</font>)
+
+ <font color="808000">def</font><font color="000080"> handle_close</font> (self):
+ <font color="808000">print</font> <font color="008000">'Closing'</font>
+ self.sender.close()
+ self.close()
+
+<font color="808000">if</font> __name__ == <font color="008000">'__main__'</font>:
+ <font color="808000">import</font> sys
+ <font color="808000">import</font> string
+ <font color="808000">if</font> len(sys.argv) < 3:
+ <font color="808000">print</font> <font color="008000">'Usage: %s <server-host> <server-port>'</font> % sys.argv[0]
+ <font color="808000">else</font>:
+ ps = proxy_server (sys.argv[1], string.atoi (sys.argv[2]))
+ asyncore.loop()
+
+</pre>
+
+ <p>
+ To try out the proxy, find a server (any SMTP, NNTP, or HTTP server should do fine),
+ and give its hostname and port as arguments:
+
+<pre>
+python proxy.py localhost 25
+</pre>
+
+ <p>
+ The proxy server will start up its server on port <code>n +
+ 8000</code>, in this case port 8025. Now, use a telnet program
+ to connect to that port on your server host. Issue a few
+ commands. See how the whole session is being echoed by your
+ proxy server. Try opening up several simultaneous connections
+ through your proxy. You might also try pointing a real client
+ (a news reader [port 119] or web browser [port 80]) at your proxy.
+
+ <h3>Pipelining</h3>
+ <p>
+ Pipelining refers to a protocol capability. Normally, a conversation
+ with a server has a back-and-forth quality to it. The client sends a
+ command, and waits for the response. If a client needs to send many commands
+ over a high-latency connection, waiting for each response can take a long
+ time.
+ <p>
+ For example, when sending a mail message to many recipients with
+ SMTP, the client will send a series of <code>RCPT</code>
+ commands, one for each recipient. For each of these commands,
+ the server will send back a reply indicating whether the mailbox
+ specified is valid. If you want to send a message to several
+ hundred recipients, this can be rather tedious if the round-trip
+ time for each command is long. You'd like to be able to send a
+ bunch of <code>RCPT</code> commands in one batch, and then count
+ off the responses to them as they come.
+
+ <p>
+ I have a favorite visual when explaining the advantages of
+ pipelining. Imagine each request to the server is a boxcar on a
+ train. The client is in Los Angeles, and the server is in New
+ York. Pipelining lets you hook all your cars in one long chain;
+ send them to New York, where they are filled and sent back to you.
+ Without pipelining you have to send one car at a time.
+
+ <p>
+ Not all protocols allow pipelining. Not all servers support it;
+ Sendmail, for example, does not support pipelining because it tends
+ to fork unpredictably, leaving buffered data in a questionable state.
+ A recent extension to the SMTP protocol allows a server to specify
+ whether it supports pipelining. HTTP/1.1 explicitly requires that
+ a server support pipelining.
+
+ <p>
+ Servers built on top of <code>async_chat</code> automatically
+ support pipelining. It is even possible to change the
+ terminator repeatedly when processing data already in the
+ input buffer. See the <code>handle_read</code> method if you're
+ interested in the gory details.
+
+ <h3>Producers</h3>
+
+ <p>
+ <code>async_chat</code> supports a sophisticated output
+ buffering model, using a queue of data-producing objects. For
+ most purposes, you will use the <code>push()</code> method to
+ send string data - but for more sophisticated usage you can push
+ a <code>producer</code>
+
+ <p>
+ A <code>producer</code> is a very simple object, requiring only
+ a single method in its implementation, <code>more()</code>. See
+ the code for <code>simple_producer</code> in
+ <code>asynchat.py</code> for an example. Many more examples are
+ available in the Medusa distribution, in the file
+ <code>producers.py</code>
+
+ <hr>
+ <address><a href="mailto:rushing at nightmare.com">Samual M. Rushing</a></address>
+<!-- Created: Mon Jan 11 03:53:15 PST 1999 -->
+<!-- hhmts start -->
+Last modified: Fri Apr 30 21:42:52 PDT 1999
+<!-- hhmts end -->
+ </body>
+</html>
Added: supervisor/trunk/src/medusa/docs/proxy_notes.txt
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/docs/proxy_notes.txt Fri May 22 23:51:54 2009
@@ -0,0 +1,36 @@
+
+# we can build 'promises' to produce external data. Each producer
+# contains a 'promise' to fetch external data (or an error
+# message). writable() for that channel will only return true if the
+# top-most producer is ready. This state can be flagged by the dns
+# client making a callback.
+
+# So, say 5 proxy requests come in, we can send out DNS queries for
+# them immediately. If the replies to these come back before the
+# promises get to the front of the queue, so much the better: no
+# resolve delay. 8^)
+#
+# ok, there's still another complication:
+# how to maintain replies in order?
+# say three requests come in, (to different hosts? can this happen?)
+# yet the connections happen third, second, and first. We can't buffer
+# the entire request! We need to be able to specify how much to buffer.
+#
+# ===========================================================================
+#
+# the current setup is a 'pull' model: whenever the channel fires FD_WRITE,
+# we 'pull' data from the producer fifo. what we need is a 'push' option/mode,
+# where
+# 1) we only check for FD_WRITE when data is in the buffer
+# 2) whoever is 'pushing' is responsible for calling 'refill_buffer()'
+#
+# what is necessary to support this 'mode'?
+# 1) writable() only fires when data is in the buffer
+# 2) refill_buffer() is only called by the 'pusher'.
+#
+# how would such a mode affect things? with this mode could we support
+# a true http/1.1 proxy? [i.e, support <n> pipelined proxy requests, possibly
+# to different hosts, possibly even mixed in with non-proxy requests?] For
+# example, it would be nice if we could have the proxy automatically apply the
+# 1.1 chunking for 1.0 close-on-eof replies when feeding it to the client. This
+# would let us keep our persistent connection.
Added: supervisor/trunk/src/medusa/docs/threads.txt
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/docs/threads.txt Fri May 22 23:51:54 2009
@@ -0,0 +1,57 @@
+# -*- Mode: Text; tab-width: 4 -*-
+
+[note, a better solution is now available, see the various modules in
+ the 'thread' directory (SMR 990105)]
+
+ A Workable Approach to Mixing Threads and Medusa.
+---------------------------------------------------------------------------
+
+When Medusa receives a request that needs to be handled by a separate
+thread, have the thread remove the socket from Medusa's control, by
+calling the 'del_channel()' method, and put the socket into
+blocking-mode:
+
+ request.channel.del_channel()
+ request.channel.socket.setblocking (0)
+
+Now your thread is responsible for managing the rest of the HTTP
+'session'. In particular, you need to send the HTTP response, followed
+by any headers, followed by the response body.
+
+Since the most common need for mixing threads and Medusa is to support
+CGI, there's one final hurdle that should be pointed out: CGI scripts
+sometimes make use of a 'Status:' hack (oops, I meant to say 'header')
+in order to tell the server to return a reply other than '200 OK'. To
+support this it is necessary to scan the output _before_ it is sent.
+Here is a sample 'write' method for a file-like object that performs
+this scan:
+
+HEADER_LINE = regex.compile ('\([A-Za-z0-9-]+\): \(.*\)')
+
+ def write (self, data):
+ if self.got_header:
+ self._write (data)
+ else:
+ # CGI scripts may optionally provide extra headers.
+ #
+ # If they do not, then the output is assumed to be
+ # text/html, with an HTTP reply code of '200 OK'.
+ #
+ # If they do, we need to scan those headers for one in
+ # particular: the 'Status:' header, which will tell us
+ # to use a different HTTP reply code [like '302 Moved']
+ #
+ self.buffer = self.buffer + data
+ lines = string.split (self.buffer, '\n')
+ # look for something un-header-like
+ for i in range(len(lines)):
+ if i == (len(lines)-1):
+ if lines[i] == '':
+ break
+ elif HEADER_LINE.match (lines[i]) == -1:
+ # this is not a header line.
+ self.got_header = 1
+ self.buffer = self.build_header (lines[:i])
+ # rejoin the rest of the data
+ self._write (string.join (lines[i:], '\n'))
+ break
Added: supervisor/trunk/src/medusa/docs/tkinter.txt
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/docs/tkinter.txt Fri May 22 23:51:54 2009
@@ -0,0 +1,31 @@
+
+Here are some notes on combining the Tk Event loop with the async lib
+and/or Medusa. Many thanks to Aaron Rhodes (alrhodes at cpis.net) for
+the info!
+
+ > Sam,
+ >
+ > Just wanted to send you a quick message about how I managed to
+ > finally integrate Tkinter with asyncore. This solution is pretty
+ > straightforward. From the main tkinter event loop i simply added
+ > a repeating alarm that calls asyncore.poll() every so often. So
+ > the code looks like this:
+ >
+ > in main:
+ > import asyncore
+ >
+ > self.socket_check()
+ >
+ > ...
+ >
+ > then, socket_check() is:
+ >
+ > def socket_check(self):
+ > asyncore.poll(timeout=0.0)
+ > self.after(100, self.socket_check)
+ >
+ >
+ > This simply causes asyncore to poll all the sockets every 100ms
+ > during the tkinter event loop. The GUI doesn't block on IO since
+ > all the IO calls are now handled with asyncore.
+
Added: supervisor/trunk/src/medusa/event_loop.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/event_loop.py Fri May 22 23:51:54 2009
@@ -0,0 +1,93 @@
+# -*- Mode: Python -*-
+
+# This is an alternative event loop that supports 'schedulable events'.
+# You can specify an event callback to take place after <n> seconds.
+
+# Important usage note: The granularity of the time-check is limited
+# by the <timeout> argument to 'go()'; if there is little or no
+# activity and you specify a 30-second timeout interval, then the
+# schedule of events may only be checked at those 30-second intervals.
+# In other words, if you need 1-second resolution, you will have to
+# poll at 1-second intervals. This facility is more useful for longer
+# timeouts ("if the channel doesn't close in 5 minutes, then forcibly
+# close it" would be a typical usage).
+
+import asyncore_25 as asyncore
+import bisect
+import time
+
+socket_map = asyncore.socket_map
+
+class event_loop:
+
+ def __init__ (self):
+ self.events = []
+ self.num_channels = 0
+ self.max_channels = 0
+
+ def go (self, timeout=30.0, granularity=15):
+ global socket_map
+ last_event_check = 0
+ while socket_map:
+ now = int(time.time())
+ if (now - last_event_check) >= granularity:
+ last_event_check = now
+ fired = []
+ # yuck. i want my lisp.
+ i = j = 0
+ while i < len(self.events):
+ when, what = self.events[i]
+ if now >= when:
+ fired.append (what)
+ j = i + 1
+ else:
+ break
+ i = i + 1
+ if fired:
+ self.events = self.events[j:]
+ for what in fired:
+ what (self, now)
+ # sample the number of channels
+ n = len(asyncore.socket_map)
+ self.num_channels = n
+ if n > self.max_channels:
+ self.max_channels = n
+ asyncore.poll (timeout)
+
+ def schedule (self, delta, callback):
+ now = int (time.time())
+ bisect.insort (self.events, (now + delta, callback))
+
+ def __len__ (self):
+ return len(self.events)
+
+class test (asyncore.dispatcher):
+
+ def __init__ (self):
+ asyncore.dispatcher.__init__ (self)
+
+ def handle_connect (self):
+ print 'Connected!'
+
+ def writable (self):
+ return not self.connected
+
+ def connect_timeout_callback (self, event_loop, when):
+ if not self.connected:
+ print 'Timeout on connect'
+ self.close()
+
+ def periodic_thing_callback (self, event_loop, when):
+ print 'A Periodic Event has Occurred!'
+ # re-schedule it.
+ event_loop.schedule (self, 15, self.periodic_thing_callback)
+
+if __name__ == '__main__':
+ import socket
+ el = event_loop()
+ t = test ()
+ t.create_socket (socket.AF_INET, socket.SOCK_STREAM)
+ el.schedule (10, t.connect_timeout_callback)
+ el.schedule (15, t.periodic_thing_callback)
+ t.connect (('squirl', 80))
+ el.go(1.0)
Added: supervisor/trunk/src/medusa/filesys.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/filesys.py Fri May 22 23:51:54 2009
@@ -0,0 +1,398 @@
+# -*- Mode: Python -*-
+# $Id: filesys.py,v 1.9 2003/12/24 16:10:56 akuchling Exp $
+# Author: Sam Rushing <rushing at nightmare.com>
+#
+# Generic filesystem interface.
+#
+
+# We want to provide a complete wrapper around any and all
+# filesystem operations.
+
+# this class is really just for documentation,
+# identifying the API for a filesystem object.
+
+# opening files for reading, and listing directories, should
+# return a producer.
+
+class abstract_filesystem:
+ def __init__ (self):
+ pass
+
+ def current_directory (self):
+ "Return a string representing the current directory."
+ pass
+
+ def listdir (self, path, long=0):
+ """Return a listing of the directory at 'path' The empty string
+ indicates the current directory. If 'long' is set, instead
+ return a list of (name, stat_info) tuples
+ """
+ pass
+
+ def open (self, path, mode):
+ "Return an open file object"
+ pass
+
+ def stat (self, path):
+ "Return the equivalent of os.stat() on the given path."
+ pass
+
+ def isdir (self, path):
+ "Does the path represent a directory?"
+ pass
+
+ def isfile (self, path):
+ "Does the path represent a plain file?"
+ pass
+
+ def cwd (self, path):
+ "Change the working directory."
+ pass
+
+ def cdup (self):
+ "Change to the parent of the current directory."
+ pass
+
+
+ def longify (self, path):
+ """Return a 'long' representation of the filename
+ [for the output of the LIST command]"""
+ pass
+
+# standard wrapper around a unix-like filesystem, with a 'false root'
+# capability.
+
+# security considerations: can symbolic links be used to 'escape' the
+# root? should we allow it? if not, then we could scan the
+# filesystem on startup, but that would not help if they were added
+# later. We will probably need to check for symlinks in the cwd method.
+
+# what to do if wd is an invalid directory?
+
+import os
+import stat
+import re
+import string
+
+def safe_stat (path):
+ try:
+ return (path, os.stat (path))
+ except:
+ return None
+
+import glob
+
+class os_filesystem:
+ path_module = os.path
+
+ # set this to zero if you want to disable pathname globbing.
+ # [we currently don't glob, anyway]
+ do_globbing = 1
+
+ def __init__ (self, root, wd='/'):
+ self.root = root
+ self.wd = wd
+
+ def current_directory (self):
+ return self.wd
+
+ def isfile (self, path):
+ p = self.normalize (self.path_module.join (self.wd, path))
+ return self.path_module.isfile (self.translate(p))
+
+ def isdir (self, path):
+ p = self.normalize (self.path_module.join (self.wd, path))
+ return self.path_module.isdir (self.translate(p))
+
+ def cwd (self, path):
+ p = self.normalize (self.path_module.join (self.wd, path))
+ translated_path = self.translate(p)
+ if not self.path_module.isdir (translated_path):
+ return 0
+ else:
+ old_dir = os.getcwd()
+ # temporarily change to that directory, in order
+ # to see if we have permission to do so.
+ try:
+ can = 0
+ try:
+ os.chdir (translated_path)
+ can = 1
+ self.wd = p
+ except:
+ pass
+ finally:
+ if can:
+ os.chdir (old_dir)
+ return can
+
+ def cdup (self):
+ return self.cwd ('..')
+
+ def listdir (self, path, long=0):
+ p = self.translate (path)
+ # I think we should glob, but limit it to the current
+ # directory only.
+ ld = os.listdir (p)
+ if not long:
+ return list_producer (ld, None)
+ else:
+ old_dir = os.getcwd()
+ try:
+ os.chdir (p)
+ # if os.stat fails we ignore that file.
+ result = filter (None, map (safe_stat, ld))
+ finally:
+ os.chdir (old_dir)
+ return list_producer (result, self.longify)
+
+ # TODO: implement a cache w/timeout for stat()
+ def stat (self, path):
+ p = self.translate (path)
+ return os.stat (p)
+
+ def open (self, path, mode):
+ p = self.translate (path)
+ return open (p, mode)
+
+ def unlink (self, path):
+ p = self.translate (path)
+ return os.unlink (p)
+
+ def mkdir (self, path):
+ p = self.translate (path)
+ return os.mkdir (p)
+
+ def rmdir (self, path):
+ p = self.translate (path)
+ return os.rmdir (p)
+
+ def rename(self, src, dst):
+ return os.rename(self.translate(src),self.translate(dst))
+
+ # utility methods
+ def normalize (self, path):
+ # watch for the ever-sneaky '/+' path element
+ path = re.sub('/+', '/', path)
+ p = self.path_module.normpath (path)
+ # remove 'dangling' cdup's.
+ if len(p) > 2 and p[:3] == '/..':
+ p = '/'
+ return p
+
+ def translate (self, path):
+ # we need to join together three separate
+ # path components, and do it safely.
+ # <real_root>/<current_directory>/<path>
+ # use the operating system's path separator.
+ path = string.join (string.split (path, '/'), os.sep)
+ p = self.normalize (self.path_module.join (self.wd, path))
+ p = self.normalize (self.path_module.join (self.root, p[1:]))
+ return p
+
+ def longify (self, (path, stat_info)):
+ return unix_longify (path, stat_info)
+
+ def __repr__ (self):
+ return '<unix-style fs root:%s wd:%s>' % (
+ self.root,
+ self.wd
+ )
+
+if os.name == 'posix':
+
+ class unix_filesystem (os_filesystem):
+ pass
+
+ class schizophrenic_unix_filesystem (os_filesystem):
+ PROCESS_UID = os.getuid()
+ PROCESS_EUID = os.geteuid()
+ PROCESS_GID = os.getgid()
+ PROCESS_EGID = os.getegid()
+
+ def __init__ (self, root, wd='/', persona=(None, None)):
+ os_filesystem.__init__ (self, root, wd)
+ self.persona = persona
+
+ def become_persona (self):
+ if self.persona is not (None, None):
+ uid, gid = self.persona
+ # the order of these is important!
+ os.setegid (gid)
+ os.seteuid (uid)
+
+ def become_nobody (self):
+ if self.persona is not (None, None):
+ os.seteuid (self.PROCESS_UID)
+ os.setegid (self.PROCESS_GID)
+
+ # cwd, cdup, open, listdir
+ def cwd (self, path):
+ try:
+ self.become_persona()
+ return os_filesystem.cwd (self, path)
+ finally:
+ self.become_nobody()
+
+ def cdup (self, path):
+ try:
+ self.become_persona()
+ return os_filesystem.cdup (self)
+ finally:
+ self.become_nobody()
+
+ def open (self, filename, mode):
+ try:
+ self.become_persona()
+ return os_filesystem.open (self, filename, mode)
+ finally:
+ self.become_nobody()
+
+ def listdir (self, path, long=0):
+ try:
+ self.become_persona()
+ return os_filesystem.listdir (self, path, long)
+ finally:
+ self.become_nobody()
+
+# For the 'real' root, we could obtain a list of drives, and then
+# use that. Doesn't win32 provide such a 'real' filesystem?
+# [yes, I think something like this "\\.\c\windows"]
+
+class msdos_filesystem (os_filesystem):
+ def longify (self, (path, stat_info)):
+ return msdos_longify (path, stat_info)
+
+# A merged filesystem will let you plug other filesystems together.
+# We really need the equivalent of a 'mount' capability - this seems
+# to be the most general idea. So you'd use a 'mount' method to place
+# another filesystem somewhere in the hierarchy.
+
+# Note: this is most likely how I will handle ~user directories
+# with the http server.
+
+class merged_filesystem:
+ def __init__ (self, *fsys):
+ pass
+
+# this matches the output of NT's ftp server (when in
+# MSDOS mode) exactly.
+
+def msdos_longify (file, stat_info):
+ if stat.S_ISDIR (stat_info[stat.ST_MODE]):
+ dir = '<DIR>'
+ else:
+ dir = ' '
+ date = msdos_date (stat_info[stat.ST_MTIME])
+ return '%s %s %8d %s' % (
+ date,
+ dir,
+ stat_info[stat.ST_SIZE],
+ file
+ )
+
+def msdos_date (t):
+ try:
+ info = time.gmtime (t)
+ except:
+ info = time.gmtime (0)
+ # year, month, day, hour, minute, second, ...
+ hour = info[3]
+ if hour > 11:
+ merid = 'PM'
+ hour = hour - 12
+ else:
+ merid = 'AM'
+ return '%02d-%02d-%02d %02d:%02d%s' % (
+ info[1],
+ info[2],
+ info[0]%100,
+ hour,
+ info[4],
+ merid
+ )
+
+months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
+ 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
+
+mode_table = {
+ '0':'---',
+ '1':'--x',
+ '2':'-w-',
+ '3':'-wx',
+ '4':'r--',
+ '5':'r-x',
+ '6':'rw-',
+ '7':'rwx'
+ }
+
+import time
+
+def unix_longify (file, stat_info):
+ # for now, only pay attention to the lower bits
+ mode = ('%o' % stat_info[stat.ST_MODE])[-3:]
+ mode = string.join (map (lambda x: mode_table[x], mode), '')
+ if stat.S_ISDIR (stat_info[stat.ST_MODE]):
+ dirchar = 'd'
+ else:
+ dirchar = '-'
+ date = ls_date (long(time.time()), stat_info[stat.ST_MTIME])
+ return '%s%s %3d %-8d %-8d %8d %s %s' % (
+ dirchar,
+ mode,
+ stat_info[stat.ST_NLINK],
+ stat_info[stat.ST_UID],
+ stat_info[stat.ST_GID],
+ stat_info[stat.ST_SIZE],
+ date,
+ file
+ )
+
+# Emulate the unix 'ls' command's date field.
+# it has two formats - if the date is more than 180
+# days in the past, then it's like this:
+# Oct 19 1995
+# otherwise, it looks like this:
+# Oct 19 17:33
+
+def ls_date (now, t):
+ try:
+ info = time.gmtime (t)
+ except:
+ info = time.gmtime (0)
+ # 15,600,000 == 86,400 * 180
+ if (now - t) > 15600000:
+ return '%s %2d %d' % (
+ months[info[1]-1],
+ info[2],
+ info[0]
+ )
+ else:
+ return '%s %2d %02d:%02d' % (
+ months[info[1]-1],
+ info[2],
+ info[3],
+ info[4]
+ )
+
+# ===========================================================================
+# Producers
+# ===========================================================================
+
+class list_producer:
+ def __init__ (self, list, func=None):
+ self.list = list
+ self.func = func
+
+ # this should do a pushd/popd
+ def more (self):
+ if not self.list:
+ return ''
+ else:
+ # do a few at a time
+ bunch = self.list[:50]
+ if self.func is not None:
+ bunch = map (self.func, bunch)
+ self.list = self.list[50:]
+ return string.joinfields (bunch, '\r\n') + '\r\n'
+
Added: supervisor/trunk/src/medusa/ftp_server.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/ftp_server.py Fri May 22 23:51:54 2009
@@ -0,0 +1,1154 @@
+# -*- Mode: Python -*-
+
+# Author: Sam Rushing <rushing at nightmare.com>
+# Copyright 1996-2000 by Sam Rushing
+# All Rights Reserved.
+#
+
+RCS_ID = '$Id: ftp_server.py,v 1.11 2003/12/24 16:05:28 akuchling Exp $'
+
+# An extensible, configurable, asynchronous FTP server.
+#
+# All socket I/O is non-blocking, however file I/O is currently
+# blocking. Eventually file I/O may be made non-blocking, too, if it
+# seems necessary. Currently the only CPU-intensive operation is
+# getting and formatting a directory listing. [this could be moved
+# into another process/directory server, or another thread?]
+#
+# Only a subset of RFC 959 is implemented, but much of that RFC is
+# vestigial anyway. I've attempted to include the most commonly-used
+# commands, using the feature set of wu-ftpd as a guide.
+
+import asyncore_25 as asyncore
+import asynchat_25 as asynchat
+
+import os
+import socket
+import stat
+import string
+import sys
+import time
+
+from medusa.producers import file_producer
+
+# TODO: implement a directory listing cache. On very-high-load
+# servers this could save a lot of disk abuse, and possibly the
+# work of computing emulated unix ls output.
+
+# Potential security problem with the FTP protocol? I don't think
+# there's any verification of the origin of a data connection. Not
+# really a problem for the server (since it doesn't send the port
+# command, except when in PASV mode) But I think a data connection
+# could be spoofed by a program with access to a sniffer - it could
+# watch for a PORT command to go over a command channel, and then
+# connect to that port before the server does.
+
+# Unix user id's:
+# In order to support assuming the id of a particular user,
+# it seems there are two options:
+# 1) fork, and seteuid in the child
+# 2) carefully control the effective uid around filesystem accessing
+# methods, using try/finally. [this seems to work]
+
+VERSION = string.split(RCS_ID)[2]
+
+from counter import counter
+import producers
+import status_handler
+import logger
+
+class ftp_channel (asynchat.async_chat):
+
+ # defaults for a reliable __repr__
+ addr = ('unknown','0')
+
+ # unset this in a derived class in order
+ # to enable the commands in 'self.write_commands'
+ read_only = 1
+ write_commands = ['appe','dele','mkd','rmd','rnfr','rnto','stor','stou']
+
+ restart_position = 0
+
+ # comply with (possibly troublesome) RFC959 requirements
+ # This is necessary to correctly run an active data connection
+ # through a firewall that triggers on the source port (expected
+ # to be 'L-1', or 20 in the normal case).
+ bind_local_minus_one = 0
+
+ def __init__ (self, server, conn, addr):
+ self.server = server
+ self.current_mode = 'a'
+ self.addr = addr
+ asynchat.async_chat.__init__ (self, conn)
+ self.set_terminator ('\r\n')
+
+ # client data port. Defaults to 'the same as the control connection'.
+ self.client_addr = (addr[0], 21)
+
+ self.client_dc = None
+ self.in_buffer = ''
+ self.closing = 0
+ self.passive_acceptor = None
+ self.passive_connection = None
+ self.filesystem = None
+ self.authorized = 0
+ # send the greeting
+ self.respond (
+ '220 %s FTP server (Medusa Async V%s [experimental]) ready.' % (
+ self.server.hostname,
+ VERSION
+ )
+ )
+
+# def __del__ (self):
+# print 'ftp_channel.__del__()'
+
+ # --------------------------------------------------
+ # async-library methods
+ # --------------------------------------------------
+
+ def handle_expt (self):
+ # this is handled below. not sure what I could
+ # do here to make that code less kludgish.
+ pass
+
+ def collect_incoming_data (self, data):
+ self.in_buffer = self.in_buffer + data
+ if len(self.in_buffer) > 4096:
+ # silently truncate really long lines
+ # (possible denial-of-service attack)
+ self.in_buffer = ''
+
+ def found_terminator (self):
+
+ line = self.in_buffer
+
+ if not len(line):
+ return
+
+ sp = string.find (line, ' ')
+ if sp != -1:
+ line = [line[:sp], line[sp+1:]]
+ else:
+ line = [line]
+
+ command = string.lower (line[0])
+ # watch especially for 'urgent' abort commands.
+ if string.find (command, 'abor') != -1:
+ # strip off telnet sync chars and the like...
+ while command and command[0] not in string.letters:
+ command = command[1:]
+ fun_name = 'cmd_%s' % command
+ if command != 'pass':
+ self.log ('<== %s' % repr(self.in_buffer)[1:-1])
+ else:
+ self.log ('<== %s' % line[0]+' <password>')
+ self.in_buffer = ''
+ if not hasattr (self, fun_name):
+ self.command_not_understood (line[0])
+ return
+ if hasattr(self,'_rnfr_src') and fun_name!='cmd_rnto':
+ del self._rnfr_src
+ self.respond ('503 RNTO Command expected!')
+ return
+
+ fun = getattr (self, fun_name)
+ if (not self.authorized) and (command not in ('user', 'pass', 'help', 'quit')):
+ self.respond ('530 Please log in with USER and PASS')
+ elif (not self.check_command_authorization (command)):
+ self.command_not_authorized (command)
+ else:
+ try:
+ result = apply (fun, (line,))
+ except:
+ self.server.total_exceptions.increment()
+ (file, fun, line), t,v, tbinfo = asyncore.compact_traceback()
+ if self.client_dc:
+ try:
+ self.client_dc.close()
+ except:
+ pass
+ self.respond (
+ '451 Server Error: %s, %s: file: %s line: %s' % (
+ t,v,file,line,
+ )
+ )
+
+ closed = 0
+ def close (self):
+ if not self.closed:
+ self.closed = 1
+ if self.passive_acceptor:
+ self.passive_acceptor.close()
+ if self.client_dc:
+ self.client_dc.close()
+ self.server.closed_sessions.increment()
+ asynchat.async_chat.close (self)
+
+ # --------------------------------------------------
+ # filesystem interface functions.
+ # override these to provide access control or perform
+ # other functions.
+ # --------------------------------------------------
+
+ def cwd (self, line):
+ return self.filesystem.cwd (line[1])
+
+ def cdup (self, line):
+ return self.filesystem.cdup()
+
+ def open (self, path, mode):
+ return self.filesystem.open (path, mode)
+
+ # returns a producer
+ def listdir (self, path, long=0):
+ return self.filesystem.listdir (path, long)
+
+ def get_dir_list (self, line, long=0):
+ # we need to scan the command line for arguments to '/bin/ls'...
+ args = line[1:]
+ path_args = []
+ for arg in args:
+ if arg[0] != '-':
+ path_args.append (arg)
+ else:
+ # ignore arguments
+ pass
+ if len(path_args) < 1:
+ dir = '.'
+ else:
+ dir = path_args[0]
+ return self.listdir (dir, long)
+
+ # --------------------------------------------------
+ # authorization methods
+ # --------------------------------------------------
+
+ def check_command_authorization (self, command):
+ if command in self.write_commands and self.read_only:
+ return 0
+ else:
+ return 1
+
+ # --------------------------------------------------
+ # utility methods
+ # --------------------------------------------------
+
+ def log (self, message):
+ self.server.logger.log (
+ self.addr[0],
+ '%d %s' % (
+ self.addr[1], message
+ )
+ )
+
+ def respond (self, resp):
+ self.log ('==> %s' % resp)
+ self.push (resp + '\r\n')
+
+ def command_not_understood (self, command):
+ self.respond ("500 '%s': command not understood." % command)
+
+ def command_not_authorized (self, command):
+ self.respond (
+ "530 You are not authorized to perform the '%s' command" % (
+ command
+ )
+ )
+
+ def make_xmit_channel (self):
+ # In PASV mode, the connection may or may _not_ have been made
+ # yet. [although in most cases it is... FTP Explorer being
+ # the only exception I've yet seen]. This gets somewhat confusing
+ # because things may happen in any order...
+ pa = self.passive_acceptor
+ if pa:
+ if pa.ready:
+ # a connection has already been made.
+ conn, addr = self.passive_acceptor.ready
+ cdc = xmit_channel (self, addr)
+ cdc.set_socket (conn)
+ cdc.connected = 1
+ self.passive_acceptor.close()
+ self.passive_acceptor = None
+ else:
+ # we're still waiting for a connect to the PASV port.
+ cdc = xmit_channel (self)
+ else:
+ # not in PASV mode.
+ ip, port = self.client_addr
+ cdc = xmit_channel (self, self.client_addr)
+ cdc.create_socket (socket.AF_INET, socket.SOCK_STREAM)
+ if self.bind_local_minus_one:
+ cdc.bind (('', self.server.port - 1))
+ try:
+ cdc.connect ((ip, port))
+ except socket.error, why:
+ self.respond ("425 Can't build data connection")
+ self.client_dc = cdc
+
+ # pretty much the same as xmit, but only right on the verge of
+ # being worth a merge.
+ def make_recv_channel (self, fd):
+ pa = self.passive_acceptor
+ if pa:
+ if pa.ready:
+ # a connection has already been made.
+ conn, addr = pa.ready
+ cdc = recv_channel (self, addr, fd)
+ cdc.set_socket (conn)
+ cdc.connected = 1
+ self.passive_acceptor.close()
+ self.passive_acceptor = None
+ else:
+ # we're still waiting for a connect to the PASV port.
+ cdc = recv_channel (self, None, fd)
+ else:
+ # not in PASV mode.
+ ip, port = self.client_addr
+ cdc = recv_channel (self, self.client_addr, fd)
+ cdc.create_socket (socket.AF_INET, socket.SOCK_STREAM)
+ try:
+ cdc.connect ((ip, port))
+ except socket.error, why:
+ self.respond ("425 Can't build data connection")
+ self.client_dc = cdc
+
+ type_map = {
+ 'a':'ASCII',
+ 'i':'Binary',
+ 'e':'EBCDIC',
+ 'l':'Binary'
+ }
+
+ type_mode_map = {
+ 'a':'t',
+ 'i':'b',
+ 'e':'b',
+ 'l':'b'
+ }
+
+ # --------------------------------------------------
+ # command methods
+ # --------------------------------------------------
+
+ def cmd_type (self, line):
+ 'specify data transfer type'
+ # ascii, ebcdic, image, local <byte size>
+ t = string.lower (line[1])
+ # no support for EBCDIC
+ # if t not in ['a','e','i','l']:
+ if t not in ['a','i','l']:
+ self.command_not_understood (string.join (line))
+ elif t == 'l' and (len(line) > 2 and line[2] != '8'):
+ self.respond ('504 Byte size must be 8')
+ else:
+ self.current_mode = t
+ self.respond ('200 Type set to %s.' % self.type_map[t])
+
+
+ def cmd_quit (self, line):
+ 'terminate session'
+ self.respond ('221 Goodbye.')
+ self.close_when_done()
+
+ def cmd_port (self, line):
+ 'specify data connection port'
+ info = string.split (line[1], ',')
+ ip = string.join (info[:4], '.')
+ port = string.atoi(info[4])*256 + string.atoi(info[5])
+ # how many data connections at a time?
+ # I'm assuming one for now...
+ # TODO: we should (optionally) verify that the
+ # ip number belongs to the client. [wu-ftpd does this?]
+ self.client_addr = (ip, port)
+ self.respond ('200 PORT command successful.')
+
+ def new_passive_acceptor (self):
+ # ensure that only one of these exists at a time.
+ if self.passive_acceptor is not None:
+ self.passive_acceptor.close()
+ self.passive_acceptor = None
+ self.passive_acceptor = passive_acceptor (self)
+ return self.passive_acceptor
+
+ def cmd_pasv (self, line):
+ 'prepare for server-to-server transfer'
+ pc = self.new_passive_acceptor()
+ port = pc.addr[1]
+ ip_addr = pc.control_channel.getsockname()[0]
+ self.respond (
+ '227 Entering Passive Mode (%s,%d,%d)' % (
+ string.replace(ip_addr, '.', ','),
+ port/256,
+ port%256
+ )
+ )
+ self.client_dc = None
+
+ def cmd_nlst (self, line):
+ 'give name list of files in directory'
+ # ncftp adds the -FC argument for the user-visible 'nlist'
+ # command. We could try to emulate ls flags, but not just yet.
+ if '-FC' in line:
+ line.remove ('-FC')
+ try:
+ dir_list_producer = self.get_dir_list (line, 0)
+ except os.error, why:
+ self.respond ('550 Could not list directory: %s' % why)
+ return
+ self.respond (
+ '150 Opening %s mode data connection for file list' % (
+ self.type_map[self.current_mode]
+ )
+ )
+ self.make_xmit_channel()
+ self.client_dc.push_with_producer (dir_list_producer)
+ self.client_dc.close_when_done()
+
+ def cmd_list (self, line):
+ 'give a list of files in a directory'
+ try:
+ dir_list_producer = self.get_dir_list (line, 1)
+ except os.error, why:
+ self.respond ('550 Could not list directory: %s' % why)
+ return
+ self.respond (
+ '150 Opening %s mode data connection for file list' % (
+ self.type_map[self.current_mode]
+ )
+ )
+ self.make_xmit_channel()
+ self.client_dc.push_with_producer (dir_list_producer)
+ self.client_dc.close_when_done()
+
+ def cmd_cwd (self, line):
+ 'change working directory'
+ if self.cwd (line):
+ self.respond ('250 CWD command successful.')
+ else:
+ self.respond ('550 No such directory.')
+
+ def cmd_cdup (self, line):
+ 'change to parent of current working directory'
+ if self.cdup(line):
+ self.respond ('250 CDUP command successful.')
+ else:
+ self.respond ('550 No such directory.')
+
+ def cmd_pwd (self, line):
+ 'print the current working directory'
+ self.respond (
+ '257 "%s" is the current directory.' % (
+ self.filesystem.current_directory()
+ )
+ )
+
+ # modification time
+ # example output:
+ # 213 19960301204320
+ def cmd_mdtm (self, line):
+ 'show last modification time of file'
+ filename = line[1]
+ if not self.filesystem.isfile (filename):
+ self.respond ('550 "%s" is not a file' % filename)
+ else:
+ mtime = time.gmtime(self.filesystem.stat(filename)[stat.ST_MTIME])
+ self.respond (
+ '213 %4d%02d%02d%02d%02d%02d' % (
+ mtime[0],
+ mtime[1],
+ mtime[2],
+ mtime[3],
+ mtime[4],
+ mtime[5]
+ )
+ )
+
+ def cmd_noop (self, line):
+ 'do nothing'
+ self.respond ('200 NOOP command successful.')
+
+ def cmd_size (self, line):
+ 'return size of file'
+ filename = line[1]
+ if not self.filesystem.isfile (filename):
+ self.respond ('550 "%s" is not a file' % filename)
+ else:
+ self.respond (
+ '213 %d' % (self.filesystem.stat(filename)[stat.ST_SIZE])
+ )
+
+ def cmd_retr (self, line):
+ 'retrieve a file'
+ if len(line) < 2:
+ self.command_not_understood (string.join (line))
+ else:
+ file = line[1]
+ if not self.filesystem.isfile (file):
+ self.log_info ('checking %s' % file)
+ self.respond ('550 No such file')
+ else:
+ try:
+ # FIXME: for some reason, 'rt' isn't working on win95
+ mode = 'r'+self.type_mode_map[self.current_mode]
+ fd = self.open (file, mode)
+ except IOError, why:
+ self.respond ('553 could not open file for reading: %s' % (repr(why)))
+ return
+ self.respond (
+ "150 Opening %s mode data connection for file '%s'" % (
+ self.type_map[self.current_mode],
+ file
+ )
+ )
+ self.make_xmit_channel()
+
+ if self.restart_position:
+ # try to position the file as requested, but
+ # give up silently on failure (the 'file object'
+ # may not support seek())
+ try:
+ fd.seek (self.restart_position)
+ except:
+ pass
+ self.restart_position = 0
+
+ self.client_dc.push_with_producer (
+ file_producer (fd)
+ )
+ self.client_dc.close_when_done()
+
+ def cmd_stor (self, line, mode='wb'):
+ 'store a file'
+ if len (line) < 2:
+ self.command_not_understood (string.join (line))
+ else:
+ if self.restart_position:
+ restart_position = 0
+ self.respond ('553 restart on STOR not yet supported')
+ return
+ file = line[1]
+ # todo: handle that type flag
+ try:
+ fd = self.open (file, mode)
+ except IOError, why:
+ self.respond ('553 could not open file for writing: %s' % (repr(why)))
+ return
+ self.respond (
+ '150 Opening %s connection for %s' % (
+ self.type_map[self.current_mode],
+ file
+ )
+ )
+ self.make_recv_channel (fd)
+
+ def cmd_abor (self, line):
+ 'abort operation'
+ if self.client_dc:
+ self.client_dc.close()
+ self.respond ('226 ABOR command successful.')
+
+ def cmd_appe (self, line):
+ 'append to a file'
+ return self.cmd_stor (line, 'ab')
+
+ def cmd_dele (self, line):
+ if len (line) != 2:
+ self.command_not_understood (string.join (line))
+ else:
+ file = line[1]
+ if self.filesystem.isfile (file):
+ try:
+ self.filesystem.unlink (file)
+ self.respond ('250 DELE command successful.')
+ except:
+ self.respond ('550 error deleting file.')
+ else:
+ self.respond ('550 %s: No such file.' % file)
+
+ def cmd_mkd (self, line):
+ if len (line) != 2:
+ self.command_not_understood (string.join (line))
+ else:
+ path = line[1]
+ try:
+ self.filesystem.mkdir (path)
+ self.respond ('257 MKD command successful.')
+ except:
+ self.respond ('550 error creating directory.')
+
+ def cmd_rnfr (self, line):
+ if not hasattr(self.filesystem,'rename'):
+ self.respond('502 RNFR not implemented.' % src)
+ return
+
+ if len(line)!=2:
+ self.command_not_understood (string.join (line))
+ else:
+ src = line[1]
+ try:
+ assert self.filesystem.isfile(src)
+ self._rfnr_src = src
+ self.respond('350 RNFR file exists, ready for destination name.')
+ except:
+ self.respond('550 %s: No such file.' % src)
+
+ def cmd_rnto (self, line):
+ src = getattr(self,'_rfnr_src',None)
+ if not src:
+ self.respond('503 RNTO command unexpected.')
+ return
+
+ if len(line)!=2:
+ self.command_not_understood (string.join (line))
+ else:
+ dst = line[1]
+ try:
+ self.filesystem.rename(src,dst)
+ self.respond('250 RNTO command successful.')
+ except:
+ t, v = sys.exc_info[:2]
+ self.respond('550 %s: %s.' % (str(t),str(v)))
+ try:
+ del self._rfnr_src
+ except:
+ pass
+
+ def cmd_rmd (self, line):
+ if len (line) != 2:
+ self.command_not_understood (string.join (line))
+ else:
+ path = line[1]
+ try:
+ self.filesystem.rmdir (path)
+ self.respond ('250 RMD command successful.')
+ except:
+ self.respond ('550 error removing directory.')
+
+ def cmd_user (self, line):
+ 'specify user name'
+ if len(line) > 1:
+ self.user = line[1]
+ self.respond ('331 Password required.')
+ else:
+ self.command_not_understood (string.join (line))
+
+ def cmd_pass (self, line):
+ 'specify password'
+ if len(line) < 2:
+ pw = ''
+ else:
+ pw = line[1]
+ result, message, fs = self.server.authorizer.authorize (self, self.user, pw)
+ if result:
+ self.respond ('230 %s' % message)
+ self.filesystem = fs
+ self.authorized = 1
+ self.log_info('Successful login: Filesystem=%s' % repr(fs))
+ else:
+ self.respond ('530 %s' % message)
+
+ def cmd_rest (self, line):
+ 'restart incomplete transfer'
+ try:
+ pos = string.atoi (line[1])
+ except ValueError:
+ self.command_not_understood (string.join (line))
+ self.restart_position = pos
+ self.respond (
+ '350 Restarting at %d. Send STORE or RETRIEVE to initiate transfer.' % pos
+ )
+
+ def cmd_stru (self, line):
+ 'obsolete - set file transfer structure'
+ if line[1] in 'fF':
+ # f == 'file'
+ self.respond ('200 STRU F Ok')
+ else:
+ self.respond ('504 Unimplemented STRU type')
+
+ def cmd_mode (self, line):
+ 'obsolete - set file transfer mode'
+ if line[1] in 'sS':
+ # f == 'file'
+ self.respond ('200 MODE S Ok')
+ else:
+ self.respond ('502 Unimplemented MODE type')
+
+# The stat command has two personalities. Normally it returns status
+# information about the current connection. But if given an argument,
+# it is equivalent to the LIST command, with the data sent over the
+# control connection. Strange. But wuftpd, ftpd, and nt's ftp server
+# all support it.
+#
+## def cmd_stat (self, line):
+## 'return status of server'
+## pass
+
+ def cmd_syst (self, line):
+ 'show operating system type of server system'
+ # Replying to this command is of questionable utility, because
+ # this server does not behave in a predictable way w.r.t. the
+ # output of the LIST command. We emulate Unix ls output, but
+ # on win32 the pathname can contain drive information at the front
+ # Currently, the combination of ensuring that os.sep == '/'
+ # and removing the leading slash when necessary seems to work.
+ # [cd'ing to another drive also works]
+ #
+ # This is how wuftpd responds, and is probably
+ # the most expected. The main purpose of this reply is so that
+ # the client knows to expect Unix ls-style LIST output.
+ self.respond ('215 UNIX Type: L8')
+ # one disadvantage to this is that some client programs
+ # assume they can pass args to /bin/ls.
+ # a few typical responses:
+ # 215 UNIX Type: L8 (wuftpd)
+ # 215 Windows_NT version 3.51
+ # 215 VMS MultiNet V3.3
+ # 500 'SYST': command not understood. (SVR4)
+
+ def cmd_help (self, line):
+ 'give help information'
+ # find all the methods that match 'cmd_xxxx',
+ # use their docstrings for the help response.
+ attrs = dir(self.__class__)
+ help_lines = []
+ for attr in attrs:
+ if attr[:4] == 'cmd_':
+ x = getattr (self, attr)
+ if type(x) == type(self.cmd_help):
+ if x.__doc__:
+ help_lines.append ('\t%s\t%s' % (attr[4:], x.__doc__))
+ if help_lines:
+ self.push ('214-The following commands are recognized\r\n')
+ self.push_with_producer (producers.lines_producer (help_lines))
+ self.push ('214\r\n')
+ else:
+ self.push ('214-\r\n\tHelp Unavailable\r\n214\r\n')
+
+class ftp_server (asyncore.dispatcher):
+ # override this to spawn a different FTP channel class.
+ ftp_channel_class = ftp_channel
+
+ SERVER_IDENT = 'FTP Server (V%s)' % VERSION
+
+ def __init__ (
+ self,
+ authorizer,
+ hostname =None,
+ ip ='',
+ port =21,
+ resolver =None,
+ logger_object=logger.file_logger (sys.stdout)
+ ):
+ self.ip = ip
+ self.port = port
+ self.authorizer = authorizer
+
+ if hostname is None:
+ self.hostname = socket.gethostname()
+ else:
+ self.hostname = hostname
+
+ # statistics
+ self.total_sessions = counter()
+ self.closed_sessions = counter()
+ self.total_files_out = counter()
+ self.total_files_in = counter()
+ self.total_bytes_out = counter()
+ self.total_bytes_in = counter()
+ self.total_exceptions = counter()
+ #
+ asyncore.dispatcher.__init__ (self)
+ self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
+
+ self.set_reuse_addr()
+ self.bind ((self.ip, self.port))
+ self.listen (5)
+
+ if not logger_object:
+ logger_object = sys.stdout
+
+ if resolver:
+ self.logger = logger.resolving_logger (resolver, logger_object)
+ else:
+ self.logger = logger.unresolving_logger (logger_object)
+
+ self.log_info('FTP server started at %s\n\tAuthorizer:%s\n\tHostname: %s\n\tPort: %d' % (
+ time.ctime(time.time()),
+ repr (self.authorizer),
+ self.hostname,
+ self.port)
+ )
+
+ def writable (self):
+ return 0
+
+ def handle_read (self):
+ pass
+
+ def handle_connect (self):
+ pass
+
+ def handle_accept (self):
+ conn, addr = self.accept()
+ self.total_sessions.increment()
+ self.log_info('Incoming connection from %s:%d' % (addr[0], addr[1]))
+ self.ftp_channel_class (self, conn, addr)
+
+ # return a producer describing the state of the server
+ def status (self):
+
+ def nice_bytes (n):
+ return string.join (status_handler.english_bytes (n))
+
+ return producers.lines_producer (
+ ['<h2>%s</h2>' % self.SERVER_IDENT,
+ '<br>Listening on <b>Host:</b> %s' % self.hostname,
+ '<b>Port:</b> %d' % self.port,
+ '<br>Sessions',
+ '<b>Total:</b> %s' % self.total_sessions,
+ '<b>Current:</b> %d' % (self.total_sessions.as_long() - self.closed_sessions.as_long()),
+ '<br>Files',
+ '<b>Sent:</b> %s' % self.total_files_out,
+ '<b>Received:</b> %s' % self.total_files_in,
+ '<br>Bytes',
+ '<b>Sent:</b> %s' % nice_bytes (self.total_bytes_out.as_long()),
+ '<b>Received:</b> %s' % nice_bytes (self.total_bytes_in.as_long()),
+ '<br>Exceptions: %s' % self.total_exceptions,
+ ]
+ )
+
+# ======================================================================
+# Data Channel Classes
+# ======================================================================
+
+# This socket accepts a data connection, used when the server has been
+# placed in passive mode. Although the RFC implies that we ought to
+# be able to use the same acceptor over and over again, this presents
+# a problem: how do we shut it off, so that we are accepting
+# connections only when we expect them? [we can't]
+#
+# wuftpd, and probably all the other servers, solve this by allowing
+# only one connection to hit this acceptor. They then close it. Any
+# subsequent data-connection command will then try for the default
+# port on the client side [which is of course never there]. So the
+# 'always-send-PORT/PASV' behavior seems required.
+#
+# Another note: wuftpd will also be listening on the channel as soon
+# as the PASV command is sent. It does not wait for a data command
+# first.
+
+# --- we need to queue up a particular behavior:
+# 1) xmit : queue up producer[s]
+# 2) recv : the file object
+#
+# It would be nice if we could make both channels the same. Hmmm..
+#
+
+class passive_acceptor (asyncore.dispatcher):
+ ready = None
+
+ def __init__ (self, control_channel):
+ # connect_fun (conn, addr)
+ asyncore.dispatcher.__init__ (self)
+ self.control_channel = control_channel
+ self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
+ # bind to an address on the interface that the
+ # control connection is coming from.
+ self.bind ((
+ self.control_channel.getsockname()[0],
+ 0
+ ))
+ self.addr = self.getsockname()
+ self.listen (1)
+
+# def __del__ (self):
+# print 'passive_acceptor.__del__()'
+
+ def log (self, *ignore):
+ pass
+
+ def handle_accept (self):
+ conn, addr = self.accept()
+ dc = self.control_channel.client_dc
+ if dc is not None:
+ dc.set_socket (conn)
+ dc.addr = addr
+ dc.connected = 1
+ self.control_channel.passive_acceptor = None
+ else:
+ self.ready = conn, addr
+ self.close()
+
+
+class xmit_channel (asynchat.async_chat):
+
+ # for an ethernet, you want this to be fairly large, in fact, it
+ # _must_ be large for performance comparable to an ftpd. [64k] we
+ # ought to investigate automatically-sized buffers...
+
+ ac_out_buffer_size = 16384
+ bytes_out = 0
+
+ def __init__ (self, channel, client_addr=None):
+ self.channel = channel
+ self.client_addr = client_addr
+ asynchat.async_chat.__init__ (self)
+
+# def __del__ (self):
+# print 'xmit_channel.__del__()'
+
+ def log (self, *args):
+ pass
+
+ def readable (self):
+ return not self.connected
+
+ def writable (self):
+ return 1
+
+ def send (self, data):
+ result = asynchat.async_chat.send (self, data)
+ self.bytes_out = self.bytes_out + result
+ return result
+
+ def handle_error (self):
+ # usually this is to catch an unexpected disconnect.
+ self.log_info ('unexpected disconnect on data xmit channel', 'error')
+ try:
+ self.close()
+ except:
+ pass
+
+ # TODO: there's a better way to do this. we need to be able to
+ # put 'events' in the producer fifo. to do this cleanly we need
+ # to reposition the 'producer' fifo as an 'event' fifo.
+
+ def close (self):
+ c = self.channel
+ s = c.server
+ c.client_dc = None
+ s.total_files_out.increment()
+ s.total_bytes_out.increment (self.bytes_out)
+ if not len(self.producer_fifo):
+ c.respond ('226 Transfer complete')
+ elif not c.closed:
+ c.respond ('426 Connection closed; transfer aborted')
+ del c
+ del s
+ del self.channel
+ asynchat.async_chat.close (self)
+
+class recv_channel (asyncore.dispatcher):
+ def __init__ (self, channel, client_addr, fd):
+ self.channel = channel
+ self.client_addr = client_addr
+ self.fd = fd
+ asyncore.dispatcher.__init__ (self)
+ self.bytes_in = counter()
+
+ def log (self, *ignore):
+ pass
+
+ def handle_connect (self):
+ pass
+
+ def writable (self):
+ return 0
+
+ def recv (*args):
+ result = apply (asyncore.dispatcher.recv, args)
+ self = args[0]
+ self.bytes_in.increment(len(result))
+ return result
+
+ buffer_size = 8192
+
+ def handle_read (self):
+ block = self.recv (self.buffer_size)
+ if block:
+ try:
+ self.fd.write (block)
+ except IOError:
+ self.log_info ('got exception writing block...', 'error')
+
+ def handle_close (self):
+ s = self.channel.server
+ s.total_files_in.increment()
+ s.total_bytes_in.increment(self.bytes_in.as_long())
+ self.fd.close()
+ self.channel.respond ('226 Transfer complete.')
+ self.close()
+
+import filesys
+
+# not much of a doorman! 8^)
+class dummy_authorizer:
+ def __init__ (self, root='/'):
+ self.root = root
+ def authorize (self, channel, username, password):
+ channel.persona = -1, -1
+ channel.read_only = 1
+ return 1, 'Ok.', filesys.os_filesystem (self.root)
+
+class anon_authorizer:
+ def __init__ (self, root='/'):
+ self.root = root
+
+ def authorize (self, channel, username, password):
+ if username in ('ftp', 'anonymous'):
+ channel.persona = -1, -1
+ channel.read_only = 1
+ return 1, 'Ok.', filesys.os_filesystem (self.root)
+ else:
+ return 0, 'Password invalid.', None
+
+# ===========================================================================
+# Unix-specific improvements
+# ===========================================================================
+
+if os.name == 'posix':
+
+ class unix_authorizer:
+ # return a trio of (success, reply_string, filesystem)
+ def authorize (self, channel, username, password):
+ import crypt
+ import pwd
+ try:
+ info = pwd.getpwnam (username)
+ except KeyError:
+ return 0, 'No such user.', None
+ mangled = info[1]
+ if crypt.crypt (password, mangled[:2]) == mangled:
+ channel.read_only = 0
+ fs = filesys.schizophrenic_unix_filesystem (
+ '/',
+ info[5],
+ persona = (info[2], info[3])
+ )
+ return 1, 'Login successful.', fs
+ else:
+ return 0, 'Password invalid.', None
+
+ def __repr__ (self):
+ return '<standard unix authorizer>'
+
+ # simple anonymous ftp support
+ class unix_authorizer_with_anonymous (unix_authorizer):
+ def __init__ (self, root=None, real_users=0):
+ self.root = root
+ self.real_users = real_users
+
+ def authorize (self, channel, username, password):
+ if string.lower(username) in ['anonymous', 'ftp']:
+ import pwd
+ try:
+ # ok, here we run into lots of confusion.
+ # on some os', anon runs under user 'nobody',
+ # on others as 'ftp'. ownership is also critical.
+ # need to investigate.
+ # linux: new linuxen seem to have nobody's UID=-1,
+ # which is an illegal value. Use ftp.
+ ftp_user_info = pwd.getpwnam ('ftp')
+ if string.lower(os.uname()[0]) == 'linux':
+ nobody_user_info = pwd.getpwnam ('ftp')
+ else:
+ nobody_user_info = pwd.getpwnam ('nobody')
+ channel.read_only = 1
+ if self.root is None:
+ self.root = ftp_user_info[5]
+ fs = filesys.unix_filesystem (self.root, '/')
+ return 1, 'Anonymous Login Successful', fs
+ except KeyError:
+ return 0, 'Anonymous account not set up', None
+ elif self.real_users:
+ return unix_authorizer.authorize (
+ self,
+ channel,
+ username,
+ password
+ )
+ else:
+ return 0, 'User logins not allowed', None
+
+# usage: ftp_server /PATH/TO/FTP/ROOT PORT
+# for example:
+# $ ftp_server /home/users/ftp 8021
+
+if os.name == 'posix':
+ def test (port='8021'):
+ fs = ftp_server (
+ unix_authorizer(),
+ port=string.atoi (port)
+ )
+ try:
+ asyncore.loop()
+ except KeyboardInterrupt:
+ fs.log_info('FTP server shutting down. (received SIGINT)', 'warning')
+ # close everything down on SIGINT.
+ # of course this should be a cleaner shutdown.
+ asyncore.close_all()
+
+ if __name__ == '__main__':
+ test (sys.argv[1])
+# not unix
+else:
+ def test ():
+ fs = ftp_server (dummy_authorizer())
+ if __name__ == '__main__':
+ test ()
+
+# this is the command list from the wuftpd man page
+# '*' means we've implemented it.
+# '!' requires write access
+#
+command_documentation = {
+ 'abor': 'abort previous command', #*
+ 'acct': 'specify account (ignored)',
+ 'allo': 'allocate storage (vacuously)',
+ 'appe': 'append to a file', #*!
+ 'cdup': 'change to parent of current working directory', #*
+ 'cwd': 'change working directory', #*
+ 'dele': 'delete a file', #!
+ 'help': 'give help information', #*
+ 'list': 'give list files in a directory', #*
+ 'mkd': 'make a directory', #!
+ 'mdtm': 'show last modification time of file', #*
+ 'mode': 'specify data transfer mode',
+ 'nlst': 'give name list of files in directory', #*
+ 'noop': 'do nothing', #*
+ 'pass': 'specify password', #*
+ 'pasv': 'prepare for server-to-server transfer', #*
+ 'port': 'specify data connection port', #*
+ 'pwd': 'print the current working directory', #*
+ 'quit': 'terminate session', #*
+ 'rest': 'restart incomplete transfer', #*
+ 'retr': 'retrieve a file', #*
+ 'rmd': 'remove a directory', #!
+ 'rnfr': 'specify rename-from file name', #*!
+ 'rnto': 'specify rename-to file name', #*!
+ 'site': 'non-standard commands (see next section)',
+ 'size': 'return size of file', #*
+ 'stat': 'return status of server', #*
+ 'stor': 'store a file', #*!
+ 'stou': 'store a file with a unique name', #!
+ 'stru': 'specify data transfer structure',
+ 'syst': 'show operating system type of server system', #*
+ 'type': 'specify data transfer type', #*
+ 'user': 'specify user name', #*
+ 'xcup': 'change to parent of current working directory (deprecated)',
+ 'xcwd': 'change working directory (deprecated)',
+ 'xmkd': 'make a directory (deprecated)', #!
+ 'xpwd': 'print the current working directory (deprecated)',
+ 'xrmd': 'remove a directory (deprecated)', #!
+}
+
+
+# debugging aid (linux)
+def get_vm_size ():
+ return string.atoi (string.split(open ('/proc/self/stat').readline())[22])
+
+def print_vm():
+ print 'vm: %8dk' % (get_vm_size()/1024)
Added: supervisor/trunk/src/medusa/http_date.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/http_date.py Fri May 22 23:51:54 2009
@@ -0,0 +1,126 @@
+# -*- Mode: Python -*-
+
+import re
+import string
+import time
+
+def concat (*args):
+ return ''.join (args)
+
+def join (seq, field=' '):
+ return field.join (seq)
+
+def group (s):
+ return '(' + s + ')'
+
+short_days = ['sun','mon','tue','wed','thu','fri','sat']
+long_days = ['sunday','monday','tuesday','wednesday','thursday','friday','saturday']
+
+short_day_reg = group (join (short_days, '|'))
+long_day_reg = group (join (long_days, '|'))
+
+daymap = {}
+for i in range(7):
+ daymap[short_days[i]] = i
+ daymap[long_days[i]] = i
+
+hms_reg = join (3 * [group('[0-9][0-9]')], ':')
+
+months = ['jan','feb','mar','apr','may','jun','jul','aug','sep','oct','nov','dec']
+
+monmap = {}
+for i in range(12):
+ monmap[months[i]] = i+1
+
+months_reg = group (join (months, '|'))
+
+# From draft-ietf-http-v11-spec-07.txt/3.3.1
+# Sun, 06 Nov 1994 08:49:37 GMT ; RFC 822, updated by RFC 1123
+# Sunday, 06-Nov-94 08:49:37 GMT ; RFC 850, obsoleted by RFC 1036
+# Sun Nov 6 08:49:37 1994 ; ANSI C's asctime() format
+
+# rfc822 format
+rfc822_date = join (
+ [concat (short_day_reg,','), # day
+ group('[0-9][0-9]?'), # date
+ months_reg, # month
+ group('[0-9]+'), # year
+ hms_reg, # hour minute second
+ 'gmt'
+ ],
+ ' '
+ )
+
+rfc822_reg = re.compile (rfc822_date)
+
+def unpack_rfc822 (m):
+ g = m.group
+ a = string.atoi
+ return (
+ a(g(4)), # year
+ monmap[g(3)], # month
+ a(g(2)), # day
+ a(g(5)), # hour
+ a(g(6)), # minute
+ a(g(7)), # second
+ 0,
+ 0,
+ 0
+ )
+
+# rfc850 format
+rfc850_date = join (
+ [concat (long_day_reg,','),
+ join (
+ [group ('[0-9][0-9]?'),
+ months_reg,
+ group ('[0-9]+')
+ ],
+ '-'
+ ),
+ hms_reg,
+ 'gmt'
+ ],
+ ' '
+ )
+
+rfc850_reg = re.compile (rfc850_date)
+# they actually unpack the same way
+def unpack_rfc850 (m):
+ g = m.group
+ a = string.atoi
+ return (
+ a(g(4)), # year
+ monmap[g(3)], # month
+ a(g(2)), # day
+ a(g(5)), # hour
+ a(g(6)), # minute
+ a(g(7)), # second
+ 0,
+ 0,
+ 0
+ )
+
+# parsdate.parsedate - ~700/sec.
+# parse_http_date - ~1333/sec.
+
+def build_http_date (when):
+ return time.strftime ('%a, %d %b %Y %H:%M:%S GMT', time.gmtime(when))
+
+def parse_http_date (d):
+ d = string.lower (d)
+ tz = time.timezone
+ m = rfc850_reg.match (d)
+ if m and m.end() == len(d):
+ retval = int (time.mktime (unpack_rfc850(m)) - tz)
+ else:
+ m = rfc822_reg.match (d)
+ if m and m.end() == len(d):
+ retval = int (time.mktime (unpack_rfc822(m)) - tz)
+ else:
+ return 0
+ # Thanks to Craig Silverstein <csilvers at google.com> for pointing
+ # out the DST discrepancy
+ if time.daylight and time.localtime(retval)[-1] == 1: # DST correction
+ retval = retval + (tz - time.altzone)
+ return retval
Added: supervisor/trunk/src/medusa/http_server.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/http_server.py Fri May 22 23:51:54 2009
@@ -0,0 +1,860 @@
+#! /usr/local/bin/python
+# -*- Mode: Python -*-
+#
+# Author: Sam Rushing <rushing at nightmare.com>
+# Copyright 1996-2000 by Sam Rushing
+# All Rights Reserved.
+#
+
+RCS_ID = '$Id: http_server.py,v 1.12 2004/04/21 15:11:44 akuchling Exp $'
+
+# python modules
+import os
+import re
+import socket
+import string
+import sys
+import time
+
+# async modules
+import asyncore_25 as asyncore
+import asynchat_25 as asynchat
+
+# medusa modules
+import http_date
+import producers
+import status_handler
+import logger
+
+VERSION_STRING = string.split(RCS_ID)[2]
+
+from counter import counter
+from urllib import unquote, splitquery
+
+# ===========================================================================
+# Request Object
+# ===========================================================================
+
+class http_request:
+
+ # default reply code
+ reply_code = 200
+
+ request_counter = counter()
+
+ # Whether to automatically use chunked encoding when
+ #
+ # HTTP version is 1.1
+ # Content-Length is not set
+ # Chunked encoding is not already in effect
+ #
+ # If your clients are having trouble, you might want to disable this.
+ use_chunked = 1
+
+ # by default, this request object ignores user data.
+ collector = None
+
+ def __init__ (self, *args):
+ # unpack information about the request
+ (self.channel, self.request,
+ self.command, self.uri, self.version,
+ self.header) = args
+
+ self.outgoing = []
+ self.reply_headers = {
+ 'Server' : 'Medusa/%s' % VERSION_STRING,
+ 'Date' : http_date.build_http_date (time.time())
+ }
+
+ # New reply header list (to support multiple
+ # headers with same name)
+ self.__reply_header_list = []
+
+ self.request_number = http_request.request_counter.increment()
+ self._split_uri = None
+ self._header_cache = {}
+
+ # --------------------------------------------------
+ # reply header management
+ # --------------------------------------------------
+ def __setitem__ (self, key, value):
+ self.reply_headers[key] = value
+
+ def __getitem__ (self, key):
+ return self.reply_headers[key]
+
+ def has_key (self, key):
+ return self.reply_headers.has_key (key)
+
+ def build_reply_header (self):
+ return string.join (
+ [self.response(self.reply_code)] + map (
+ lambda x: '%s: %s' % x,
+ self.reply_headers.items()
+ ),
+ '\r\n'
+ ) + '\r\n\r\n'
+
+ ####################################################
+ # multiple reply header management
+ ####################################################
+ # These are intended for allowing multiple occurrences
+ # of the same header.
+ # Usually you can fold such headers together, separating
+ # their contents by a comma (e.g. Accept: text/html, text/plain)
+ # but the big exception is the Set-Cookie header.
+ # dictionary centric.
+ #---------------------------------------------------
+
+ def add_header(self, name, value):
+ """ Adds a header to the reply headers """
+ self.__reply_header_list.append((name, value))
+
+ def clear_headers(self):
+ """ Clears the reply header list """
+
+ # Remove things from the old dict as well
+ self.reply_headers.clear()
+
+ self.__reply_header_list[:] = []
+
+ def remove_header(self, name, value=None):
+ """ Removes the specified header.
+ If a value is provided, the name and
+ value must match to remove the header.
+ If the value is None, removes all headers
+ with that name."""
+
+ found_it = 0
+
+ # Remove things from the old dict as well
+ if (self.reply_headers.has_key(name) and
+ (value is None or
+ self.reply_headers[name] == value)):
+ del self.reply_headers[name]
+ found_it = 1
+
+
+ if not value is None:
+ if (name, value) in self.__reply_header_list:
+ removed_headers = [(name, value)]
+ found_it = 1
+ else:
+ removed_headers = []
+ for h in self.__reply_header_list:
+ if h[0] == name:
+ removed_headers.append(h)
+ found_it = 1
+
+ if not found_it:
+ if value is None:
+ search_value = "%s" % name
+ else:
+ search_value = "%s: %s" % (name, value)
+
+ raise LookupError("Header '%s' not found" % search_value)
+
+ for h in removed_headers:
+ self.__reply_header_list.remove(h)
+
+
+ def get_reply_headers(self):
+ """ Get the tuple of headers that will be used
+ for generating reply headers"""
+ header_tuples = self.__reply_header_list[:]
+
+ # The idea here is to insert the headers from
+ # the old header dict into the new header list,
+ # UNLESS there's already an entry in the list
+ # that would have overwritten the dict entry
+ # if the dict was the only storage...
+ header_names = [n for n,v in header_tuples]
+ for n,v in self.reply_headers.items():
+ if n not in header_names:
+ header_tuples.append((n,v))
+ header_names.append(n)
+ # Ok, that should do it. Now, if there were any
+ # headers in the dict that weren't in the list,
+ # they should have been copied in. If the name
+ # was already in the list, we didn't copy it,
+ # because the value from the dict has been
+ # 'overwritten' by the one in the list.
+
+ return header_tuples
+
+ def get_reply_header_text(self):
+ """ Gets the reply header (including status and
+ additional crlf)"""
+
+ header_tuples = self.get_reply_headers()
+
+ headers = [self.response(self.reply_code)]
+ headers += ["%s: %s" % h for h in header_tuples]
+
+ return string.join(headers, '\r\n') + '\r\n\r\n'
+
+ #---------------------------------------------------
+ # This is the end of the new reply header
+ # management section.
+ ####################################################
+
+
+ # --------------------------------------------------
+ # split a uri
+ # --------------------------------------------------
+
+ # <path>;<params>?<query>#<fragment>
+ path_regex = re.compile (
+ # path params query fragment
+ r'([^;?#]*)(;[^?#]*)?(\?[^#]*)?(#.*)?'
+ )
+
+ def split_uri (self):
+ if self._split_uri is None:
+ m = self.path_regex.match (self.uri)
+ if m.end() != len(self.uri):
+ raise ValueError, "Broken URI"
+ else:
+ self._split_uri = m.groups()
+ return self._split_uri
+
+ def get_header_with_regex (self, head_reg, group):
+ for line in self.header:
+ m = head_reg.match (line)
+ if m.end() == len(line):
+ return m.group (group)
+ return ''
+
+ def get_header (self, header):
+ header = string.lower (header)
+ hc = self._header_cache
+ if not hc.has_key (header):
+ h = header + ': '
+ hl = len(h)
+ for line in self.header:
+ if string.lower (line[:hl]) == h:
+ r = line[hl:]
+ hc[header] = r
+ return r
+ hc[header] = None
+ return None
+ else:
+ return hc[header]
+
+ # --------------------------------------------------
+ # user data
+ # --------------------------------------------------
+
+ def collect_incoming_data (self, data):
+ if self.collector:
+ self.collector.collect_incoming_data (data)
+ else:
+ self.log_info(
+ 'Dropping %d bytes of incoming request data' % len(data),
+ 'warning'
+ )
+
+ def found_terminator (self):
+ if self.collector:
+ self.collector.found_terminator()
+ else:
+ self.log_info (
+ 'Unexpected end-of-record for incoming request',
+ 'warning'
+ )
+
+ def push (self, thing):
+ if type(thing) == type(''):
+ self.outgoing.append(producers.simple_producer(thing))
+ else:
+ self.outgoing.append(thing)
+
+ def response (self, code=200):
+ message = self.responses[code]
+ self.reply_code = code
+ return 'HTTP/%s %d %s' % (self.version, code, message)
+
+ def error (self, code):
+ self.reply_code = code
+ message = self.responses[code]
+ s = self.DEFAULT_ERROR_MESSAGE % {
+ 'code': code,
+ 'message': message,
+ }
+ self['Content-Length'] = len(s)
+ self['Content-Type'] = 'text/html'
+ # make an error reply
+ self.push (s)
+ self.done()
+
+ # can also be used for empty replies
+ reply_now = error
+
+ def done (self):
+ "finalize this transaction - send output to the http channel"
+
+ # ----------------------------------------
+ # persistent connection management
+ # ----------------------------------------
+
+ # --- BUCKLE UP! ----
+
+ connection = string.lower (get_header (CONNECTION, self.header))
+
+ close_it = 0
+ wrap_in_chunking = 0
+
+ if self.version == '1.0':
+ if connection == 'keep-alive':
+ if not self.has_key ('Content-Length'):
+ close_it = 1
+ else:
+ self['Connection'] = 'Keep-Alive'
+ else:
+ close_it = 1
+ elif self.version == '1.1':
+ if connection == 'close':
+ close_it = 1
+ elif not self.has_key ('Content-Length'):
+ if self.has_key ('Transfer-Encoding'):
+ if not self['Transfer-Encoding'] == 'chunked':
+ close_it = 1
+ elif self.use_chunked:
+ self['Transfer-Encoding'] = 'chunked'
+ wrap_in_chunking = 1
+ else:
+ close_it = 1
+ elif self.version is None:
+ # Although we don't *really* support http/0.9 (because we'd have to
+ # use \r\n as a terminator, and it would just yuck up a lot of stuff)
+ # it's very common for developers to not want to type a version number
+ # when using telnet to debug a server.
+ close_it = 1
+
+ outgoing_header = producers.simple_producer(self.get_reply_header_text())
+
+ if close_it:
+ self['Connection'] = 'close'
+
+ if wrap_in_chunking:
+ outgoing_producer = producers.chunked_producer (
+ producers.composite_producer (self.outgoing)
+ )
+ # prepend the header
+ outgoing_producer = producers.composite_producer(
+ [outgoing_header, outgoing_producer]
+ )
+ else:
+ # prepend the header
+ self.outgoing.insert(0, outgoing_header)
+ outgoing_producer = producers.composite_producer (self.outgoing)
+
+ # apply a few final transformations to the output
+ self.channel.push_with_producer (
+ # globbing gives us large packets
+ producers.globbing_producer (
+ # hooking lets us log the number of bytes sent
+ producers.hooked_producer (
+ outgoing_producer,
+ self.log
+ )
+ )
+ )
+
+ self.channel.current_request = None
+
+ if close_it:
+ self.channel.close_when_done()
+
+ def log_date_string (self, when):
+ gmt = time.gmtime(when)
+ if time.daylight and gmt[8]:
+ tz = time.altzone
+ else:
+ tz = time.timezone
+ if tz > 0:
+ neg = 1
+ else:
+ neg = 0
+ tz = -tz
+ h, rem = divmod (tz, 3600)
+ m, rem = divmod (rem, 60)
+ if neg:
+ offset = '-%02d%02d' % (h, m)
+ else:
+ offset = '+%02d%02d' % (h, m)
+
+ return time.strftime ( '%d/%b/%Y:%H:%M:%S ', gmt) + offset
+
+ def log (self, bytes):
+ self.channel.server.logger.log (
+ self.channel.addr[0],
+ '%d - - [%s] "%s" %d %d\n' % (
+ self.channel.addr[1],
+ self.log_date_string (time.time()),
+ self.request,
+ self.reply_code,
+ bytes
+ )
+ )
+
+ responses = {
+ 100: "Continue",
+ 101: "Switching Protocols",
+ 200: "OK",
+ 201: "Created",
+ 202: "Accepted",
+ 203: "Non-Authoritative Information",
+ 204: "No Content",
+ 205: "Reset Content",
+ 206: "Partial Content",
+ 300: "Multiple Choices",
+ 301: "Moved Permanently",
+ 302: "Moved Temporarily",
+ 303: "See Other",
+ 304: "Not Modified",
+ 305: "Use Proxy",
+ 400: "Bad Request",
+ 401: "Unauthorized",
+ 402: "Payment Required",
+ 403: "Forbidden",
+ 404: "Not Found",
+ 405: "Method Not Allowed",
+ 406: "Not Acceptable",
+ 407: "Proxy Authentication Required",
+ 408: "Request Time-out",
+ 409: "Conflict",
+ 410: "Gone",
+ 411: "Length Required",
+ 412: "Precondition Failed",
+ 413: "Request Entity Too Large",
+ 414: "Request-URI Too Large",
+ 415: "Unsupported Media Type",
+ 500: "Internal Server Error",
+ 501: "Not Implemented",
+ 502: "Bad Gateway",
+ 503: "Service Unavailable",
+ 504: "Gateway Time-out",
+ 505: "HTTP Version not supported"
+ }
+
+ # Default error message
+ DEFAULT_ERROR_MESSAGE = string.join (
+ ['<head>',
+ '<title>Error response</title>',
+ '</head>',
+ '<body>',
+ '<h1>Error response</h1>',
+ '<p>Error code %(code)d.',
+ '<p>Message: %(message)s.',
+ '</body>',
+ ''
+ ],
+ '\r\n'
+ )
+
+
+# ===========================================================================
+# HTTP Channel Object
+# ===========================================================================
+
+class http_channel (asynchat.async_chat):
+
+ # use a larger default output buffer
+ ac_out_buffer_size = 1<<16
+
+ current_request = None
+ channel_counter = counter()
+
+ def __init__ (self, server, conn, addr):
+ self.channel_number = http_channel.channel_counter.increment()
+ self.request_counter = counter()
+ asynchat.async_chat.__init__ (self, conn)
+ self.server = server
+ self.addr = addr
+ self.set_terminator ('\r\n\r\n')
+ self.in_buffer = ''
+ self.creation_time = int (time.time())
+ self.check_maintenance()
+
+ def __repr__ (self):
+ ar = asynchat.async_chat.__repr__(self)[1:-1]
+ return '<%s channel#: %s requests:%s>' % (
+ ar,
+ self.channel_number,
+ self.request_counter
+ )
+
+ # Channel Counter, Maintenance Interval...
+ maintenance_interval = 500
+
+ def check_maintenance (self):
+ if not self.channel_number % self.maintenance_interval:
+ self.maintenance()
+
+ def maintenance (self):
+ self.kill_zombies()
+
+ # 30-minute zombie timeout. status_handler also knows how to kill zombies.
+ zombie_timeout = 30 * 60
+
+ def kill_zombies (self):
+ now = int (time.time())
+ for channel in asyncore.socket_map.values():
+ if channel.__class__ == self.__class__:
+ if (now - channel.creation_time) > channel.zombie_timeout:
+ channel.close()
+
+ # --------------------------------------------------
+ # send/recv overrides, good place for instrumentation.
+ # --------------------------------------------------
+
+ # this information needs to get into the request object,
+ # so that it may log correctly.
+ def send (self, data):
+ result = asynchat.async_chat.send (self, data)
+ self.server.bytes_out.increment (len(data))
+ return result
+
+ def recv (self, buffer_size):
+ try:
+ result = asynchat.async_chat.recv (self, buffer_size)
+ self.server.bytes_in.increment (len(result))
+ return result
+ except MemoryError:
+ # --- Save a Trip to Your Service Provider ---
+ # It's possible for a process to eat up all the memory of
+ # the machine, and put it in an extremely wedged state,
+ # where medusa keeps running and can't be shut down. This
+ # is where MemoryError tends to get thrown, though of
+ # course it could get thrown elsewhere.
+ sys.exit ("Out of Memory!")
+
+ def handle_error (self):
+ t, v = sys.exc_info()[:2]
+ if t is SystemExit:
+ raise t, v
+ else:
+ asynchat.async_chat.handle_error (self)
+
+ def log (self, *args):
+ pass
+
+ # --------------------------------------------------
+ # async_chat methods
+ # --------------------------------------------------
+
+ def collect_incoming_data (self, data):
+ if self.current_request:
+ # we are receiving data (probably POST data) for a request
+ self.current_request.collect_incoming_data (data)
+ else:
+ # we are receiving header (request) data
+ self.in_buffer = self.in_buffer + data
+
+ def found_terminator (self):
+ if self.current_request:
+ self.current_request.found_terminator()
+ else:
+ header = self.in_buffer
+ self.in_buffer = ''
+ lines = string.split (header, '\r\n')
+
+ # --------------------------------------------------
+ # crack the request header
+ # --------------------------------------------------
+
+ while lines and not lines[0]:
+ # as per the suggestion of http-1.1 section 4.1, (and
+ # Eric Parker <eparker at zyvex.com>), ignore a leading
+ # blank lines (buggy browsers tack it onto the end of
+ # POST requests)
+ lines = lines[1:]
+
+ if not lines:
+ self.close_when_done()
+ return
+
+ request = lines[0]
+
+ command, uri, version = crack_request (request)
+ header = join_headers (lines[1:])
+
+ # unquote path if necessary (thanks to Skip Montanaro for pointing
+ # out that we must unquote in piecemeal fashion).
+ rpath, rquery = splitquery(uri)
+ if '%' in rpath:
+ if rquery:
+ uri = unquote (rpath) + '?' + rquery
+ else:
+ uri = unquote (rpath)
+
+ r = http_request (self, request, command, uri, version, header)
+ self.request_counter.increment()
+ self.server.total_requests.increment()
+
+ if command is None:
+ self.log_info ('Bad HTTP request: %s' % repr(request), 'error')
+ r.error (400)
+ return
+
+ # --------------------------------------------------
+ # handler selection and dispatch
+ # --------------------------------------------------
+ for h in self.server.handlers:
+ if h.match (r):
+ try:
+ self.current_request = r
+ # This isn't used anywhere.
+ # r.handler = h # CYCLE
+ h.handle_request (r)
+ except:
+ self.server.exceptions.increment()
+ (file, fun, line), t, v, tbinfo = asyncore.compact_traceback()
+ self.log_info(
+ 'Server Error: %s, %s: file: %s line: %s' % (t,v,file,line),
+ 'error')
+ try:
+ r.error (500)
+ except:
+ pass
+ return
+
+ # no handlers, so complain
+ r.error (404)
+
+ def writable_for_proxy (self):
+ # this version of writable supports the idea of a 'stalled' producer
+ # [i.e., it's not ready to produce any output yet] This is needed by
+ # the proxy, which will be waiting for the magic combination of
+ # 1) hostname resolved
+ # 2) connection made
+ # 3) data available.
+ if self.ac_out_buffer:
+ return 1
+ elif len(self.producer_fifo):
+ p = self.producer_fifo.first()
+ if hasattr (p, 'stalled'):
+ return not p.stalled()
+ else:
+ return 1
+
+# ===========================================================================
+# HTTP Server Object
+# ===========================================================================
+
+class http_server (asyncore.dispatcher):
+
+ SERVER_IDENT = 'HTTP Server (V%s)' % VERSION_STRING
+
+ channel_class = http_channel
+
+ def __init__ (self, ip, port, resolver=None, logger_object=None):
+ self.ip = ip
+ self.port = port
+ asyncore.dispatcher.__init__ (self)
+ self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
+
+ self.handlers = []
+
+ if not logger_object:
+ logger_object = logger.file_logger (sys.stdout)
+
+ self.set_reuse_addr()
+ self.bind ((ip, port))
+
+ # lower this to 5 if your OS complains
+ self.listen (1024)
+
+ host, port = self.socket.getsockname()
+ if not ip:
+ self.log_info('Computing default hostname', 'warning')
+ ip = socket.gethostbyname (socket.gethostname())
+ try:
+ self.server_name = socket.gethostbyaddr (ip)[0]
+ except socket.error:
+ self.log_info('Cannot do reverse lookup', 'warning')
+ self.server_name = ip # use the IP address as the "hostname"
+
+ self.server_port = port
+ self.total_clients = counter()
+ self.total_requests = counter()
+ self.exceptions = counter()
+ self.bytes_out = counter()
+ self.bytes_in = counter()
+
+ if not logger_object:
+ logger_object = logger.file_logger (sys.stdout)
+
+ if resolver:
+ self.logger = logger.resolving_logger (resolver, logger_object)
+ else:
+ self.logger = logger.unresolving_logger (logger_object)
+
+ self.log_info (
+ 'Medusa (V%s) started at %s'
+ '\n\tHostname: %s'
+ '\n\tPort:%d'
+ '\n' % (
+ VERSION_STRING,
+ time.ctime(time.time()),
+ self.server_name,
+ port,
+ )
+ )
+
+ def writable (self):
+ return 0
+
+ def handle_read (self):
+ pass
+
+ def readable (self):
+ return self.accepting
+
+ def handle_connect (self):
+ pass
+
+ def handle_accept (self):
+ self.total_clients.increment()
+ try:
+ conn, addr = self.accept()
+ except socket.error:
+ # linux: on rare occasions we get a bogus socket back from
+ # accept. socketmodule.c:makesockaddr complains that the
+ # address family is unknown. We don't want the whole server
+ # to shut down because of this.
+ self.log_info ('warning: server accept() threw an exception', 'warning')
+ return
+ except TypeError:
+ # unpack non-sequence. this can happen when a read event
+ # fires on a listening socket, but when we call accept()
+ # we get EWOULDBLOCK, so dispatcher.accept() returns None.
+ # Seen on FreeBSD3.
+ self.log_info ('warning: server accept() threw EWOULDBLOCK', 'warning')
+ return
+
+ self.channel_class (self, conn, addr)
+
+ def install_handler (self, handler, back=0):
+ if back:
+ self.handlers.append (handler)
+ else:
+ self.handlers.insert (0, handler)
+
+ def remove_handler (self, handler):
+ self.handlers.remove (handler)
+
+ def status (self):
+ def nice_bytes (n):
+ return string.join (status_handler.english_bytes (n))
+
+ handler_stats = filter (None, map (maybe_status, self.handlers))
+
+ if self.total_clients:
+ ratio = self.total_requests.as_long() / float(self.total_clients.as_long())
+ else:
+ ratio = 0.0
+
+ return producers.composite_producer (
+ [producers.lines_producer (
+ ['<h2>%s</h2>' % self.SERVER_IDENT,
+ '<br>Listening on: <b>Host:</b> %s' % self.server_name,
+ '<b>Port:</b> %d' % self.port,
+ '<p><ul>'
+ '<li>Total <b>Clients:</b> %s' % self.total_clients,
+ '<b>Requests:</b> %s' % self.total_requests,
+ '<b>Requests/Client:</b> %.1f' % (ratio),
+ '<li>Total <b>Bytes In:</b> %s' % (nice_bytes (self.bytes_in.as_long())),
+ '<b>Bytes Out:</b> %s' % (nice_bytes (self.bytes_out.as_long())),
+ '<li>Total <b>Exceptions:</b> %s' % self.exceptions,
+ '</ul><p>'
+ '<b>Extension List</b><ul>',
+ ])] + handler_stats + [producers.simple_producer('</ul>')]
+ )
+
+def maybe_status (thing):
+ if hasattr (thing, 'status'):
+ return thing.status()
+ else:
+ return None
+
+CONNECTION = re.compile ('Connection: (.*)', re.IGNORECASE)
+
+# merge multi-line headers
+# [486dx2: ~500/sec]
+def join_headers (headers):
+ r = []
+ for i in range(len(headers)):
+ if headers[i][0] in ' \t':
+ r[-1] = r[-1] + headers[i][1:]
+ else:
+ r.append (headers[i])
+ return r
+
+def get_header (head_reg, lines, group=1):
+ for line in lines:
+ m = head_reg.match (line)
+ if m and m.end() == len(line):
+ return m.group (group)
+ return ''
+
+def get_header_match (head_reg, lines):
+ for line in lines:
+ m = head_reg.match (line)
+ if m and m.end() == len(line):
+ return m
+ return ''
+
+REQUEST = re.compile ('([^ ]+) ([^ ]+)(( HTTP/([0-9.]+))$|$)')
+
+def crack_request (r):
+ m = REQUEST.match (r)
+ if m and m.end() == len(r):
+ if m.group(3):
+ version = m.group(5)
+ else:
+ version = None
+ return m.group(1), m.group(2), version
+ else:
+ return None, None, None
+
+if __name__ == '__main__':
+ import sys
+ if len(sys.argv) < 2:
+ print 'usage: %s <root> <port>' % (sys.argv[0])
+ else:
+ import monitor
+ import filesys
+ import default_handler
+ import status_handler
+ import ftp_server
+ import chat_server
+ import resolver
+ import logger
+ rs = resolver.caching_resolver ('127.0.0.1')
+ lg = logger.file_logger (sys.stdout)
+ ms = monitor.secure_monitor_server ('fnord', '127.0.0.1', 9999)
+ fs = filesys.os_filesystem (sys.argv[1])
+ dh = default_handler.default_handler (fs)
+ hs = http_server ('', string.atoi (sys.argv[2]), rs, lg)
+ hs.install_handler (dh)
+ ftp = ftp_server.ftp_server (
+ ftp_server.dummy_authorizer(sys.argv[1]),
+ port=8021,
+ resolver=rs,
+ logger_object=lg
+ )
+ cs = chat_server.chat_server ('', 7777)
+ sh = status_handler.status_extension([hs,ms,ftp,cs,rs])
+ hs.install_handler (sh)
+ if ('-p' in sys.argv):
+ def profile_loop ():
+ try:
+ asyncore.loop()
+ except KeyboardInterrupt:
+ pass
+ import profile
+ profile.run ('profile_loop()', 'profile.out')
+ else:
+ asyncore.loop()
Added: supervisor/trunk/src/medusa/logger.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/logger.py Fri May 22 23:51:54 2009
@@ -0,0 +1,261 @@
+# -*- Mode: Python -*-
+
+import asynchat_25 as asynchat
+import socket
+import time # these three are for the rotating logger
+import os # |
+import stat # v
+
+#
+# three types of log:
+# 1) file
+# with optional flushing. Also, one that rotates the log.
+# 2) socket
+# dump output directly to a socket connection. [how do we
+# keep it open?]
+# 3) syslog
+# log to syslog via tcp. this is a per-line protocol.
+#
+
+#
+# The 'standard' interface to a logging object is simply
+# log_object.log (message)
+#
+
+# a file-like object that captures output, and
+# makes sure to flush it always... this could
+# be connected to:
+# o stdio file
+# o low-level file
+# o socket channel
+# o syslog output...
+
+class file_logger:
+
+ # pass this either a path or a file object.
+ def __init__ (self, file, flush=1, mode='a'):
+ if type(file) == type(''):
+ if (file == '-'):
+ import sys
+ self.file = sys.stdout
+ else:
+ self.file = open (file, mode)
+ else:
+ self.file = file
+ self.do_flush = flush
+
+ def __repr__ (self):
+ return '<file logger: %s>' % self.file
+
+ def write (self, data):
+ self.file.write (data)
+ self.maybe_flush()
+
+ def writeline (self, line):
+ self.file.writeline (line)
+ self.maybe_flush()
+
+ def writelines (self, lines):
+ self.file.writelines (lines)
+ self.maybe_flush()
+
+ def maybe_flush (self):
+ if self.do_flush:
+ self.file.flush()
+
+ def flush (self):
+ self.file.flush()
+
+ def softspace (self, *args):
+ pass
+
+ def log (self, message):
+ if message[-1] not in ('\r', '\n'):
+ self.write (message + '\n')
+ else:
+ self.write (message)
+
+# like a file_logger, but it must be attached to a filename.
+# When the log gets too full, or a certain time has passed,
+# it backs up the log and starts a new one. Note that backing
+# up the log is done via "mv" because anything else (cp, gzip)
+# would take time, during which medusa would do nothing else.
+
+class rotating_file_logger (file_logger):
+
+ # If freq is non-None we back up "daily", "weekly", or "monthly".
+ # Else if maxsize is non-None we back up whenever the log gets
+ # to big. If both are None we never back up.
+ def __init__ (self, file, freq=None, maxsize=None, flush=1, mode='a'):
+ self.filename = file
+ self.mode = mode
+ self.file = open (file, mode)
+ self.freq = freq
+ self.maxsize = maxsize
+ self.rotate_when = self.next_backup(self.freq)
+ self.do_flush = flush
+
+ def __repr__ (self):
+ return '<rotating-file logger: %s>' % self.file
+
+ # We back up at midnight every 1) day, 2) monday, or 3) 1st of month
+ def next_backup (self, freq):
+ (yr, mo, day, hr, min, sec, wd, jday, dst) = time.localtime(time.time())
+ if freq == 'daily':
+ return time.mktime((yr,mo,day+1, 0,0,0, 0,0,-1))
+ elif freq == 'weekly':
+ return time.mktime((yr,mo,day-wd+7, 0,0,0, 0,0,-1)) # wd(monday)==0
+ elif freq == 'monthly':
+ return time.mktime((yr,mo+1,1, 0,0,0, 0,0,-1))
+ else:
+ return None # not a date-based backup
+
+ def maybe_flush (self): # rotate first if necessary
+ self.maybe_rotate()
+ if self.do_flush: # from file_logger()
+ self.file.flush()
+
+ def maybe_rotate (self):
+ if self.freq and time.time() > self.rotate_when:
+ self.rotate()
+ self.rotate_when = self.next_backup(self.freq)
+ elif self.maxsize: # rotate when we get too big
+ try:
+ if os.stat(self.filename)[stat.ST_SIZE] > self.maxsize:
+ self.rotate()
+ except os.error: # file not found, probably
+ self.rotate() # will create a new file
+
+ def rotate (self):
+ (yr, mo, day, hr, min, sec, wd, jday, dst) = time.localtime(time.time())
+ try:
+ self.file.close()
+ newname = '%s.ends%04d%02d%02d' % (self.filename, yr, mo, day)
+ try:
+ open(newname, "r").close() # check if file exists
+ newname = newname + "-%02d%02d%02d" % (hr, min, sec)
+ except: # YEARMODY is unique
+ pass
+ os.rename(self.filename, newname)
+ self.file = open(self.filename, self.mode)
+ except:
+ pass
+
+# syslog is a line-oriented log protocol - this class would be
+# appropriate for FTP or HTTP logs, but not for dumping stderr to.
+
+# TODO: a simple safety wrapper that will ensure that the line sent
+# to syslog is reasonable.
+
+# TODO: async version of syslog_client: now, log entries use blocking
+# send()
+
+import m_syslog
+syslog_logger = m_syslog.syslog_client
+
+class syslog_logger (m_syslog.syslog_client):
+ def __init__ (self, address, facility='user'):
+ m_syslog.syslog_client.__init__ (self, address)
+ self.facility = m_syslog.facility_names[facility]
+ self.address=address
+
+ def __repr__ (self):
+ return '<syslog logger address=%s>' % (repr(self.address))
+
+ def log (self, message):
+ m_syslog.syslog_client.log (
+ self,
+ message,
+ facility=self.facility,
+ priority=m_syslog.LOG_INFO
+ )
+
+# log to a stream socket, asynchronously
+
+class socket_logger (asynchat.async_chat):
+
+ def __init__ (self, address):
+
+ if type(address) == type(''):
+ self.create_socket (socket.AF_UNIX, socket.SOCK_STREAM)
+ else:
+ self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
+
+ self.connect (address)
+ self.address = address
+
+ def __repr__ (self):
+ return '<socket logger: address=%s>' % (self.address)
+
+ def log (self, message):
+ if message[-2:] != '\r\n':
+ self.socket.push (message + '\r\n')
+ else:
+ self.socket.push (message)
+
+# log to multiple places
+class multi_logger:
+ def __init__ (self, loggers):
+ self.loggers = loggers
+
+ def __repr__ (self):
+ return '<multi logger: %s>' % (repr(self.loggers))
+
+ def log (self, message):
+ for logger in self.loggers:
+ logger.log (message)
+
+class resolving_logger:
+ """Feed (ip, message) combinations into this logger to get a
+ resolved hostname in front of the message. The message will not
+ be logged until the PTR request finishes (or fails)."""
+
+ def __init__ (self, resolver, logger):
+ self.resolver = resolver
+ self.logger = logger
+
+ class logger_thunk:
+ def __init__ (self, message, logger):
+ self.message = message
+ self.logger = logger
+
+ def __call__ (self, host, ttl, answer):
+ if not answer:
+ answer = host
+ self.logger.log ('%s:%s' % (answer, self.message))
+
+ def log (self, ip, message):
+ self.resolver.resolve_ptr (
+ ip,
+ self.logger_thunk (
+ message,
+ self.logger
+ )
+ )
+
+class unresolving_logger:
+ "Just in case you don't want to resolve"
+ def __init__ (self, logger):
+ self.logger = logger
+
+ def log (self, ip, message):
+ self.logger.log ('%s:%s' % (ip, message))
+
+
+def strip_eol (line):
+ while line and line[-1] in '\r\n':
+ line = line[:-1]
+ return line
+
+class tail_logger:
+ "Keep track of the last <size> log messages"
+ def __init__ (self, logger, size=500):
+ self.size = size
+ self.logger = logger
+ self.messages = []
+
+ def log (self, message):
+ self.messages.append (strip_eol (message))
+ if len (self.messages) > self.size:
+ del self.messages[0]
+ self.logger.log (message)
Added: supervisor/trunk/src/medusa/m_syslog.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/m_syslog.py Fri May 22 23:51:54 2009
@@ -0,0 +1,182 @@
+# -*- Mode: Python -*-
+
+# ======================================================================
+# Copyright 1997 by Sam Rushing
+#
+# All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of Sam
+# Rushing not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+# ======================================================================
+
+"""socket interface to unix syslog.
+On Unix, there are usually two ways of getting to syslog: via a
+local unix-domain socket, or via the TCP service.
+
+Usually "/dev/log" is the unix domain socket. This may be different
+for other systems.
+
+>>> my_client = syslog_client ('/dev/log')
+
+Otherwise, just use the UDP version, port 514.
+
+>>> my_client = syslog_client (('my_log_host', 514))
+
+On win32, you will have to use the UDP version. Note that
+you can use this to log to other hosts (and indeed, multiple
+hosts).
+
+This module is not a drop-in replacement for the python
+<syslog> extension module - the interface is different.
+
+Usage:
+
+>>> c = syslog_client()
+>>> c = syslog_client ('/strange/non_standard_log_location')
+>>> c = syslog_client (('other_host.com', 514))
+>>> c.log ('testing', facility='local0', priority='debug')
+
+"""
+
+# TODO: support named-pipe syslog.
+# [see ftp://sunsite.unc.edu/pub/Linux/system/Daemons/syslog-fifo.tar.z]
+
+# from <linux/sys/syslog.h>:
+# ===========================================================================
+# priorities/facilities are encoded into a single 32-bit quantity, where the
+# bottom 3 bits are the priority (0-7) and the top 28 bits are the facility
+# (0-big number). Both the priorities and the facilities map roughly
+# one-to-one to strings in the syslogd(8) source code. This mapping is
+# included in this file.
+#
+# priorities (these are ordered)
+
+LOG_EMERG = 0 # system is unusable
+LOG_ALERT = 1 # action must be taken immediately
+LOG_CRIT = 2 # critical conditions
+LOG_ERR = 3 # error conditions
+LOG_WARNING = 4 # warning conditions
+LOG_NOTICE = 5 # normal but significant condition
+LOG_INFO = 6 # informational
+LOG_DEBUG = 7 # debug-level messages
+
+# facility codes
+LOG_KERN = 0 # kernel messages
+LOG_USER = 1 # random user-level messages
+LOG_MAIL = 2 # mail system
+LOG_DAEMON = 3 # system daemons
+LOG_AUTH = 4 # security/authorization messages
+LOG_SYSLOG = 5 # messages generated internally by syslogd
+LOG_LPR = 6 # line printer subsystem
+LOG_NEWS = 7 # network news subsystem
+LOG_UUCP = 8 # UUCP subsystem
+LOG_CRON = 9 # clock daemon
+LOG_AUTHPRIV = 10 # security/authorization messages (private)
+
+# other codes through 15 reserved for system use
+LOG_LOCAL0 = 16 # reserved for local use
+LOG_LOCAL1 = 17 # reserved for local use
+LOG_LOCAL2 = 18 # reserved for local use
+LOG_LOCAL3 = 19 # reserved for local use
+LOG_LOCAL4 = 20 # reserved for local use
+LOG_LOCAL5 = 21 # reserved for local use
+LOG_LOCAL6 = 22 # reserved for local use
+LOG_LOCAL7 = 23 # reserved for local use
+
+priority_names = {
+ "alert": LOG_ALERT,
+ "crit": LOG_CRIT,
+ "debug": LOG_DEBUG,
+ "emerg": LOG_EMERG,
+ "err": LOG_ERR,
+ "error": LOG_ERR, # DEPRECATED
+ "info": LOG_INFO,
+ "notice": LOG_NOTICE,
+ "panic": LOG_EMERG, # DEPRECATED
+ "warn": LOG_WARNING, # DEPRECATED
+ "warning": LOG_WARNING,
+ }
+
+facility_names = {
+ "auth": LOG_AUTH,
+ "authpriv": LOG_AUTHPRIV,
+ "cron": LOG_CRON,
+ "daemon": LOG_DAEMON,
+ "kern": LOG_KERN,
+ "lpr": LOG_LPR,
+ "mail": LOG_MAIL,
+ "news": LOG_NEWS,
+ "security": LOG_AUTH, # DEPRECATED
+ "syslog": LOG_SYSLOG,
+ "user": LOG_USER,
+ "uucp": LOG_UUCP,
+ "local0": LOG_LOCAL0,
+ "local1": LOG_LOCAL1,
+ "local2": LOG_LOCAL2,
+ "local3": LOG_LOCAL3,
+ "local4": LOG_LOCAL4,
+ "local5": LOG_LOCAL5,
+ "local6": LOG_LOCAL6,
+ "local7": LOG_LOCAL7,
+ }
+
+import socket
+
+class syslog_client:
+ def __init__ (self, address='/dev/log'):
+ self.address = address
+ self.stream = 0
+ if isinstance(address, type('')):
+ try:
+ self.socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
+ self.socket.connect(address)
+ except socket.error:
+ # Some Linux installations have /dev/log
+ # a stream socket instead of a datagram socket.
+ self.socket = socket.socket (socket.AF_UNIX,
+ socket.SOCK_STREAM)
+ self.stream = 1
+ else:
+ self.socket = socket.socket (socket.AF_INET, socket.SOCK_DGRAM)
+
+ # curious: when talking to the unix-domain '/dev/log' socket, a
+ # zero-terminator seems to be required. this string is placed
+ # into a class variable so that it can be overridden if
+ # necessary.
+
+ log_format_string = '<%d>%s\000'
+
+ def log (self, message, facility=LOG_USER, priority=LOG_INFO):
+ message = self.log_format_string % (
+ self.encode_priority (facility, priority),
+ message
+ )
+ if self.stream:
+ self.socket.send (message)
+ else:
+ self.socket.sendto (message, self.address)
+
+ def encode_priority (self, facility, priority):
+ if type(facility) == type(''):
+ facility = facility_names[facility]
+ if type(priority) == type(''):
+ priority = priority_names[priority]
+ return (facility<<3) | priority
+
+ def close (self):
+ if self.stream:
+ self.socket.close()
Added: supervisor/trunk/src/medusa/medusa_gif.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/medusa_gif.py Fri May 22 23:51:54 2009
@@ -0,0 +1,8 @@
+# -*- Mode: Python -*-
+
+# the medusa icon as a python source file.
+
+width = 97
+height = 61
+
+data = 'GIF89aa\000=\000\204\000\000\000\000\000\255\255\255\245\245\245ssskkkccc111)))\326\326\326!!!\316\316\316\300\300\300\204\204\000\224\224\224\214\214\214\200\200\200RRR\377\377\377JJJ\367\367\367BBB\347\347\347\000\204\000\020\020\020\265\265\265\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000!\371\004\001\000\000\021\000,\000\000\000\000a\000=\000\000\005\376`$\216di\236h\252\256l\353\276p,\317tm\337x\256\357|m\001@\240E\305\000\364\2164\206R)$\005\201\214\007r\012{X\255\312a\004\260\\>\026\3240\353)\224n\001W+X\334\373\231~\344.\303b\216\024\027x<\273\307\255G,rJiWN\014{S}k"?ti\013EdPQ\207G at _%\000\026yy\\\201\202\227\224<\221Fs$pOjWz\241<r at vO\236\231\233k\247M\2544\203F\177\235\236L#\247\256Z\270,\266BxJ[\276\256A]iE\304\305\262\273E\313\201\275i#\\\303\321\'h\203V\\\177\326\276\216\220P~\335\230_\264\013\342\275\344KF\233\360Q\212\352\246\000\367\274s\361\236\334\347T\341;\341\246\2202\177\3142\211`\242o\325 at S\202\264\031\252\207\260\323\256\205\311\036\236\270\002\'\013\302\177\274H\010\324X\002\0176\212\037\376\321\360\032\226\207\244\2674(+^\202\346r\205J\0211\375\241Y#\256f\0127\315>\272\002\325\307g\012(\007\205\312#j\317(\012A\200\224.\241\003\346GS\247\033\245\344\264\366\015L\'PXQl]\266\263\243\232\260?\245\316\371\362\225\035\332\243J\273\332Q\263\357-D\241T\327\270\265\013W&\330\010u\371b\322IW0\214\261]\003\033Va\365Z#\207\213a\030k\2647\262\014p\354\024[n\321N\363\346\317\003\037P\000\235C\302\000\3228(\244\363YaA\005\022\255_\237@\260\000A\212\326\256qbp\321\332\266\011\334=T\023\010"!B\005\003A\010\224\020\220 H\002\337#\020 O\276E\357h\221\327\003\\\000b at v\004\351A.h\365\354\342B\002\011\257\025\\ \220\340\301\353\006\000\024\214\200pA\300\353\012\364\241k/\340\033C\202\003\000\310fZ\011\003V\240R\005\007\354\376\026A\000\000\360\'\202\177\024\004\210\003\000\305\215\360\000\000\015\220\240\332\203\027@\'\202\004\025VpA\000%\210x\321\206\032J\341\316\010\262\211H"l\333\341\200\200>"]P\002\212\011\010`\002\0066FP\200\001\'\024p]\004\027(8B\221\306]\000\201w>\002iB\001\007\340\260"v7J1\343(\257\020\251\243\011\242i\263\017\215\337\035\220\200\221\365m4d\015\016D\251\341iN\354\346Ng\253\200I\240\031\35609\245\2057\311I\302\2007t\231"&`\314\310\244\011e\226(\236\010w\212\300\234\011\012HX(\214\253\311@\001\233^\222pg{% \340\035\224&H\000\246\201\362\215`@\001"L\340\004\030\234\022\250\'\015(V:\302\235\030\240q\337\205\224\212h@\177\006\000\250\210\004\007\310\207\337\005\257-P\346\257\367]p\353\203\271\256:\203\236\211F\340\247\010\3329g\244\010\307*=A\000\203\260y\012\304s#\014\007D\207,N\007\304\265\027\021C\233\207%B\366[m\353\006\006\034j\360\306+\357\274a\204\000\000;'
Added: supervisor/trunk/src/medusa/monitor.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/monitor.py Fri May 22 23:51:54 2009
@@ -0,0 +1,346 @@
+# -*- Mode: Python -*-
+# Author: Sam Rushing <rushing at nightmare.com>
+
+#
+# python REPL channel.
+#
+
+RCS_ID = '$Id: monitor.py,v 1.5 2002/03/23 15:08:06 amk Exp $'
+
+import md5
+import socket
+import string
+import sys
+import time
+
+VERSION = string.split(RCS_ID)[2]
+
+import asyncore_25 as asyncore
+import asynchat_25 as asynchat
+
+from counter import counter
+import producers
+
+class monitor_channel (asynchat.async_chat):
+ try_linemode = 1
+
+ def __init__ (self, server, sock, addr):
+ asynchat.async_chat.__init__ (self, sock)
+ self.server = server
+ self.addr = addr
+ self.set_terminator ('\r\n')
+ self.data = ''
+ # local bindings specific to this channel
+ self.local_env = sys.modules['__main__'].__dict__.copy()
+ self.push ('Python ' + sys.version + '\r\n')
+ self.push (sys.copyright+'\r\n')
+ self.push ('Welcome to %s\r\n' % self)
+ self.push ("[Hint: try 'from __main__ import *']\r\n")
+ self.prompt()
+ self.number = server.total_sessions.as_long()
+ self.line_counter = counter()
+ self.multi_line = []
+
+ def handle_connect (self):
+ # send IAC DO LINEMODE
+ self.push ('\377\375\"')
+
+ def close (self):
+ self.server.closed_sessions.increment()
+ asynchat.async_chat.close(self)
+
+ def prompt (self):
+ self.push ('>>> ')
+
+ def collect_incoming_data (self, data):
+ self.data = self.data + data
+ if len(self.data) > 1024:
+ # denial of service.
+ self.push ('BCNU\r\n')
+ self.close_when_done()
+
+ def found_terminator (self):
+ line = self.clean_line (self.data)
+ self.data = ''
+ self.line_counter.increment()
+ # check for special case inputs...
+ if not line and not self.multi_line:
+ self.prompt()
+ return
+ if line in ['\004', 'exit']:
+ self.push ('BCNU\r\n')
+ self.close_when_done()
+ return
+ oldout = sys.stdout
+ olderr = sys.stderr
+ try:
+ p = output_producer(self, olderr)
+ sys.stdout = p
+ sys.stderr = p
+ try:
+ # this is, of course, a blocking operation.
+ # if you wanted to thread this, you would have
+ # to synchronize, etc... and treat the output
+ # like a pipe. Not Fun.
+ #
+ # try eval first. If that fails, try exec. If that fails,
+ # hurl.
+ try:
+ if self.multi_line:
+ # oh, this is horrible...
+ raise SyntaxError
+ co = compile (line, repr(self), 'eval')
+ result = eval (co, self.local_env)
+ method = 'eval'
+ if result is not None:
+ print repr(result)
+ self.local_env['_'] = result
+ except SyntaxError:
+ try:
+ if self.multi_line:
+ if line and line[0] in [' ','\t']:
+ self.multi_line.append (line)
+ self.push ('... ')
+ return
+ else:
+ self.multi_line.append (line)
+ line = string.join (self.multi_line, '\n')
+ co = compile (line, repr(self), 'exec')
+ self.multi_line = []
+ else:
+ co = compile (line, repr(self), 'exec')
+ except SyntaxError, why:
+ if why[0] == 'unexpected EOF while parsing':
+ self.push ('... ')
+ self.multi_line.append (line)
+ return
+ else:
+ t,v,tb = sys.exc_info()
+ del tb
+ raise t,v
+ exec co in self.local_env
+ method = 'exec'
+ except:
+ method = 'exception'
+ self.multi_line = []
+ (file, fun, line), t, v, tbinfo = asyncore.compact_traceback()
+ self.log_info('%s %s %s' %(t, v, tbinfo), 'warning')
+ finally:
+ sys.stdout = oldout
+ sys.stderr = olderr
+ self.log_info('%s:%s (%s)> %s' % (
+ self.number,
+ self.line_counter,
+ method,
+ repr(line))
+ )
+ self.push_with_producer (p)
+ self.prompt()
+
+ # for now, we ignore any telnet option stuff sent to
+ # us, and we process the backspace key ourselves.
+ # gee, it would be fun to write a full-blown line-editing
+ # environment, etc...
+ def clean_line (self, line):
+ chars = []
+ for ch in line:
+ oc = ord(ch)
+ if oc < 127:
+ if oc in [8,177]:
+ # backspace
+ chars = chars[:-1]
+ else:
+ chars.append (ch)
+ return string.join (chars, '')
+
+class monitor_server (asyncore.dispatcher):
+
+ SERVER_IDENT = 'Monitor Server (V%s)' % VERSION
+
+ channel_class = monitor_channel
+
+ def __init__ (self, hostname='127.0.0.1', port=8023):
+ asyncore.dispatcher.__init__(self)
+ self.hostname = hostname
+ self.port = port
+ self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
+ self.set_reuse_addr()
+ self.bind ((hostname, port))
+ self.log_info('%s started on port %d' % (self.SERVER_IDENT, port))
+ self.listen (5)
+ self.closed = 0
+ self.failed_auths = 0
+ self.total_sessions = counter()
+ self.closed_sessions = counter()
+
+ def writable (self):
+ return 0
+
+ def handle_accept (self):
+ conn, addr = self.accept()
+ self.log_info('Incoming monitor connection from %s:%d' % addr)
+ self.channel_class (self, conn, addr)
+ self.total_sessions.increment()
+
+ def status (self):
+ return producers.simple_producer (
+ '<h2>%s</h2>' % self.SERVER_IDENT
+ + '<br><b>Total Sessions:</b> %s' % self.total_sessions
+ + '<br><b>Current Sessions:</b> %d' % (
+ self.total_sessions.as_long()-self.closed_sessions.as_long()
+ )
+ )
+
+def hex_digest (s):
+ m = md5.md5()
+ m.update (s)
+ return string.joinfields (
+ map (lambda x: hex (ord (x))[2:], map (None, m.digest())),
+ '',
+ )
+
+class secure_monitor_channel (monitor_channel):
+ authorized = 0
+
+ def __init__ (self, server, sock, addr):
+ asynchat.async_chat.__init__ (self, sock)
+ self.server = server
+ self.addr = addr
+ self.set_terminator ('\r\n')
+ self.data = ''
+ # local bindings specific to this channel
+ self.local_env = {}
+ # send timestamp string
+ self.timestamp = str(time.time())
+ self.count = 0
+ self.line_counter = counter()
+ self.number = int(server.total_sessions.as_long())
+ self.multi_line = []
+ self.push (self.timestamp + '\r\n')
+
+ def found_terminator (self):
+ if not self.authorized:
+ if hex_digest ('%s%s' % (self.timestamp, self.server.password)) != self.data:
+ self.log_info ('%s: failed authorization' % self, 'warning')
+ self.server.failed_auths = self.server.failed_auths + 1
+ self.close()
+ else:
+ self.authorized = 1
+ self.push ('Python ' + sys.version + '\r\n')
+ self.push (sys.copyright+'\r\n')
+ self.push ('Welcome to %s\r\n' % self)
+ self.prompt()
+ self.data = ''
+ else:
+ monitor_channel.found_terminator (self)
+
+class secure_encrypted_monitor_channel (secure_monitor_channel):
+ "Wrap send() and recv() with a stream cipher"
+
+ def __init__ (self, server, conn, addr):
+ key = server.password
+ self.outgoing = server.cipher.new (key)
+ self.incoming = server.cipher.new (key)
+ secure_monitor_channel.__init__ (self, server, conn, addr)
+
+ def send (self, data):
+ # send the encrypted data instead
+ ed = self.outgoing.encrypt (data)
+ return secure_monitor_channel.send (self, ed)
+
+ def recv (self, block_size):
+ data = secure_monitor_channel.recv (self, block_size)
+ if data:
+ dd = self.incoming.decrypt (data)
+ return dd
+ else:
+ return data
+
+class secure_monitor_server (monitor_server):
+ channel_class = secure_monitor_channel
+
+ def __init__ (self, password, hostname='', port=8023):
+ monitor_server.__init__ (self, hostname, port)
+ self.password = password
+
+ def status (self):
+ p = monitor_server.status (self)
+ # kludge
+ p.data = p.data + ('<br><b>Failed Authorizations:</b> %d' % self.failed_auths)
+ return p
+
+# don't try to print from within any of the methods
+# of this object. 8^)
+
+class output_producer:
+ def __init__ (self, channel, real_stderr):
+ self.channel = channel
+ self.data = ''
+ # use _this_ for debug output
+ self.stderr = real_stderr
+
+ def check_data (self):
+ if len(self.data) > 1<<16:
+ # runaway output, close it.
+ self.channel.close()
+
+ def write (self, data):
+ lines = string.splitfields (data, '\n')
+ data = string.join (lines, '\r\n')
+ self.data = self.data + data
+ self.check_data()
+
+ def writeline (self, line):
+ self.data = self.data + line + '\r\n'
+ self.check_data()
+
+ def writelines (self, lines):
+ self.data = self.data + string.joinfields (
+ lines,
+ '\r\n'
+ ) + '\r\n'
+ self.check_data()
+
+ def flush (self):
+ pass
+
+ def softspace (self, *args):
+ pass
+
+ def more (self):
+ if self.data:
+ result = self.data[:512]
+ self.data = self.data[512:]
+ return result
+ else:
+ return ''
+
+if __name__ == '__main__':
+ if '-s' in sys.argv:
+ sys.argv.remove ('-s')
+ print 'Enter password: ',
+ password = raw_input()
+ else:
+ password = None
+
+ if '-e' in sys.argv:
+ sys.argv.remove ('-e')
+ encrypt = 1
+ else:
+ encrypt = 0
+
+ if len(sys.argv) > 1:
+ port = string.atoi (sys.argv[1])
+ else:
+ port = 8023
+
+ if password is not None:
+ s = secure_monitor_server (password, '', port)
+ if encrypt:
+ s.channel_class = secure_encrypted_monitor_channel
+ import sapphire
+ s.cipher = sapphire
+ else:
+ s = monitor_server ('', port)
+
+ asyncore.loop(use_poll=1)
Added: supervisor/trunk/src/medusa/monitor_client.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/monitor_client.py Fri May 22 23:51:54 2009
@@ -0,0 +1,123 @@
+# -*- Mode: Python -*-
+
+# monitor client, unix version.
+
+import asyncore_25 as asyncore
+import asynchat_25 as asynchat
+import socket
+import string
+import sys
+import os
+
+import md5
+
+class stdin_channel (asyncore.file_dispatcher):
+ def handle_read (self):
+ data = self.recv(512)
+ if not data:
+ print '\nclosed.'
+ self.sock_channel.close()
+ try:
+ self.close()
+ except:
+ pass
+
+ data = string.replace(data, '\n', '\r\n')
+ self.sock_channel.push (data)
+
+ def writable (self):
+ return 0
+
+ def log (self, *ignore):
+ pass
+
+class monitor_client (asynchat.async_chat):
+ def __init__ (self, password, addr=('',8023), socket_type=socket.AF_INET):
+ asynchat.async_chat.__init__ (self)
+ self.create_socket (socket_type, socket.SOCK_STREAM)
+ self.terminator = '\r\n'
+ self.connect (addr)
+ self.sent_auth = 0
+ self.timestamp = ''
+ self.password = password
+
+ def collect_incoming_data (self, data):
+ if not self.sent_auth:
+ self.timestamp = self.timestamp + data
+ else:
+ sys.stdout.write (data)
+ sys.stdout.flush()
+
+ def found_terminator (self):
+ if not self.sent_auth:
+ self.push (hex_digest (self.timestamp + self.password) + '\r\n')
+ self.sent_auth = 1
+ else:
+ print
+
+ def handle_close (self):
+ # close all the channels, which will make the standard main
+ # loop exit.
+ map (lambda x: x.close(), asyncore.socket_map.values())
+
+ def log (self, *ignore):
+ pass
+
+class encrypted_monitor_client (monitor_client):
+ "Wrap push() and recv() with a stream cipher"
+
+ def init_cipher (self, cipher, key):
+ self.outgoing = cipher.new (key)
+ self.incoming = cipher.new (key)
+
+ def push (self, data):
+ # push the encrypted data instead
+ return monitor_client.push (self, self.outgoing.encrypt (data))
+
+ def recv (self, block_size):
+ data = monitor_client.recv (self, block_size)
+ if data:
+ return self.incoming.decrypt (data)
+ else:
+ return data
+
+def hex_digest (s):
+ m = md5.md5()
+ m.update (s)
+ return string.join (
+ map (lambda x: hex (ord (x))[2:], map (None, m.digest())),
+ '',
+ )
+
+if __name__ == '__main__':
+ if len(sys.argv) == 1:
+ print 'Usage: %s host port' % sys.argv[0]
+ sys.exit(0)
+
+ if ('-e' in sys.argv):
+ encrypt = 1
+ sys.argv.remove ('-e')
+ else:
+ encrypt = 0
+
+ sys.stderr.write ('Enter Password: ')
+ sys.stderr.flush()
+ try:
+ os.system ('stty -echo')
+ p = raw_input()
+ print
+ finally:
+ os.system ('stty echo')
+ stdin = stdin_channel (0)
+ if len(sys.argv) > 1:
+ if encrypt:
+ client = encrypted_monitor_client (p, (sys.argv[1], string.atoi (sys.argv[2])))
+ import sapphire
+ client.init_cipher (sapphire, p)
+ else:
+ client = monitor_client (p, (sys.argv[1], string.atoi (sys.argv[2])))
+ else:
+ # default to local host, 'standard' port
+ client = monitor_client (p)
+ stdin.sock_channel = client
+ asyncore.loop()
Added: supervisor/trunk/src/medusa/monitor_client_win32.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/monitor_client_win32.py Fri May 22 23:51:54 2009
@@ -0,0 +1,52 @@
+# -*- Mode: Python -*-
+
+# monitor client, win32 version
+
+# since we can't do select() on stdin/stdout, we simply
+# use threads and blocking sockets. <sigh>
+
+import socket
+import string
+import sys
+import thread
+import md5
+
+def hex_digest (s):
+ m = md5.md5()
+ m.update (s)
+ return string.join (
+ map (lambda x: hex (ord (x))[2:], map (None, m.digest())),
+ '',
+ )
+
+def reader (lock, sock, password):
+ # first grab the timestamp
+ ts = sock.recv (1024)[:-2]
+ sock.send (hex_digest (ts+password) + '\r\n')
+ while 1:
+ d = sock.recv (1024)
+ if not d:
+ lock.release()
+ print 'Connection closed. Hit <return> to exit'
+ thread.exit()
+ sys.stdout.write (d)
+ sys.stdout.flush()
+
+def writer (lock, sock, barrel="just kidding"):
+ while lock.locked():
+ sock.send (
+ sys.stdin.readline()[:-1] + '\r\n'
+ )
+
+if __name__ == '__main__':
+ if len(sys.argv) == 1:
+ print 'Usage: %s host port'
+ sys.exit(0)
+ print 'Enter Password: ',
+ p = raw_input()
+ s = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
+ s.connect ((sys.argv[1], string.atoi(sys.argv[2])))
+ l = thread.allocate_lock()
+ l.acquire()
+ thread.start_new_thread (reader, (l, s, p))
+ writer (l, s)
Added: supervisor/trunk/src/medusa/producers.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/producers.py Fri May 22 23:51:54 2009
@@ -0,0 +1,319 @@
+# -*- Mode: Python -*-
+
+RCS_ID = '$Id: producers.py,v 1.9 2004/04/21 13:56:28 akuchling Exp $'
+
+"""
+A collection of producers.
+Each producer implements a particular feature: They can be combined
+in various ways to get interesting and useful behaviors.
+
+For example, you can feed dynamically-produced output into the compressing
+producer, then wrap this with the 'chunked' transfer-encoding producer.
+"""
+
+import string
+from asynchat import find_prefix_at_end
+
+class simple_producer:
+ "producer for a string"
+ def __init__ (self, data, buffer_size=1024):
+ self.data = data
+ self.buffer_size = buffer_size
+
+ def more (self):
+ if len (self.data) > self.buffer_size:
+ result = self.data[:self.buffer_size]
+ self.data = self.data[self.buffer_size:]
+ return result
+ else:
+ result = self.data
+ self.data = ''
+ return result
+
+class scanning_producer:
+ "like simple_producer, but more efficient for large strings"
+ def __init__ (self, data, buffer_size=1024):
+ self.data = data
+ self.buffer_size = buffer_size
+ self.pos = 0
+
+ def more (self):
+ if self.pos < len(self.data):
+ lp = self.pos
+ rp = min (
+ len(self.data),
+ self.pos + self.buffer_size
+ )
+ result = self.data[lp:rp]
+ self.pos = self.pos + len(result)
+ return result
+ else:
+ return ''
+
+class lines_producer:
+ "producer for a list of lines"
+
+ def __init__ (self, lines):
+ self.lines = lines
+
+ def more (self):
+ if self.lines:
+ chunk = self.lines[:50]
+ self.lines = self.lines[50:]
+ return string.join (chunk, '\r\n') + '\r\n'
+ else:
+ return ''
+
+class buffer_list_producer:
+ "producer for a list of strings"
+
+ # i.e., data == string.join (buffers, '')
+
+ def __init__ (self, buffers):
+
+ self.index = 0
+ self.buffers = buffers
+
+ def more (self):
+ if self.index >= len(self.buffers):
+ return ''
+ else:
+ data = self.buffers[self.index]
+ self.index = self.index + 1
+ return data
+
+class file_producer:
+ "producer wrapper for file[-like] objects"
+
+ # match http_channel's outgoing buffer size
+ out_buffer_size = 1<<16
+
+ def __init__ (self, file):
+ self.done = 0
+ self.file = file
+
+ def more (self):
+ if self.done:
+ return ''
+ else:
+ data = self.file.read (self.out_buffer_size)
+ if not data:
+ self.file.close()
+ del self.file
+ self.done = 1
+ return ''
+ else:
+ return data
+
+# A simple output producer. This one does not [yet] have
+# the safety feature builtin to the monitor channel: runaway
+# output will not be caught.
+
+# don't try to print from within any of the methods
+# of this object.
+
+class output_producer:
+ "Acts like an output file; suitable for capturing sys.stdout"
+ def __init__ (self):
+ self.data = ''
+
+ def write (self, data):
+ lines = string.splitfields (data, '\n')
+ data = string.join (lines, '\r\n')
+ self.data = self.data + data
+
+ def writeline (self, line):
+ self.data = self.data + line + '\r\n'
+
+ def writelines (self, lines):
+ self.data = self.data + string.joinfields (
+ lines,
+ '\r\n'
+ ) + '\r\n'
+
+ def flush (self):
+ pass
+
+ def softspace (self, *args):
+ pass
+
+ def more (self):
+ if self.data:
+ result = self.data[:512]
+ self.data = self.data[512:]
+ return result
+ else:
+ return ''
+
+class composite_producer:
+ "combine a fifo of producers into one"
+ def __init__ (self, producers):
+ self.producers = producers
+
+ def more (self):
+ while len(self.producers):
+ p = self.producers[0]
+ d = p.more()
+ if d:
+ return d
+ else:
+ self.producers.pop(0)
+ else:
+ return ''
+
+
+class globbing_producer:
+ """
+ 'glob' the output from a producer into a particular buffer size.
+ helps reduce the number of calls to send(). [this appears to
+ gain about 30% performance on requests to a single channel]
+ """
+
+ def __init__ (self, producer, buffer_size=1<<16):
+ self.producer = producer
+ self.buffer = ''
+ self.buffer_size = buffer_size
+
+ def more (self):
+ while len(self.buffer) < self.buffer_size:
+ data = self.producer.more()
+ if data:
+ self.buffer = self.buffer + data
+ else:
+ break
+ r = self.buffer
+ self.buffer = ''
+ return r
+
+
+class hooked_producer:
+ """
+ A producer that will call <function> when it empties,.
+ with an argument of the number of bytes produced. Useful
+ for logging/instrumentation purposes.
+ """
+
+ def __init__ (self, producer, function):
+ self.producer = producer
+ self.function = function
+ self.bytes = 0
+
+ def more (self):
+ if self.producer:
+ result = self.producer.more()
+ if not result:
+ self.producer = None
+ self.function (self.bytes)
+ else:
+ self.bytes = self.bytes + len(result)
+ return result
+ else:
+ return ''
+
+# HTTP 1.1 emphasizes that an advertised Content-Length header MUST be
+# correct. In the face of Strange Files, it is conceivable that
+# reading a 'file' may produce an amount of data not matching that
+# reported by os.stat() [text/binary mode issues, perhaps the file is
+# being appended to, etc..] This makes the chunked encoding a True
+# Blessing, and it really ought to be used even with normal files.
+# How beautifully it blends with the concept of the producer.
+
+class chunked_producer:
+ """A producer that implements the 'chunked' transfer coding for HTTP/1.1.
+ Here is a sample usage:
+ request['Transfer-Encoding'] = 'chunked'
+ request.push (
+ producers.chunked_producer (your_producer)
+ )
+ request.done()
+ """
+
+ def __init__ (self, producer, footers=None):
+ self.producer = producer
+ self.footers = footers
+
+ def more (self):
+ if self.producer:
+ data = self.producer.more()
+ if data:
+ return '%x\r\n%s\r\n' % (len(data), data)
+ else:
+ self.producer = None
+ if self.footers:
+ return string.join (
+ ['0'] + self.footers,
+ '\r\n'
+ ) + '\r\n\r\n'
+ else:
+ return '0\r\n\r\n'
+ else:
+ return ''
+
+try:
+ import zlib
+except ImportError:
+ zlib = None
+
+class compressed_producer:
+ """
+ Compress another producer on-the-fly, using ZLIB
+ """
+
+ # Note: It's not very efficient to have the server repeatedly
+ # compressing your outgoing files: compress them ahead of time, or
+ # use a compress-once-and-store scheme. However, if you have low
+ # bandwidth and low traffic, this may make more sense than
+ # maintaining your source files compressed.
+ #
+ # Can also be used for compressing dynamically-produced output.
+
+ def __init__ (self, producer, level=5):
+ self.producer = producer
+ self.compressor = zlib.compressobj (level)
+
+ def more (self):
+ if self.producer:
+ cdata = ''
+ # feed until we get some output
+ while not cdata:
+ data = self.producer.more()
+ if not data:
+ self.producer = None
+ return self.compressor.flush()
+ else:
+ cdata = self.compressor.compress (data)
+ return cdata
+ else:
+ return ''
+
+class escaping_producer:
+
+ "A producer that escapes a sequence of characters"
+ " Common usage: escaping the CRLF.CRLF sequence in SMTP, NNTP, etc..."
+
+ def __init__ (self, producer, esc_from='\r\n.', esc_to='\r\n..'):
+ self.producer = producer
+ self.esc_from = esc_from
+ self.esc_to = esc_to
+ self.buffer = ''
+ self.find_prefix_at_end = find_prefix_at_end
+
+ def more (self):
+ esc_from = self.esc_from
+ esc_to = self.esc_to
+
+ buffer = self.buffer + self.producer.more()
+
+ if buffer:
+ buffer = string.replace (buffer, esc_from, esc_to)
+ i = self.find_prefix_at_end (buffer, esc_from)
+ if i:
+ # we found a prefix
+ self.buffer = buffer[-i:]
+ return buffer[:-i]
+ else:
+ # no prefix, return it all
+ self.buffer = ''
+ return buffer
+ else:
+ return buffer
Added: supervisor/trunk/src/medusa/put_handler.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/put_handler.py Fri May 22 23:51:54 2009
@@ -0,0 +1,115 @@
+# -*- Mode: Python -*-
+#
+# Author: Sam Rushing <rushing at nightmare.com>
+# Copyright 1996-2000 by Sam Rushing
+# All Rights Reserved.
+#
+
+RCS_ID = '$Id: put_handler.py,v 1.4 2002/08/01 18:15:45 akuchling Exp $'
+
+import re
+import string
+
+import default_handler
+unquote = default_handler.unquote
+get_header = default_handler.get_header
+
+last_request = None
+
+class put_handler:
+ def __init__ (self, filesystem, uri_regex):
+ self.filesystem = filesystem
+ if type (uri_regex) == type(''):
+ self.uri_regex = re.compile (uri_regex)
+ else:
+ self.uri_regex = uri_regex
+
+ def match (self, request):
+ uri = request.uri
+ if request.command == 'PUT':
+ m = self.uri_regex.match (uri)
+ if m and m.end() == len(uri):
+ return 1
+ return 0
+
+ def handle_request (self, request):
+
+ path, params, query, fragment = request.split_uri()
+
+ # strip off leading slashes
+ while path and path[0] == '/':
+ path = path[1:]
+
+ if '%' in path:
+ path = unquote (path)
+
+ # make sure there's a content-length header
+ cl = get_header (CONTENT_LENGTH, request.header)
+ if not cl:
+ request.error (411)
+ return
+ else:
+ cl = string.atoi (cl)
+
+ # don't let the try to overwrite a directory
+ if self.filesystem.isdir (path):
+ request.error (405)
+ return
+
+ is_update = self.filesystem.isfile (path)
+
+ try:
+ output_file = self.filesystem.open (path, 'wb')
+ except:
+ request.error (405)
+ return
+
+ request.collector = put_collector (output_file, cl, request, is_update)
+
+ # no terminator while receiving PUT data
+ request.channel.set_terminator (None)
+
+ # don't respond yet, wait until we've received the data...
+
+class put_collector:
+ def __init__ (self, file, length, request, is_update):
+ self.file = file
+ self.length = length
+ self.request = request
+ self.is_update = is_update
+ self.bytes_in = 0
+
+ def collect_incoming_data (self, data):
+ ld = len(data)
+ bi = self.bytes_in
+ if (bi + ld) >= self.length:
+ # last bit of data
+ chunk = self.length - bi
+ self.file.write (data[:chunk])
+ self.file.close()
+
+ if chunk != ld:
+ print 'orphaned %d bytes: <%s>' % (ld - chunk, repr(data[chunk:]))
+
+ # do some housekeeping
+ r = self.request
+ ch = r.channel
+ ch.current_request = None
+ # set the terminator back to the default
+ ch.set_terminator ('\r\n\r\n')
+ if self.is_update:
+ r.reply_code = 204 # No content
+ r.done()
+ else:
+ r.reply_now (201) # Created
+ # avoid circular reference
+ del self.request
+ else:
+ self.file.write (data)
+ self.bytes_in = self.bytes_in + ld
+
+ def found_terminator (self):
+ # shouldn't be called
+ pass
+
+CONTENT_LENGTH = re.compile ('Content-Length: ([0-9]+)', re.IGNORECASE)
Added: supervisor/trunk/src/medusa/redirecting_handler.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/redirecting_handler.py Fri May 22 23:51:54 2009
@@ -0,0 +1,46 @@
+# -*- Mode: Python -*-
+#
+# Author: Sam Rushing <rushing at nightmare.com>
+# Copyright 1996-2000 by Sam Rushing
+# All Rights Reserved.
+#
+
+RCS_ID = '$Id: redirecting_handler.py,v 1.4 2002/03/20 17:37:48 amk Exp $'
+
+import re
+import counter
+
+class redirecting_handler:
+
+ def __init__ (self, pattern, redirect, regex_flag=re.IGNORECASE):
+ self.pattern = pattern
+ self.redirect = redirect
+ self.patreg = re.compile (pattern, regex_flag)
+ self.hits = counter.counter()
+
+ def match (self, request):
+ m = self.patreg.match (request.uri)
+ return (m and (m.end() == len(request.uri)))
+
+ def handle_request (self, request):
+ self.hits.increment()
+ m = self.patreg.match (request.uri)
+ part = m.group(1)
+
+ request['Location'] = self.redirect % part
+ request.error (302) # moved temporarily
+
+ def __repr__ (self):
+ return '<Redirecting Handler at %08x [%s => %s]>' % (
+ id(self),
+ repr(self.pattern),
+ repr(self.redirect)
+ )
+
+ def status (self):
+ import producers
+ return producers.simple_producer (
+ '<li> Redirecting Handler %s => %s <b>Hits</b>: %s' % (
+ self.pattern, self.redirect, self.hits
+ )
+ )
Added: supervisor/trunk/src/medusa/resolver.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/resolver.py Fri May 22 23:51:54 2009
@@ -0,0 +1,440 @@
+# -*- Mode: Python -*-
+
+#
+# Author: Sam Rushing <rushing at nightmare.com>
+#
+
+RCS_ID = '$Id: resolver.py,v 1.4 2002/03/20 17:37:48 amk Exp $'
+
+
+# Fast, low-overhead asynchronous name resolver. uses 'pre-cooked'
+# DNS requests, unpacks only as much as it needs of the reply.
+
+# see rfc1035 for details
+
+import string
+import asyncore_25 as asyncore
+import socket
+import sys
+import time
+from counter import counter
+
+VERSION = string.split(RCS_ID)[2]
+
+# header
+# 1 1 1 1 1 1
+# 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5
+# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+# | ID |
+# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+# |QR| Opcode |AA|TC|RD|RA| Z | RCODE |
+# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+# | QDCOUNT |
+# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+# | ANCOUNT |
+# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+# | NSCOUNT |
+# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+# | ARCOUNT |
+# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+
+
+# question
+# 1 1 1 1 1 1
+# 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5
+# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+# | |
+# / QNAME /
+# / /
+# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+# | QTYPE |
+# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+# | QCLASS |
+# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+
+# build a DNS address request, _quickly_
+def fast_address_request (host, id=0):
+ return (
+ '%c%c' % (chr((id>>8)&0xff),chr(id&0xff))
+ + '\001\000\000\001\000\000\000\000\000\000%s\000\000\001\000\001' % (
+ string.join (
+ map (
+ lambda part: '%c%s' % (chr(len(part)),part),
+ string.split (host, '.')
+ ), ''
+ )
+ )
+ )
+
+def fast_ptr_request (host, id=0):
+ return (
+ '%c%c' % (chr((id>>8)&0xff),chr(id&0xff))
+ + '\001\000\000\001\000\000\000\000\000\000%s\000\000\014\000\001' % (
+ string.join (
+ map (
+ lambda part: '%c%s' % (chr(len(part)),part),
+ string.split (host, '.')
+ ), ''
+ )
+ )
+ )
+
+def unpack_name (r,pos):
+ n = []
+ while 1:
+ ll = ord(r[pos])
+ if (ll&0xc0):
+ # compression
+ pos = (ll&0x3f << 8) + (ord(r[pos+1]))
+ elif ll == 0:
+ break
+ else:
+ pos = pos + 1
+ n.append (r[pos:pos+ll])
+ pos = pos + ll
+ return string.join (n,'.')
+
+def skip_name (r,pos):
+ s = pos
+ while 1:
+ ll = ord(r[pos])
+ if (ll&0xc0):
+ # compression
+ return pos + 2
+ elif ll == 0:
+ pos = pos + 1
+ break
+ else:
+ pos = pos + ll + 1
+ return pos
+
+def unpack_ttl (r,pos):
+ return reduce (
+ lambda x,y: (x<<8)|y,
+ map (ord, r[pos:pos+4])
+ )
+
+# resource record
+# 1 1 1 1 1 1
+# 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5
+# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+# | |
+# / /
+# / NAME /
+# | |
+# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+# | TYPE |
+# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+# | CLASS |
+# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+# | TTL |
+# | |
+# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+# | RDLENGTH |
+# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--|
+# / RDATA /
+# / /
+# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
+
+def unpack_address_reply (r):
+ ancount = (ord(r[6])<<8) + (ord(r[7]))
+ # skip question, first name starts at 12,
+ # this is followed by QTYPE and QCLASS
+ pos = skip_name (r, 12) + 4
+ if ancount:
+ # we are looking very specifically for
+ # an answer with TYPE=A, CLASS=IN (\000\001\000\001)
+ for an in range(ancount):
+ pos = skip_name (r, pos)
+ if r[pos:pos+4] == '\000\001\000\001':
+ return (
+ unpack_ttl (r,pos+4),
+ '%d.%d.%d.%d' % tuple(map(ord,r[pos+10:pos+14]))
+ )
+ # skip over TYPE, CLASS, TTL, RDLENGTH, RDATA
+ pos = pos + 8
+ rdlength = (ord(r[pos])<<8) + (ord(r[pos+1]))
+ pos = pos + 2 + rdlength
+ return 0, None
+ else:
+ return 0, None
+
+def unpack_ptr_reply (r):
+ ancount = (ord(r[6])<<8) + (ord(r[7]))
+ # skip question, first name starts at 12,
+ # this is followed by QTYPE and QCLASS
+ pos = skip_name (r, 12) + 4
+ if ancount:
+ # we are looking very specifically for
+ # an answer with TYPE=PTR, CLASS=IN (\000\014\000\001)
+ for an in range(ancount):
+ pos = skip_name (r, pos)
+ if r[pos:pos+4] == '\000\014\000\001':
+ return (
+ unpack_ttl (r,pos+4),
+ unpack_name (r, pos+10)
+ )
+ # skip over TYPE, CLASS, TTL, RDLENGTH, RDATA
+ pos = pos + 8
+ rdlength = (ord(r[pos])<<8) + (ord(r[pos+1]))
+ pos = pos + 2 + rdlength
+ return 0, None
+ else:
+ return 0, None
+
+
+# This is a UDP (datagram) resolver.
+
+#
+# It may be useful to implement a TCP resolver. This would presumably
+# give us more reliable behavior when things get too busy. A TCP
+# client would have to manage the connection carefully, since the
+# server is allowed to close it at will (the RFC recommends closing
+# after 2 minutes of idle time).
+#
+# Note also that the TCP client will have to prepend each request
+# with a 2-byte length indicator (see rfc1035).
+#
+
+class resolver (asyncore.dispatcher):
+ id = counter()
+ def __init__ (self, server='127.0.0.1'):
+ asyncore.dispatcher.__init__ (self)
+ self.create_socket (socket.AF_INET, socket.SOCK_DGRAM)
+ self.server = server
+ self.request_map = {}
+ self.last_reap_time = int(time.time()) # reap every few minutes
+
+ def writable (self):
+ return 0
+
+ def log (self, *args):
+ pass
+
+ def handle_close (self):
+ self.log_info('closing!')
+ self.close()
+
+ def handle_error (self): # don't close the connection on error
+ (file,fun,line), t, v, tbinfo = asyncore.compact_traceback()
+ self.log_info(
+ 'Problem with DNS lookup (%s:%s %s)' % (t, v, tbinfo),
+ 'error')
+
+ def get_id (self):
+ return (self.id.as_long() % (1<<16))
+
+ def reap (self): # find DNS requests that have timed out
+ now = int(time.time())
+ if now - self.last_reap_time > 180: # reap every 3 minutes
+ self.last_reap_time = now # update before we forget
+ for k,(host,unpack,callback,when) in self.request_map.items():
+ if now - when > 180: # over 3 minutes old
+ del self.request_map[k]
+ try: # same code as in handle_read
+ callback (host, 0, None) # timeout val is (0,None)
+ except:
+ (file,fun,line), t, v, tbinfo = asyncore.compact_traceback()
+ self.log_info('%s %s %s' % (t,v,tbinfo), 'error')
+
+ def resolve (self, host, callback):
+ self.reap() # first, get rid of old guys
+ self.socket.sendto (
+ fast_address_request (host, self.get_id()),
+ (self.server, 53)
+ )
+ self.request_map [self.get_id()] = (
+ host, unpack_address_reply, callback, int(time.time()))
+ self.id.increment()
+
+ def resolve_ptr (self, host, callback):
+ self.reap() # first, get rid of old guys
+ ip = string.split (host, '.')
+ ip.reverse()
+ ip = string.join (ip, '.') + '.in-addr.arpa'
+ self.socket.sendto (
+ fast_ptr_request (ip, self.get_id()),
+ (self.server, 53)
+ )
+ self.request_map [self.get_id()] = (
+ host, unpack_ptr_reply, callback, int(time.time()))
+ self.id.increment()
+
+ def handle_read (self):
+ reply, whence = self.socket.recvfrom (512)
+ # for security reasons we may want to double-check
+ # that <whence> is the server we sent the request to.
+ id = (ord(reply[0])<<8) + ord(reply[1])
+ if self.request_map.has_key (id):
+ host, unpack, callback, when = self.request_map[id]
+ del self.request_map[id]
+ ttl, answer = unpack (reply)
+ try:
+ callback (host, ttl, answer)
+ except:
+ (file,fun,line), t, v, tbinfo = asyncore.compact_traceback()
+ self.log_info('%s %s %s' % ( t,v,tbinfo), 'error')
+
+class rbl (resolver):
+
+ def resolve_maps (self, host, callback):
+ ip = string.split (host, '.')
+ ip.reverse()
+ ip = string.join (ip, '.') + '.rbl.maps.vix.com'
+ self.socket.sendto (
+ fast_ptr_request (ip, self.get_id()),
+ (self.server, 53)
+ )
+ self.request_map [self.get_id()] = host, self.check_reply, callback
+ self.id.increment()
+
+ def check_reply (self, r):
+ # we only need to check RCODE.
+ rcode = (ord(r[3])&0xf)
+ self.log_info('MAPS RBL; RCODE =%02x\n %s' % (rcode, repr(r)))
+ return 0, rcode # (ttl, answer)
+
+
+class hooked_callback:
+ def __init__ (self, hook, callback):
+ self.hook, self.callback = hook, callback
+
+ def __call__ (self, *args):
+ apply (self.hook, args)
+ apply (self.callback, args)
+
+class caching_resolver (resolver):
+ "Cache DNS queries. Will need to honor the TTL value in the replies"
+
+ def __init__ (*args):
+ apply (resolver.__init__, args)
+ self = args[0]
+ self.cache = {}
+ self.forward_requests = counter()
+ self.reverse_requests = counter()
+ self.cache_hits = counter()
+
+ def resolve (self, host, callback):
+ self.forward_requests.increment()
+ if self.cache.has_key (host):
+ when, ttl, answer = self.cache[host]
+ # ignore TTL for now
+ callback (host, ttl, answer)
+ self.cache_hits.increment()
+ else:
+ resolver.resolve (
+ self,
+ host,
+ hooked_callback (
+ self.callback_hook,
+ callback
+ )
+ )
+
+ def resolve_ptr (self, host, callback):
+ self.reverse_requests.increment()
+ if self.cache.has_key (host):
+ when, ttl, answer = self.cache[host]
+ # ignore TTL for now
+ callback (host, ttl, answer)
+ self.cache_hits.increment()
+ else:
+ resolver.resolve_ptr (
+ self,
+ host,
+ hooked_callback (
+ self.callback_hook,
+ callback
+ )
+ )
+
+ def callback_hook (self, host, ttl, answer):
+ self.cache[host] = time.time(), ttl, answer
+
+ SERVER_IDENT = 'Caching DNS Resolver (V%s)' % VERSION
+
+ def status (self):
+ import producers
+ return producers.simple_producer (
+ '<h2>%s</h2>' % self.SERVER_IDENT
+ + '<br>Server: %s' % self.server
+ + '<br>Cache Entries: %d' % len(self.cache)
+ + '<br>Outstanding Requests: %d' % len(self.request_map)
+ + '<br>Forward Requests: %s' % self.forward_requests
+ + '<br>Reverse Requests: %s' % self.reverse_requests
+ + '<br>Cache Hits: %s' % self.cache_hits
+ )
+
+#test_reply = """\000\000\205\200\000\001\000\001\000\002\000\002\006squirl\011nightmare\003com\000\000\001\000\001\300\014\000\001\000\001\000\001Q\200\000\004\315\240\260\005\011nightmare\003com\000\000\002\000\001\000\001Q\200\000\002\300\014\3006\000\002\000\001\000\001Q\200\000\015\003ns1\003iag\003net\000\300\014\000\001\000\001\000\001Q\200\000\004\315\240\260\005\300]\000\001\000\001\000\000\350\227\000\004\314\033\322\005"""
+# def test_unpacker ():
+# print unpack_address_reply (test_reply)
+#
+# import time
+# class timer:
+# def __init__ (self):
+# self.start = time.time()
+# def end (self):
+# return time.time() - self.start
+#
+# # I get ~290 unpacks per second for the typical case, compared to ~48
+# # using dnslib directly. also, that latter number does not include
+# # picking the actual data out.
+#
+# def benchmark_unpacker():
+#
+# r = range(1000)
+# t = timer()
+# for i in r:
+# unpack_address_reply (test_reply)
+# print '%.2f unpacks per second' % (1000.0 / t.end())
+
+if __name__ == '__main__':
+ if len(sys.argv) == 1:
+ print 'usage: %s [-r] [-s <server_IP>] host [host ...]' % sys.argv[0]
+ sys.exit(0)
+ elif ('-s' in sys.argv):
+ i = sys.argv.index('-s')
+ server = sys.argv[i+1]
+ del sys.argv[i:i+2]
+ else:
+ server = '127.0.0.1'
+
+ if ('-r' in sys.argv):
+ reverse = 1
+ i = sys.argv.index('-r')
+ del sys.argv[i]
+ else:
+ reverse = 0
+
+ if ('-m' in sys.argv):
+ maps = 1
+ sys.argv.remove ('-m')
+ else:
+ maps = 0
+
+ if maps:
+ r = rbl (server)
+ else:
+ r = caching_resolver(server)
+
+ count = len(sys.argv) - 1
+
+ def print_it (host, ttl, answer):
+ global count
+ print '%s: %s' % (host, answer)
+ count = count - 1
+ if not count:
+ r.close()
+
+ for host in sys.argv[1:]:
+ if reverse:
+ r.resolve_ptr (host, print_it)
+ elif maps:
+ r.resolve_maps (host, print_it)
+ else:
+ r.resolve (host, print_it)
+
+ # hooked asyncore.loop()
+ while asyncore.socket_map:
+ asyncore.poll (30.0)
+ print 'requests outstanding: %d' % len(r.request_map)
Added: supervisor/trunk/src/medusa/rpc_client.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/rpc_client.py Fri May 22 23:51:54 2009
@@ -0,0 +1,312 @@
+# -*- Mode: Python -*-
+
+# Copyright 1999, 2000 by eGroups, Inc.
+#
+# All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of
+# eGroups not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# EGROUPS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+# NO EVENT SHALL EGROUPS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+import marshal
+import socket
+import string
+import exceptions
+import string
+import sys
+
+#
+# there are three clients in here.
+#
+# 1) rpc client
+# 2) fastrpc client
+# 3) async fastrpc client
+#
+# we hope that *whichever* choice you make, that you will enjoy the
+# excellent hand-made construction, and return to do business with us
+# again in the near future.
+#
+
+class RPC_Error (exceptions.StandardError):
+ pass
+
+# ===========================================================================
+# RPC Client
+# ===========================================================================
+
+# request types:
+# 0 call
+# 1 getattr
+# 2 setattr
+# 3 repr
+# 4 del
+
+
+class rpc_proxy:
+
+ DEBUG = 0
+
+ def __init__ (self, conn, oid):
+ # route around __setattr__
+ self.__dict__['conn'] = conn
+ self.__dict__['oid'] = oid
+
+ # Warning: be VERY CAREFUL with attribute references, keep
+ # this __getattr__ in mind!
+
+ def __getattr__ (self, attr):
+ # __getattr__ and __call__
+ if attr == '__call__':
+ # 0 == __call__
+ return self.__remote_call__
+ elif attr == '__repr__':
+ # 3 == __repr__
+ return self.__remote_repr__
+ elif attr == '__getitem__':
+ return self.__remote_getitem__
+ elif attr == '__setitem__':
+ return self.__remote_setitem__
+ elif attr == '__len__':
+ return self.__remote_len__
+ else:
+ # 1 == __getattr__
+ return self.__send_request__ (1, attr)
+
+ def __setattr__ (self, attr, value):
+ return self.__send_request__ (2, (attr, value))
+
+ def __del__ (self):
+ try:
+ self.__send_request__ (4, None)
+ except:
+ import who_calls
+ info = who_calls.compact_traceback()
+ print info
+
+ def __remote_repr__ (self):
+ r = self.__send_request__ (3, None)
+ return '<remote object [%s]>' % r[1:-1]
+
+ def __remote_call__ (self, *args):
+ return self.__send_request__ (0, args)
+
+ def __remote_getitem__ (self, key):
+ return self.__send_request__ (5, key)
+
+ def __remote_setitem__ (self, key, value):
+ return self.__send_request__ (6, (key, value))
+
+ def __remote_len__ (self):
+ return self.__send_request__ (7, None)
+
+ _request_types_ = ['call', 'getattr', 'setattr', 'repr', 'del', 'getitem', 'setitem', 'len']
+
+ def __send_request__ (self, *args):
+ if self.DEBUG:
+ kind = args[0]
+ print (
+ 'RPC: ==> %s:%08x:%s:%s' % (
+ self.conn.address,
+ self.oid,
+ self._request_types_[kind],
+ repr(args[1:])
+ )
+ )
+ packet = marshal.dumps ((self.oid,)+args)
+ # send request
+ self.conn.send_packet (packet)
+ # get response
+ data = self.conn.receive_packet()
+ # types of response:
+ # 0: proxy
+ # 1: error
+ # 2: marshal'd data
+
+ kind, value = marshal.loads (data)
+
+ if kind == 0:
+ # proxy (value == oid)
+ if self.DEBUG:
+ print 'RPC: <== proxy(%08x)' % (value)
+ return rpc_proxy (self.conn, value)
+ elif kind == 1:
+ raise RPC_Error, value
+ else:
+ if self.DEBUG:
+ print 'RPC: <== %s' % (repr(value))
+ return value
+
+class rpc_connection:
+
+ cache = {}
+
+ def __init__ (self, address):
+ self.address = address
+ self.connect ()
+
+ def connect (self):
+ s = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
+ s.connect (self.address)
+ self.socket = s
+
+ def receive_packet (self):
+ packet_len = string.atoi (self.socket.recv (8), 16)
+ packet = []
+ while packet_len:
+ data = self.socket.recv (8192)
+ packet.append (data)
+ packet_len = packet_len - len(data)
+ return string.join (packet, '')
+
+ def send_packet (self, packet):
+ self.socket.send ('%08x%s' % (len(packet), packet))
+
+def rpc_connect (address = ('localhost', 8746)):
+ if not rpc_connection.cache.has_key (address):
+ conn = rpc_connection (address)
+ # get oid of remote object
+ data = conn.receive_packet()
+ (oid,) = marshal.loads (data)
+ rpc_connection.cache[address] = rpc_proxy (conn, oid)
+ return rpc_connection.cache[address]
+
+# ===========================================================================
+# fastrpc client
+# ===========================================================================
+
+class fastrpc_proxy:
+
+ def __init__ (self, conn, path=()):
+ self.conn = conn
+ self.path = path
+
+ def __getattr__ (self, attr):
+ if attr == '__call__':
+ return self.__method_caller__
+ else:
+ return fastrpc_proxy (self.conn, self.path + (attr,))
+
+ def __method_caller__ (self, *args):
+ # send request
+ packet = marshal.dumps ((self.path, args))
+ self.conn.send_packet (packet)
+ # get response
+ data = self.conn.receive_packet()
+ error, result = marshal.loads (data)
+ if error is None:
+ return result
+ else:
+ raise RPC_Error, error
+
+ def __repr__ (self):
+ return '<remote-method-%s at %x>' % (string.join (self.path, '.'), id (self))
+
+def fastrpc_connect (address = ('localhost', 8748)):
+ if not rpc_connection.cache.has_key (address):
+ conn = rpc_connection (address)
+ rpc_connection.cache[address] = fastrpc_proxy (conn)
+ return rpc_connection.cache[address]
+
+# ===========================================================================
+# async fastrpc client
+# ===========================================================================
+
+import asynchat_25 as asynchat
+
+class async_fastrpc_client (asynchat.async_chat):
+
+ STATE_LENGTH = 'length state'
+ STATE_PACKET = 'packet state'
+
+ def __init__ (self, address=('idb', 3001)):
+
+ asynchat.async_chat.__init__ (self)
+
+ if type(address) is type(''):
+ family = socket.AF_UNIX
+ else:
+ family = socket.AF_INET
+
+ self.create_socket (family, socket.SOCK_STREAM)
+ self.address = address
+ self.request_fifo = []
+ self.buffer = []
+ self.pstate = self.STATE_LENGTH
+ self.set_terminator (8)
+ self._connected = 0
+ self.connect (self.address)
+
+ def log (self, *args):
+ pass
+
+ def handle_connect (self):
+ self._connected = 1
+
+ def close (self):
+ self._connected = 0
+ self.flush_pending_requests ('lost connection to rpc server')
+ asynchat.async_chat.close(self)
+
+ def flush_pending_requests (self, why):
+ f = self.request_fifo
+ while len(f):
+ callback = f.pop(0)
+ callback (why, None)
+
+ def collect_incoming_data (self, data):
+ self.buffer.append (data)
+
+ def found_terminator (self):
+ self.buffer, data = [], string.join (self.buffer, '')
+
+ if self.pstate is self.STATE_LENGTH:
+ packet_length = string.atoi (data, 16)
+ self.set_terminator (packet_length)
+ self.pstate = self.STATE_PACKET
+ else:
+ # modified to fix socket leak in chat server, 2000-01-27, schiller at eGroups.net
+ #self.set_terminator (8)
+ #self.pstate = self.STATE_LENGTH
+ error, result = marshal.loads (data)
+ callback = self.request_fifo.pop(0)
+ callback (error, result)
+ self.close() # for chat server
+
+ def call_method (self, method, args, callback):
+ if not self._connected:
+ # might be a unix socket...
+ family, type = self.family_and_type
+ self.create_socket (family, type)
+ self.connect (self.address)
+ # push the request out the socket
+ path = string.split (method, '.')
+ packet = marshal.dumps ((path, args))
+ self.push ('%08x%s' % (len(packet), packet))
+ self.request_fifo.append(callback)
+
+
+if __name__ == '__main__':
+ if '-f' in sys.argv:
+ connect = fastrpc_connect
+ else:
+ connect = rpc_connect
+
+ print 'connecting...'
+ c = connect()
+ print 'calling <remote>.calc.sum (1,2,3)'
+ print c.calc.sum (1,2,3)
+ print 'calling <remote>.calc.nonexistent(), expect an exception!'
+ print c.calc.nonexistent()
Added: supervisor/trunk/src/medusa/rpc_server.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/rpc_server.py Fri May 22 23:51:54 2009
@@ -0,0 +1,322 @@
+# -*- Mode: Python -*-
+
+# Copyright 1999, 2000 by eGroups, Inc.
+#
+# All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of
+# eGroups not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# EGROUPS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+# NO EVENT SHALL EGROUPS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+# There are two RPC implementations here.
+
+# The first ('rpc') attempts to be as transparent as possible, and
+# passes along 'internal' methods like __getattr__, __getitem__, and
+# __del__. It is rather 'chatty', and may not be suitable for a
+# high-performance system.
+
+# The second ('fastrpc') is less flexible, but has much less overhead,
+# and is easier to use from an asynchronous client.
+
+import marshal
+import socket
+import string
+import sys
+import types
+
+import asyncore_25 as asyncore
+import asynchat_25 as asynchat
+
+from producers import scanning_producer
+from counter import counter
+
+MY_NAME = string.split (socket.gethostname(), '.')[0]
+
+# ===========================================================================
+# RPC server
+# ===========================================================================
+
+# marshal is good for low-level data structures.
+# but when passing an 'object' (any non-marshallable object)
+# we really want to pass a 'reference', which will act on
+# the other side as a proxy. How transparent can we make this?
+
+class rpc_channel (asynchat.async_chat):
+
+ 'Simple RPC server.'
+
+ # a 'packet': NNNNNNNNmmmmmmmmmmmmmmmm
+ # (hex length in 8 bytes, followed by marshal'd packet data)
+ # same protocol used in both directions.
+
+ STATE_LENGTH = 'length state'
+ STATE_PACKET = 'packet state'
+
+ ac_out_buffer_size = 65536
+
+ request_counter = counter()
+ exception_counter = counter()
+ client_counter = counter()
+
+ def __init__ (self, root, conn, addr):
+ self.root = root
+ self.addr = addr
+ asynchat.async_chat.__init__ (self, conn)
+ self.pstate = self.STATE_LENGTH
+ self.set_terminator (8)
+ self.buffer = []
+ self.proxies = {}
+ rid = id(root)
+ self.new_reference (root)
+ p = marshal.dumps ((rid,))
+ # send root oid to the other side
+ self.push ('%08x%s' % (len(p), p))
+ self.client_counter.increment()
+
+ def new_reference (self, object):
+ oid = id(object)
+ ignore, refcnt = self.proxies.get (oid, (None, 0))
+ self.proxies[oid] = (object, refcnt + 1)
+
+ def forget_reference (self, oid):
+ object, refcnt = self.proxies.get (oid, (None, 0))
+ if refcnt > 1:
+ self.proxies[oid] = (object, refcnt - 1)
+ else:
+ del self.proxies[oid]
+
+ def log (self, *ignore):
+ pass
+
+ def collect_incoming_data (self, data):
+ self.buffer.append (data)
+
+ def found_terminator (self):
+ self.buffer, data = [], string.join (self.buffer, '')
+
+ if self.pstate is self.STATE_LENGTH:
+ packet_length = string.atoi (data, 16)
+ self.set_terminator (packet_length)
+ self.pstate = self.STATE_PACKET
+ else:
+
+ self.set_terminator (8)
+ self.pstate = self.STATE_LENGTH
+
+ oid, kind, arg = marshal.loads (data)
+
+ obj, refcnt = self.proxies[oid]
+ e = None
+ reply_kind = 2
+
+ try:
+ if kind == 0:
+ # __call__
+ result = apply (obj, arg)
+ elif kind == 1:
+ # __getattr__
+ result = getattr (obj, arg)
+ elif kind == 2:
+ # __setattr__
+ key, value = arg
+ setattr (obj, key, value)
+ result = None
+ elif kind == 3:
+ # __repr__
+ result = repr(obj)
+ elif kind == 4:
+ # __del__
+ self.forget_reference (oid)
+ result = None
+ elif kind == 5:
+ # __getitem__
+ result = obj[arg]
+ elif kind == 6:
+ # __setitem__
+ (key, value) = arg
+ obj[key] = value
+ result = None
+ elif kind == 7:
+ # __len__
+ result = len(obj)
+
+ except:
+ reply_kind = 1
+ (file,fun,line), t, v, tbinfo = asyncore.compact_traceback()
+ result = '%s:%s:%s:%s (%s:%s)' % (MY_NAME, file, fun, line, t, str(v))
+ self.log_info (result, 'error')
+ self.exception_counter.increment()
+
+ self.request_counter.increment()
+
+ # optimize a common case
+ if type(result) is types.InstanceType:
+ can_marshal = 0
+ else:
+ can_marshal = 1
+
+ try:
+ rb = marshal.dumps ((reply_kind, result))
+ except ValueError:
+ can_marshal = 0
+
+ if not can_marshal:
+ # unmarshallable object, return a reference
+ rid = id(result)
+ self.new_reference (result)
+ rb = marshal.dumps ((0, rid))
+
+ self.push_with_producer (
+ scanning_producer (
+ ('%08x' % len(rb)) + rb,
+ buffer_size = 65536
+ )
+ )
+
+class rpc_server_root:
+ pass
+
+class rpc_server (asyncore.dispatcher):
+
+ def __init__ (self, root, address = ('', 8746)):
+ self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
+ self.set_reuse_addr()
+ self.bind (address)
+ self.listen (128)
+ self.root = root
+
+ def handle_accept (self):
+ conn, addr = self.accept()
+ rpc_channel (self.root, conn, addr)
+
+
+# ===========================================================================
+# Fast RPC server
+# ===========================================================================
+
+# no proxies, request consists
+# of a 'chain' of getattrs terminated by a __call__.
+
+# Protocol:
+# <path>.<to>.<object> ( <param1>, <param2>, ... )
+# => ( <value1>, <value2>, ... )
+#
+#
+# (<path>, <params>)
+# path: tuple of strings
+# params: tuple of objects
+
+class fastrpc_channel (asynchat.async_chat):
+
+ 'Simple RPC server'
+
+ # a 'packet': NNNNNNNNmmmmmmmmmmmmmmmm
+ # (hex length in 8 bytes, followed by marshal'd packet data)
+ # same protocol used in both directions.
+
+ # A request consists of (<path-tuple>, <args-tuple>)
+ # where <path-tuple> is a list of strings (eqv to string.split ('a.b.c', '.'))
+
+ STATE_LENGTH = 'length state'
+ STATE_PACKET = 'packet state'
+
+ def __init__ (self, root, conn, addr):
+ self.root = root
+ self.addr = addr
+ asynchat.async_chat.__init__ (self, conn)
+ self.pstate = self.STATE_LENGTH
+ self.set_terminator (8)
+ self.buffer = []
+
+ def log (*ignore):
+ pass
+
+ def collect_incoming_data (self, data):
+ self.buffer.append (data)
+
+ def found_terminator (self):
+ self.buffer, data = [], string.join (self.buffer, '')
+
+ if self.pstate is self.STATE_LENGTH:
+ packet_length = string.atoi (data, 16)
+ self.set_terminator (packet_length)
+ self.pstate = self.STATE_PACKET
+ else:
+ self.set_terminator (8)
+ self.pstate = self.STATE_LENGTH
+ (path, params) = marshal.loads (data)
+ o = self.root
+
+ e = None
+
+ try:
+ for p in path:
+ o = getattr (o, p)
+ result = apply (o, params)
+ except:
+ e = repr (asyncore.compact_traceback())
+ result = None
+
+ rb = marshal.dumps ((e,result))
+ self.push (('%08x' % len(rb)) + rb)
+
+class fastrpc_server (asyncore.dispatcher):
+
+ def __init__ (self, root, address = ('', 8748)):
+ self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
+ self.set_reuse_addr()
+ self.bind (address)
+ self.listen (128)
+ self.root = root
+
+ def handle_accept (self):
+ conn, addr = self.accept()
+ fastrpc_channel (self.root, conn, addr)
+
+# ===========================================================================
+
+if __name__ == '__main__':
+
+ class thing:
+ def __del__ (self):
+ print 'a thing has gone away %08x' % id(self)
+
+ class sample_calc:
+
+ def product (self, *values):
+ return reduce (lambda a,b: a*b, values, 1)
+
+ def sum (self, *values):
+ return reduce (lambda a,b: a+b, values, 0)
+
+ def eval (self, string):
+ return eval (string)
+
+ def make_a_thing (self):
+ return thing()
+
+ if '-f' in sys.argv:
+ server_class = fastrpc_server
+ address = ('', 8748)
+ else:
+ server_class = rpc_server
+ address = ('', 8746)
+
+ root = rpc_server_root()
+ root.calc = sample_calc()
+ root.sys = sys
+ rs = server_class (root, address)
+ asyncore.loop()
Added: supervisor/trunk/src/medusa/script_handler.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/script_handler.py Fri May 22 23:51:54 2009
@@ -0,0 +1,215 @@
+# -*- Mode: Python -*-
+
+# This is a simple python server-side script handler.
+
+# A note about performance: This is really only suited for 'fast'
+# scripts: The script should generate its output quickly, since the
+# whole web server will stall otherwise. This doesn't mean you have
+# to write 'fast code' or anything, it simply means that you shouldn't
+# call any long-running code, [like say something that opens up an
+# internet connection, or a database query that will hold up the
+# server]. If you need this sort of feature, you can support it using
+# the asynchronous I/O 'api' that the rest of medusa is built on. [or
+# you could probably use threads]
+
+# Put your script into your web docs directory (like a cgi-bin
+# script), make sure it has the correct extension [see the overridable
+# script_handler.extension member below].
+#
+# There's lots of things that can be done to tweak the restricted
+# execution model. Also, of course you could just use 'execfile'
+# instead (this is now the default, see class variable
+# script_handler.restricted)
+
+import rexec
+import re
+import string
+import StringIO
+import sys
+
+import counter
+import default_handler
+import producers
+
+unquote = default_handler.unquote
+
+class script_handler:
+
+ extension = 'mpy'
+ restricted = 0
+
+ script_regex = re.compile (
+ r'.*/([^/]+\.%s)' % extension,
+ re.IGNORECASE
+ )
+
+ def __init__ (self, filesystem):
+ self.filesystem = filesystem
+ self.hits = counter.counter()
+ self.exceptions = counter.counter()
+
+ def match (self, request):
+ [path, params, query, fragment] = request.split_uri()
+ m = self.script_regex.match (path)
+ return (m and (m.end() == len(path)))
+
+ def handle_request (self, request):
+
+ [path, params, query, fragment] = request.split_uri()
+
+ while path and path[0] == '/':
+ path = path[1:]
+
+ if '%' in path:
+ path = unquote (path)
+
+ if not self.filesystem.isfile (path):
+ request.error (404)
+ return
+ else:
+
+ self.hits.increment()
+
+ request.script_filename = self.filesystem.translate (path)
+
+ if request.command in ('PUT', 'POST'):
+ # look for a Content-Length header.
+ cl = request.get_header ('content-length')
+ length = int(cl)
+ if not cl:
+ request.error (411)
+ else:
+ collector (self, length, request)
+ else:
+ self.continue_request (
+ request,
+ StringIO.StringIO() # empty stdin
+ )
+
+ def continue_request (self, request, stdin):
+ temp_files = stdin, StringIO.StringIO(), StringIO.StringIO()
+ old_files = sys.stdin, sys.stdout, sys.stderr
+
+ if self.restricted:
+ r = rexec.RExec()
+
+ try:
+ sys.request = request
+ sys.stdin, sys.stdout, sys.stderr = temp_files
+ try:
+ if self.restricted:
+ r.s_execfile (request.script_filename)
+ else:
+ execfile (request.script_filename)
+ request.reply_code = 200
+ except:
+ request.reply_code = 500
+ self.exceptions.increment()
+ finally:
+ sys.stdin, sys.stdout, sys.stderr = old_files
+ del sys.request
+
+ i,o,e = temp_files
+
+ if request.reply_code != 200:
+ s = e.getvalue()
+ else:
+ s = o.getvalue()
+
+ request['Content-Length'] = len(s)
+ request.push (s)
+ request.done()
+
+ def status (self):
+ return producers.simple_producer (
+ '<li>Server-Side Script Handler'
+ + '<ul>'
+ + ' <li><b>Hits:</b> %s' % self.hits
+ + ' <li><b>Exceptions:</b> %s' % self.exceptions
+ + '</ul>'
+ )
+
+
+class persistent_script_handler:
+
+ def __init__ (self):
+ self.modules = {}
+ self.hits = counter.counter()
+ self.exceptions = counter.counter()
+
+ def add_module (self, name, module):
+ self.modules[name] = module
+
+ def del_module (self, name):
+ del self.modules[name]
+
+ def match (self, request):
+ [path, params, query, fragment] = request.split_uri()
+ parts = string.split (path, '/')
+ if (len(parts)>1) and self.modules.has_key (parts[1]):
+ module = self.modules[parts[1]]
+ request.module = module
+ return 1
+ else:
+ return 0
+
+ def handle_request (self, request):
+ if request.command in ('PUT', 'POST'):
+ # look for a Content-Length header.
+ cl = request.get_header ('content-length')
+ length = int(cl)
+ if not cl:
+ request.error (411)
+ else:
+ collector (self, length, request)
+ else:
+ self.continue_request (request, StringIO.StringIO())
+
+ def continue_request (self, request, input_data):
+ temp_files = input_data, StringIO.StringIO(), StringIO.StringIO()
+ old_files = sys.stdin, sys.stdout, sys.stderr
+
+ try:
+ sys.stdin, sys.stdout, sys.stderr = temp_files
+ # provide a default
+ request['Content-Type'] = 'text/html'
+ try:
+ request.module.main (request)
+ request.reply_code = 200
+ except:
+ request.reply_code = 500
+ self.exceptions.increment()
+ finally:
+ sys.stdin, sys.stdout, sys.stderr = old_files
+
+ i,o,e = temp_files
+
+ if request.reply_code != 200:
+ s = e.getvalue()
+ else:
+ s = o.getvalue()
+
+ request['Content-Length'] = len(s)
+ request.push (s)
+ request.done()
+
+class collector:
+
+ def __init__ (self, handler, length, request):
+ self.handler = handler
+ self.request = request
+ self.request.collector = self
+ self.request.channel.set_terminator (length)
+ self.buffer = StringIO.StringIO()
+
+ def collect_incoming_data (self, data):
+ self.buffer.write (data)
+
+ def found_terminator (self):
+ self.buffer.seek(0)
+ self.request.collector = None
+ self.request.channel.set_terminator ('\r\n\r\n')
+ self.handler.continue_request (
+ self.request,
+ self.buffer
+ )
Added: supervisor/trunk/src/medusa/setup.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/setup.py Fri May 22 23:51:54 2009
@@ -0,0 +1,18 @@
+
+__revision__ = '$Id: setup.py,v 1.9 2003/08/22 13:07:07 akuchling Exp $'
+
+from distutils.core import setup
+
+setup(
+ name = 'medusa',
+ version = "0.5.4",
+ description = "A framework for implementing asynchronous servers.",
+ author = "Sam Rushing",
+ author_email = "rushing at nightmare.com",
+ maintainer = "A.M. Kuchling",
+ maintainer_email = "amk at amk.ca",
+ url = "http://oedipus.sourceforge.net/medusa/",
+
+ packages = ['medusa'],
+ package_dir = {'medusa':'.'},
+ )
Added: supervisor/trunk/src/medusa/status_handler.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/status_handler.py Fri May 22 23:51:54 2009
@@ -0,0 +1,278 @@
+# -*- Mode: Python -*-
+
+VERSION_STRING = "$Id: status_handler.py,v 1.7 2003/12/24 16:08:16 akuchling Exp $"
+
+#
+# medusa status extension
+#
+
+import string
+import time
+import re
+from cgi import escape
+
+import asyncore_25 as asyncore
+import http_server
+import medusa_gif
+import producers
+from counter import counter
+
+START_TIME = long(time.time())
+
+class status_extension:
+ hit_counter = counter()
+
+ def __init__ (self, objects, statusdir='/status', allow_emergency_debug=0):
+ self.objects = objects
+ self.statusdir = statusdir
+ self.allow_emergency_debug = allow_emergency_debug
+ # We use /status instead of statusdir here because it's too
+ # hard to pass statusdir to the logger, who makes the HREF
+ # to the object dir. We don't need the security-through-
+ # obscurity here in any case, because the id is obscurity enough
+ self.hyper_regex = re.compile('/status/object/([0-9]+)/.*')
+ self.hyper_objects = []
+ for object in objects:
+ self.register_hyper_object (object)
+
+ def __repr__ (self):
+ return '<Status Extension (%s hits) at %x>' % (
+ self.hit_counter,
+ id(self)
+ )
+
+ def match (self, request):
+ path, params, query, fragment = request.split_uri()
+ # For reasons explained above, we don't use statusdir for /object
+ return (path[:len(self.statusdir)] == self.statusdir or
+ path[:len("/status/object/")] == '/status/object/')
+
+ # Possible Targets:
+ # /status
+ # /status/channel_list
+ # /status/medusa.gif
+
+ # can we have 'clickable' objects?
+ # [yes, we can use id(x) and do a linear search]
+
+ # Dynamic producers:
+ # HTTP/1.0: we must close the channel, because it's dynamic output
+ # HTTP/1.1: we can use the chunked transfer-encoding, and leave
+ # it open.
+
+ def handle_request (self, request):
+ [path, params, query, fragment] = request.split_uri()
+ self.hit_counter.increment()
+ if path == self.statusdir: # and not a subdirectory
+ up_time = string.join (english_time (long(time.time()) - START_TIME))
+ request['Content-Type'] = 'text/html'
+ request.push (
+ '<html>'
+ '<title>Medusa Status Reports</title>'
+ '<body bgcolor="#ffffff">'
+ '<h1>Medusa Status Reports</h1>'
+ '<b>Up:</b> %s' % up_time
+ )
+ for i in range(len(self.objects)):
+ try:
+ request.push (self.objects[i].status())
+ except:
+ import traceback, StringIO
+ stream = StringIO.StringIO()
+ traceback.print_exc(None,stream)
+ request.push('<h2><font color="red">Error in Channel %3d: %s</font><pre>%s</pre>' % (i,escape(repr(self.objects[i])), escape(stream.getvalue())))
+ request.push ('<hr>\r\n')
+ request.push (
+ '<p><a href="%s/channel_list">Channel List</a>'
+ '<hr>'
+ '<img src="%s/medusa.gif" align=right width=%d height=%d>'
+ '</body></html>' % (
+ self.statusdir,
+ self.statusdir,
+ medusa_gif.width,
+ medusa_gif.height
+ )
+ )
+ request.done()
+ elif path == self.statusdir + '/channel_list':
+ request['Content-Type'] = 'text/html'
+ request.push ('<html><body>')
+ request.push(channel_list_producer(self.statusdir))
+ request.push (
+ '<hr>'
+ '<img src="%s/medusa.gif" align=right width=%d height=%d>' % (
+ self.statusdir,
+ medusa_gif.width,
+ medusa_gif.height
+ ) +
+ '</body></html>'
+ )
+ request.done()
+
+ elif path == self.statusdir + '/medusa.gif':
+ request['Content-Type'] = 'image/gif'
+ request['Content-Length'] = len(medusa_gif.data)
+ request.push (medusa_gif.data)
+ request.done()
+
+ elif path == self.statusdir + '/close_zombies':
+ message = (
+ '<h2>Closing all zombie http client connections...</h2>'
+ '<p><a href="%s">Back to the status page</a>' % self.statusdir
+ )
+ request['Content-Type'] = 'text/html'
+ request['Content-Length'] = len (message)
+ request.push (message)
+ now = int (time.time())
+ for channel in asyncore.socket_map.keys():
+ if channel.__class__ == http_server.http_channel:
+ if channel != request.channel:
+ if (now - channel.creation_time) > channel.zombie_timeout:
+ channel.close()
+ request.done()
+
+ # Emergency Debug Mode
+ # If a server is running away from you, don't KILL it!
+ # Move all the AF_INET server ports and perform an autopsy...
+ # [disabled by default to protect the innocent]
+ elif self.allow_emergency_debug and path == self.statusdir + '/emergency_debug':
+ request.push ('<html>Moving All Servers...</html>')
+ request.done()
+ for channel in asyncore.socket_map.keys():
+ if channel.accepting:
+ if type(channel.addr) is type(()):
+ ip, port = channel.addr
+ channel.socket.close()
+ channel.del_channel()
+ channel.addr = (ip, port+10000)
+ fam, typ = channel.family_and_type
+ channel.create_socket (fam, typ)
+ channel.set_reuse_addr()
+ channel.bind (channel.addr)
+ channel.listen(5)
+
+ else:
+ m = self.hyper_regex.match (path)
+ if m:
+ oid = string.atoi (m.group (1))
+ for object in self.hyper_objects:
+ if id (object) == oid:
+ if hasattr (object, 'hyper_respond'):
+ object.hyper_respond (self, path, request)
+ else:
+ request.error (404)
+ return
+
+ def status (self):
+ return producers.simple_producer (
+ '<li>Status Extension <b>Hits</b> : %s' % self.hit_counter
+ )
+
+ def register_hyper_object (self, object):
+ if not object in self.hyper_objects:
+ self.hyper_objects.append (object)
+
+import logger
+
+class logger_for_status (logger.tail_logger):
+
+ def status (self):
+ return 'Last %d log entries for: %s' % (
+ len (self.messages),
+ html_repr (self)
+ )
+
+ def hyper_respond (self, sh, path, request):
+ request['Content-Type'] = 'text/plain'
+ messages = self.messages[:]
+ messages.reverse()
+ request.push (lines_producer (messages))
+ request.done()
+
+class lines_producer:
+ def __init__ (self, lines):
+ self.lines = lines
+
+ def more (self):
+ if self.lines:
+ chunk = self.lines[:50]
+ self.lines = self.lines[50:]
+ return string.join (chunk, '\r\n') + '\r\n'
+ else:
+ return ''
+
+class channel_list_producer (lines_producer):
+ def __init__ (self, statusdir):
+ channel_reprs = map (
+ lambda x: '<' + repr(x)[1:-1] + '>',
+ asyncore.socket_map.values()
+ )
+ channel_reprs.sort()
+ lines_producer.__init__ (
+ self,
+ ['<h1>Active Channel List</h1>',
+ '<pre>'
+ ] + channel_reprs + [
+ '</pre>',
+ '<p><a href="%s">Status Report</a>' % statusdir
+ ]
+ )
+
+
+def html_repr (object):
+ so = escape (repr (object))
+ if hasattr (object, 'hyper_respond'):
+ return '<a href="/status/object/%d/">%s</a>' % (id (object), so)
+ else:
+ return so
+
+def html_reprs (list, front='', back=''):
+ reprs = map (
+ lambda x,f=front,b=back: '%s%s%s' % (f,x,b),
+ map (lambda x: escape (html_repr(x)), list)
+ )
+ reprs.sort()
+ return reprs
+
+# for example, tera, giga, mega, kilo
+# p_d (n, (1024, 1024, 1024, 1024))
+# smallest divider goes first - for example
+# minutes, hours, days
+# p_d (n, (60, 60, 24))
+
+def progressive_divide (n, parts):
+ result = []
+ for part in parts:
+ n, rem = divmod (n, part)
+ result.append (rem)
+ result.append (n)
+ return result
+
+# b,k,m,g,t
+def split_by_units (n, units, dividers, format_string):
+ divs = progressive_divide (n, dividers)
+ result = []
+ for i in range(len(units)):
+ if divs[i]:
+ result.append (format_string % (divs[i], units[i]))
+ result.reverse()
+ if not result:
+ return [format_string % (0, units[0])]
+ else:
+ return result
+
+def english_bytes (n):
+ return split_by_units (
+ n,
+ ('','K','M','G','T'),
+ (1024, 1024, 1024, 1024, 1024),
+ '%d %sB'
+ )
+
+def english_time (n):
+ return split_by_units (
+ n,
+ ('secs', 'mins', 'hours', 'days', 'weeks', 'years'),
+ ( 60, 60, 24, 7, 52),
+ '%d %s'
+ )
Added: supervisor/trunk/src/medusa/test/asyn_http_bench.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/test/asyn_http_bench.py Fri May 22 23:51:54 2009
@@ -0,0 +1,98 @@
+#! /usr/local/bin/python1.4
+# -*- Mode: Python -*-
+
+import asyncore
+import socket
+import string
+import sys
+
+def blurt (thing):
+ sys.stdout.write (thing)
+ sys.stdout.flush ()
+
+total_sessions = 0
+
+class http_client (asyncore.dispatcher_with_send):
+ def __init__ (self, host='127.0.0.1', port=80, uri='/', num=10):
+ asyncore.dispatcher_with_send.__init__ (self)
+ self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
+ self.host = host
+ self.port = port
+ self.uri = uri
+ self.num = num
+ self.bytes = 0
+ self.connect ((host, port))
+
+ def log (self, *info):
+ pass
+
+ def handle_connect (self):
+ self.connected = 1
+# blurt ('o')
+ self.send ('GET %s HTTP/1.0\r\n\r\n' % self.uri)
+
+ def handle_read (self):
+# blurt ('.')
+ d = self.recv (8192)
+ self.bytes = self.bytes + len(d)
+
+ def handle_close (self):
+ global total_sessions
+# blurt ('(%d)' % (self.bytes))
+ self.close()
+ total_sessions = total_sessions + 1
+ if self.num:
+ http_client (self.host, self.port, self.uri, self.num-1)
+
+import time
+class timer:
+ def __init__ (self):
+ self.start = time.time()
+ def end (self):
+ return time.time() - self.start
+
+from asyncore import socket_map, poll
+
+MAX = 0
+
+def loop (timeout=30.0):
+ global MAX
+ while socket_map:
+ if len(socket_map) > MAX:
+ MAX = len(socket_map)
+ poll (timeout)
+
+if __name__ == '__main__':
+ if len(sys.argv) < 6:
+ print 'usage: %s <host> <port> <uri> <hits> <num_clients>' % sys.argv[0]
+ else:
+ [host, port, uri, hits, num] = sys.argv[1:]
+ hits = string.atoi (hits)
+ num = string.atoi (num)
+ port = string.atoi (port)
+ t = timer()
+ clients = map (lambda x: http_client (host, port, uri, hits-1), range(num))
+ #import profile
+ #profile.run ('loop')
+ loop()
+ total_time = t.end()
+ print (
+ '\n%d clients\n%d hits/client\n'
+ 'total_hits:%d\n%.3f seconds\ntotal hits/sec:%.3f' % (
+ num,
+ hits,
+ total_sessions,
+ total_time,
+ total_sessions / total_time
+ )
+ )
+ print 'Max. number of concurrent sessions: %d' % (MAX)
+
+
+# linux 2.x, talking to medusa
+# 50 clients
+# 1000 hits/client
+# total_hits:50000
+# 2255.858 seconds
+# total hits/sec:22.165
+# Max. number of concurrent sessions: 50
Added: supervisor/trunk/src/medusa/test/bench.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/test/bench.py Fri May 22 23:51:54 2009
@@ -0,0 +1,35 @@
+# -*- Mode: Python -*-
+
+# benchmark a single channel, pipelined
+
+request = 'GET /index.html HTTP/1.0\r\nConnection: Keep-Alive\r\n\r\n'
+last_request = 'GET /index.html HTTP/1.0\r\nConnection: close\r\n\r\n'
+
+import socket
+import time
+
+class timer:
+ def __init__ (self):
+ self.start = time.time()
+ def end (self):
+ return time.time() - self.start
+
+def bench (host, port=80, n=100):
+ s = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
+ s.connect ((host, port))
+ t = timer()
+ s.send ((request * n) + last_request)
+ while 1:
+ d = s.recv(65536)
+ if not d:
+ break
+ total = t.end()
+ print 'time: %.2f seconds (%.2f hits/sec)' % (total, n/total)
+
+if __name__ == '__main__':
+ import sys
+ import string
+ if len(sys.argv) < 3:
+ print 'usage: %s <host> <port> <count>' % (sys.argv[0])
+ else:
+ bench (sys.argv[1], string.atoi (sys.argv[2]), string.atoi (sys.argv[3]))
Added: supervisor/trunk/src/medusa/test/max_sockets.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/test/max_sockets.py Fri May 22 23:51:54 2009
@@ -0,0 +1,64 @@
+# -*- Mode: Python -*-
+
+import socket
+import select
+
+# several factors here we might want to test:
+# 1) max we can create
+# 2) max we can bind
+# 3) max we can listen on
+# 4) max we can connect
+
+def max_server_sockets():
+ sl = []
+ while 1:
+ try:
+ s = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
+ s.bind (('',0))
+ s.listen(5)
+ sl.append (s)
+ except:
+ break
+ num = len(sl)
+ for s in sl:
+ s.close()
+ del sl
+ return num
+
+def max_client_sockets():
+ # make a server socket
+ server = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
+ server.bind (('', 9999))
+ server.listen (5)
+ sl = []
+ while 1:
+ try:
+ s = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
+ s.connect (('', 9999))
+ conn, addr = server.accept()
+ sl.append ((s,conn))
+ except:
+ break
+ num = len(sl)
+ for s,c in sl:
+ s.close()
+ c.close()
+ del sl
+ return num
+
+def max_select_sockets():
+ sl = []
+ while 1:
+ try:
+ s = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
+ s.bind (('',0))
+ s.listen(5)
+ sl.append (s)
+ select.select(sl,[],[],0)
+ except:
+ break
+ num = len(sl) - 1
+ for s in sl:
+ s.close()
+ del sl
+ return num
Added: supervisor/trunk/src/medusa/test/test_11.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/test/test_11.py Fri May 22 23:51:54 2009
@@ -0,0 +1,110 @@
+# -*- Mode: Python -*-
+
+import socket
+import string
+from medusa import asyncore_25 as asyncore
+from medusa import asynchat_25 as asynchat
+
+# get some performance figures for an HTTP/1.1 server.
+# use pipelining.
+
+class test_client (asynchat.async_chat):
+
+ ac_in_buffer_size = 16384
+ ac_out_buffer_size = 16384
+
+ total_in = 0
+
+ concurrent = 0
+ max_concurrent = 0
+
+ def __init__ (self, addr, chain):
+ asynchat.async_chat.__init__ (self)
+ self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
+ self.set_terminator ('\r\n\r\n')
+ self.connect (addr)
+ self.push (chain)
+
+ def handle_connect (self):
+ test_client.concurrent = test_client.concurrent + 1
+ if (test_client.concurrent > test_client.max_concurrent):
+ test_client.max_concurrent = test_client.concurrent
+
+ def handle_expt (self):
+ print 'unexpected FD_EXPT thrown. closing()'
+ self.close()
+
+ def close (self):
+ test_client.concurrent = test_client.concurrent - 1
+ asynchat.async_chat.close(self)
+
+ def collect_incoming_data (self, data):
+ test_client.total_in = test_client.total_in + len(data)
+
+ def found_terminator (self):
+ pass
+
+ def log (self, *args):
+ pass
+
+
+import time
+
+class timer:
+ def __init__ (self):
+ self.start = time.time()
+
+ def end (self):
+ return time.time() - self.start
+
+def build_request_chain (num, host, request_size):
+ s = 'GET /test%d.html HTTP/1.1\r\nHost: %s\r\n\r\n' % (request_size, host)
+ sl = [s] * (num-1)
+ sl.append (
+ 'GET /test%d.html HTTP/1.1\r\nHost: %s\r\nConnection: close\r\n\r\n' % (
+ request_size, host
+ )
+ )
+ return string.join (sl, '')
+
+if __name__ == '__main__':
+ import string
+ import sys
+ if len(sys.argv) != 6:
+ print 'usage: %s <host> <port> <request-size> <num-requests> <num-connections>\n' % sys.argv[0]
+ else:
+ host = sys.argv[1]
+
+ ip = socket.gethostbyname (host)
+
+ [port, request_size, num_requests, num_conns] = map (
+ string.atoi, sys.argv[2:]
+ )
+
+ chain = build_request_chain (num_requests, host, request_size)
+
+ t = timer()
+ for i in range (num_conns):
+ test_client ((host,port), chain)
+ asyncore.loop()
+ total_time = t.end()
+
+ # ok, now do some numbers
+ total_bytes = test_client.total_in
+ num_trans = num_requests * num_conns
+ throughput = float (total_bytes) / total_time
+ trans_per_sec = num_trans / total_time
+
+ sys.stderr.write ('total time: %.2f\n' % total_time)
+ sys.stderr.write ('number of transactions: %d\n' % num_trans)
+ sys.stderr.write ('total bytes sent: %d\n' % total_bytes)
+ sys.stderr.write ('total throughput (bytes/sec): %.2f\n' % throughput)
+ sys.stderr.write ('transactions/second: %.2f\n' % trans_per_sec)
+ sys.stderr.write ('max concurrent connections: %d\n' % test_client.max_concurrent)
+
+ sys.stdout.write (
+ string.join (
+ map (str, (num_conns, num_requests, request_size, throughput, trans_per_sec)),
+ ','
+ ) + '\n'
+ )
Added: supervisor/trunk/src/medusa/test/test_lb.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/test/test_lb.py Fri May 22 23:51:54 2009
@@ -0,0 +1,159 @@
+# -*- Mode: Python -*-
+
+# Get a lower bound for Medusa performance with a simple async
+# client/server benchmark built on the async lib. The idea is to test
+# all the underlying machinery [select, asyncore, asynchat, etc...] in
+# a context where there is virtually no processing of the data.
+
+import socket
+import select
+import sys
+
+# ==================================================
+# server
+# ==================================================
+
+from medusa import asyncore_25 as asyncore
+from medusa import asynchat_25 as asynchat
+
+class test_channel (asynchat.async_chat):
+
+ ac_in_buffer_size = 16384
+ ac_out_buffer_size = 16384
+
+ total_in = 0
+
+ def __init__ (self, conn, addr):
+ asynchat.async_chat.__init__ (self, conn)
+ self.set_terminator ('\r\n\r\n')
+ self.buffer = ''
+
+ def collect_incoming_data (self, data):
+ self.buffer = self.buffer + data
+ test_channel.total_in = test_channel.total_in + len(data)
+
+ def found_terminator (self):
+ # we've gotten the data, now send it back
+ data = self.buffer
+ self.buffer = ''
+ self.push (data+'\r\n\r\n')
+
+ def handle_close (self):
+ sys.stdout.write ('.'); sys.stdout.flush()
+ self.close()
+
+ def log (self, *args):
+ pass
+
+class test_server (asyncore.dispatcher):
+ def __init__ (self, addr):
+
+ if type(addr) == type(''):
+ f = socket.AF_UNIX
+ else:
+ f = socket.AF_INET
+
+ self.create_socket (f, socket.SOCK_STREAM)
+ self.bind (addr)
+ self.listen (5)
+ print 'server started on',addr
+
+ def handle_accept (self):
+ conn, addr = self.accept()
+ test_channel (conn, addr)
+
+# ==================================================
+# client
+# ==================================================
+
+# pretty much the same behavior, except that we kick
+# off the exchange and decide when to quit
+
+class test_client (test_channel):
+
+ def __init__ (self, addr, packet, number):
+ if type(addr) == type(''):
+ f = socket.AF_UNIX
+ else:
+ f = socket.AF_INET
+
+ asynchat.async_chat.__init__ (self)
+ self.create_socket (f, socket.SOCK_STREAM)
+ self.set_terminator ('\r\n\r\n')
+ self.buffer = ''
+ self.connect (addr)
+ self.push (packet + '\r\n\r\n')
+ self.number = number
+ self.count = 0
+
+ def handle_connect (self):
+ pass
+
+ def found_terminator (self):
+ self.count = self.count + 1
+ if self.count == self.number:
+ sys.stdout.write('.'); sys.stdout.flush()
+ self.close()
+ else:
+ test_channel.found_terminator (self)
+
+import time
+
+class timer:
+ def __init__ (self):
+ self.start = time.time()
+
+ def end (self):
+ return time.time() - self.start
+
+if __name__ == '__main__':
+ import string
+
+ if '--poll' in sys.argv:
+ sys.argv.remove ('--poll')
+ use_poll=1
+ else:
+ use_poll=0
+
+ if len(sys.argv) == 1:
+ print 'usage: %s\n' \
+ ' (as a server) [--poll] -s <ip> <port>\n' \
+ ' (as a client) [--poll] -c <ip> <port> <packet-size> <num-packets> <num-connections>\n' % sys.argv[0]
+ sys.exit(0)
+ if sys.argv[1] == '-s':
+ s = test_server ((sys.argv[2], string.atoi (sys.argv[3])))
+ asyncore.loop(use_poll=use_poll)
+ elif sys.argv[1] == '-c':
+ # create the packet
+ packet = string.atoi(sys.argv[4]) * 'B'
+ host = sys.argv[2]
+ port = string.atoi (sys.argv[3])
+ num_packets = string.atoi (sys.argv[5])
+ num_conns = string.atoi (sys.argv[6])
+
+ t = timer()
+ for i in range (num_conns):
+ test_client ((host,port), packet, num_packets)
+ asyncore.loop(use_poll=use_poll)
+ total_time = t.end()
+
+ # ok, now do some numbers
+ bytes = test_client.total_in
+ num_trans = num_packets * num_conns
+ total_bytes = num_trans * len(packet)
+ throughput = float (total_bytes) / total_time
+ trans_per_sec = num_trans / total_time
+
+ sys.stderr.write ('total time: %.2f\n' % total_time)
+ sys.stderr.write ( 'number of transactions: %d\n' % num_trans)
+ sys.stderr.write ( 'total bytes sent: %d\n' % total_bytes)
+ sys.stderr.write ( 'total throughput (bytes/sec): %.2f\n' % throughput)
+ sys.stderr.write ( ' [note, throughput is this amount in each direction]\n')
+ sys.stderr.write ( 'transactions/second: %.2f\n' % trans_per_sec)
+
+ sys.stdout.write (
+ string.join (
+ map (str, (num_conns, num_packets, len(packet), throughput, trans_per_sec)),
+ ','
+ ) + '\n'
+ )
Added: supervisor/trunk/src/medusa/test/test_medusa.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/test/test_medusa.py Fri May 22 23:51:54 2009
@@ -0,0 +1,51 @@
+# -*- Mode: Python -*-
+
+import socket
+import string
+import time
+from medusa import http_date
+
+now = http_date.build_http_date (time.time())
+
+cache_request = string.joinfields (
+ ['GET / HTTP/1.0',
+ 'If-Modified-Since: %s' % now,
+ ],
+ '\r\n'
+ ) + '\r\n\r\n'
+
+nocache_request = 'GET / HTTP/1.0\r\n\r\n'
+
+def get (request, host='', port=80):
+ s = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
+ s.connect((host, port))
+ s.send (request)
+ while 1:
+ d = s.recv (8192)
+ if not d:
+ break
+ s.close()
+
+class timer:
+ def __init__ (self):
+ self.start = time.time()
+ def end (self):
+ return time.time() - self.start
+
+def test_cache (n=1000):
+ t = timer()
+ for i in xrange (n):
+ get(cache_request)
+ end = t.end()
+ print 'cache: %d requests, %.2f seconds, %.2f hits/sec' % (n, end, n/end)
+
+def test_nocache (n=1000):
+ t = timer()
+ for i in xrange (n):
+ get(nocache_request)
+ end = t.end()
+ print 'nocache: %d requests, %.2f seconds, %.2f hits/sec' % (n, end, n/end)
+
+if __name__ == '__main__':
+ test_cache()
+ test_nocache()
Added: supervisor/trunk/src/medusa/test/test_producers.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/test/test_producers.py Fri May 22 23:51:54 2009
@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+
+#
+# Test script for producers.py
+#
+
+__revision__ = "$Id: test_producers.py,v 1.2 2002/09/18 20:16:40 akuchling Exp $"
+
+import StringIO, zlib
+from sancho.unittest import TestScenario, parse_args, run_scenarios
+
+tested_modules = ["medusa.producers"]
+
+
+from medusa import producers
+
+test_string = ''
+for i in range(16385):
+ test_string += chr(48 + (i%10))
+
+class ProducerTest (TestScenario):
+
+ def setup (self):
+ pass
+
+ def shutdown (self):
+ pass
+
+ def _check_all (self, p, expected_string):
+ # Check that a producer returns all of the string,
+ # and that it's the unchanged string.
+ count = 0
+ data = ""
+ while 1:
+ s = p.more()
+ if s == "":
+ break
+ count += len(s)
+ data += s
+ self.test_val('count', len(expected_string))
+ self.test_val('data', expected_string)
+ self.test_val('p.more()', '')
+ return data
+
+ def check_simple (self):
+ p = producers.simple_producer(test_string)
+ self.test_val('p.more()', test_string[:1024])
+
+ p = producers.simple_producer(test_string, buffer_size = 5)
+ self._check_all(p, test_string)
+
+ def check_scanning (self):
+ p = producers.scanning_producer(test_string)
+ self.test_val('p.more()', test_string[:1024])
+
+ p = producers.scanning_producer(test_string, buffer_size = 5)
+ self._check_all(p, test_string)
+
+ def check_lines (self):
+ p = producers.lines_producer(['a']* 65)
+ self._check_all(p, 'a\r\n'*65)
+
+ def check_buffer (self):
+ p = producers.buffer_list_producer(['a']* 1027)
+ self._check_all(p, 'a'*1027)
+
+ def check_file (self):
+ f = StringIO.StringIO(test_string)
+ p = producers.file_producer(f)
+ self._check_all(p, test_string)
+
+ def check_output (self):
+ p = producers.output_producer()
+ for i in range(0,66):
+ p.write('a')
+ for i in range(0,65):
+ p.write('b\n')
+ self._check_all(p, 'a'*66 + 'b\r\n'*65)
+
+ def check_composite (self):
+ p1 = producers.simple_producer('a'*66, buffer_size = 5)
+ p2 = producers.lines_producer(['b']*65)
+ p = producers.composite_producer([p1, p2])
+ self._check_all(p, 'a'*66 + 'b\r\n'*65)
+
+ def check_glob (self):
+ p1 = producers.simple_producer(test_string, buffer_size = 5)
+ p = producers.globbing_producer(p1, buffer_size = 1024)
+ self.test_true('1024 <= len(p.more())')
+
+ def check_hooked (self):
+ def f (num_bytes):
+ self.test_val('num_bytes', len(test_string))
+ p1 = producers.simple_producer(test_string, buffer_size = 5)
+ p = producers.hooked_producer(p1, f)
+ self._check_all(p, test_string)
+
+ def check_chunked (self):
+ p1 = producers.simple_producer('the quick brown fox', buffer_size = 5)
+ p = producers.chunked_producer(p1, footers=['FOOTER'])
+ self._check_all(p, """5\r
+the q\r
+5\r
+uick \r
+5\r
+brown\r
+4\r
+ fox\r
+0\r
+FOOTER\r
+\r\n""")
+
+ def check_compressed (self):
+ p1 = producers.simple_producer(test_string, buffer_size = 5)
+ p = producers.compressed_producer(p1)
+ compr_data = self._check_all(p, zlib.compress(test_string, 5))
+ self.test_val('zlib.decompress(compr_data)', test_string)
+
+ def check_escaping (self):
+ p1 = producers.simple_producer('the quick brown fox', buffer_size = 5)
+ p = producers.escaping_producer(p1,
+ esc_from = ' ',
+ esc_to = '_')
+ self._check_all(p, 'the_quick_brown_fox')
+
+# class ProducerTest
+
+
+if __name__ == "__main__":
+ (scenarios, options) = parse_args()
+ run_scenarios(scenarios, options)
Added: supervisor/trunk/src/medusa/test/test_single_11.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/test/test_single_11.py Fri May 22 23:51:54 2009
@@ -0,0 +1,53 @@
+# -*- Mode: Python -*-
+
+# no-holds barred, test a single channel's pipelining speed
+
+import string
+import socket
+
+def build_request_chain (num, host, request_size):
+ s = 'GET /test%d.html HTTP/1.1\r\nHost: %s\r\n\r\n' % (request_size, host)
+ sl = [s] * (num-1)
+ sl.append (
+ 'GET /test%d.html HTTP/1.1\r\nHost: %s\r\nConnection: close\r\n\r\n' % (
+ request_size, host
+ )
+ )
+ return string.join (sl, '')
+
+import time
+
+class timer:
+ def __init__ (self):
+ self.start = time.time()
+
+ def end (self):
+ return time.time() - self.start
+
+if __name__ == '__main__':
+ import sys
+ if len(sys.argv) != 5:
+ print 'usage: %s <host> <port> <request-size> <num-requests>' % (sys.argv[0])
+ else:
+ host = sys.argv[1]
+ [port, request_size, num_requests] = map (
+ string.atoi,
+ sys.argv[2:]
+ )
+ chain = build_request_chain (num_requests, host, request_size)
+ import socket
+ s = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
+ s.connect ((host,port))
+ t = timer()
+ s.send (chain)
+ num_bytes = 0
+ while 1:
+ data = s.recv(16384)
+ if not data:
+ break
+ else:
+ num_bytes = num_bytes + len(data)
+ total_time = t.end()
+ print 'total bytes received: %d' % num_bytes
+ print 'total time: %.2f sec' % (total_time)
+ print 'transactions/sec: %.2f' % (num_requests/total_time)
Added: supervisor/trunk/src/medusa/test/tests.txt
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/test/tests.txt Fri May 22 23:51:54 2009
@@ -0,0 +1,73 @@
+# server: linux, 486dx2/66
+# client: win95, cyrix 6x86 p166+
+# over ethernet.
+#
+# number of connections
+# | number of requests per connection
+# | | packet size
+# | | | throughput (bytes/sec)
+# | | | | transactions/sec
+# | | | | |
+ 1 50 64 3440.86 53.76
+ 1 100 64 3422.45 53.47
+ 1 1 256 5120.00 20.00
+ 1 50 256 13763.44 53.76
+ 1 100 256 13333.33 52.08
+ 1 1 1024 6400.00 6.25
+ 1 50 1024 6909.58 6.74
+ 1 100 1024 6732.41 6.57
+ 1 1 4096 14628.56 3.57
+ 1 50 4096 17181.20 4.19
+ 1 100 4096 16835.18 4.11
+ 5 1 64 1882.35 29.41
+ 5 50 64 3990.02 62.34
+ 5 100 64 3907.20 61.05
+ 5 1 256 5818.18 22.72
+ 5 50 256 15533.98 60.67
+ 5 100 256 15744.15 61.50
+ 5 1 1024 15515.14 15.15
+ 5 50 1024 23188.40 22.64
+ 5 100 1024 23659.88 23.10
+ 5 1 4096 28444.44 6.94
+ 5 50 4096 34913.05 8.52
+ 5 100 4096 35955.05 8.77
+ 10 1 64 191.04 2.98
+ 10 50 64 4045.51 63.21
+ 10 100 64 4045.51 63.21
+ 10 1 256 764.17 2.98
+ 10 50 256 15552.85 60.75
+ 10 100 256 15581.25 60.86
+ 10 1 1024 2959.53 2.89
+ 10 50 1024 25061.18 24.47
+ 10 100 1024 25498.00 24.90
+ 10 1 4096 11314.91 2.76
+ 10 50 4096 39002.09 9.52
+ 10 100 4096 38780.53 9.46
+ 15 1 64 277.45 4.33
+ 15 50 64 4067.79 63.55
+ 15 100 64 4083.36 63.80
+ 15 1 256 386.31 1.50
+ 15 50 256 15262.32 59.61
+ 15 100 256 15822.00 61.80
+ 15 1 1024 1528.35 1.49
+ 15 50 1024 27263.04 26.62
+ 15 100 1024 27800.90 27.14
+ 15 1 4096 6047.24 1.47
+ 15 50 4096 39695.05 9.69
+ 15 100 4096 37112.65 9.06
+ 20 1 64 977.09 15.26
+ 20 50 64 2538.67 39.66
+ 20 100 64 3377.30 52.77
+ 20 1 256 221.93 0.86
+ 20 50 256 10815.37 42.24
+ 20 100 256 15880.89 62.03
+ 20 1 1024 883.52 0.86
+ 20 50 1024 29315.77 28.62
+ 20 100 1024 29569.73 28.87
+ 20 1 4096 7892.10 1.92
+ 20 50 4096 40223.90 9.82
+ 20 100 4096 41325.73 10.08
+#
+# There's a big gap in trans/sec between 256 and 1024 bytes, we should
+# probably stick a 512 in there.
+#
Added: supervisor/trunk/src/medusa/thread/pi_module.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/thread/pi_module.py Fri May 22 23:51:54 2009
@@ -0,0 +1,62 @@
+# -*- Mode: Python -*-
+
+# [reworking of the version in Python-1.5.1/Demo/scripts/pi.py]
+
+# Print digits of pi forever.
+#
+# The algorithm, using Python's 'long' integers ("bignums"), works
+# with continued fractions, and was conceived by Lambert Meertens.
+#
+# See also the ABC Programmer's Handbook, by Geurts, Meertens & Pemberton,
+# published by Prentice-Hall (UK) Ltd., 1990.
+
+import string
+
+StopException = "Stop!"
+
+def go (file):
+ try:
+ k, a, b, a1, b1 = 2L, 4L, 1L, 12L, 4L
+ while 1:
+ # Next approximation
+ p, q, k = k*k, 2L*k+1L, k+1L
+ a, b, a1, b1 = a1, b1, p*a+q*a1, p*b+q*b1
+ # Print common digits
+ d, d1 = a/b, a1/b1
+ while d == d1:
+ if file.write (str(int(d))):
+ raise StopException
+ a, a1 = 10L*(a%b), 10L*(a1%b1)
+ d, d1 = a/b, a1/b1
+ except StopException:
+ return
+
+class line_writer:
+
+ "partition the endless line into 80-character ones"
+
+ def __init__ (self, file, digit_limit=10000):
+ self.file = file
+ self.buffer = ''
+ self.count = 0
+ self.digit_limit = digit_limit
+
+ def write (self, data):
+ self.buffer = self.buffer + data
+ if len(self.buffer) > 80:
+ line, self.buffer = self.buffer[:80], self.buffer[80:]
+ self.file.write (line+'\r\n')
+ self.count = self.count + 80
+ if self.count > self.digit_limit:
+ return 1
+ else:
+ return 0
+
+def main (env, stdin, stdout):
+ parts = string.split (env['REQUEST_URI'], '/')
+ if len(parts) >= 3:
+ ndigits = string.atoi (parts[2])
+ else:
+ ndigits = 5000
+ stdout.write ('Content-Type: text/plain\r\n\r\n')
+ go (line_writer (stdout, ndigits))
Added: supervisor/trunk/src/medusa/thread/select_trigger.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/thread/select_trigger.py Fri May 22 23:51:54 2009
@@ -0,0 +1,328 @@
+# -*- Mode: Python -*-
+
+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+__revision__ = "$Id: select_trigger.py,v 1.4 2003/01/09 15:49:15 akuchling Exp $"
+
+import asyncore_25 as asyncore
+import asynchat_25 as asynchat
+
+import os
+import socket
+import string
+import thread
+
+if os.name == 'posix':
+
+ class trigger (asyncore.file_dispatcher):
+
+ "Wake up a call to select() running in the main thread"
+
+ # This is useful in a context where you are using Medusa's I/O
+ # subsystem to deliver data, but the data is generated by another
+ # thread. Normally, if Medusa is in the middle of a call to
+ # select(), new output data generated by another thread will have
+ # to sit until the call to select() either times out or returns.
+ # If the trigger is 'pulled' by another thread, it should immediately
+ # generate a READ event on the trigger object, which will force the
+ # select() invocation to return.
+
+ # A common use for this facility: letting Medusa manage I/O for a
+ # large number of connections; but routing each request through a
+ # thread chosen from a fixed-size thread pool. When a thread is
+ # acquired, a transaction is performed, but output data is
+ # accumulated into buffers that will be emptied more efficiently
+ # by Medusa. [picture a server that can process database queries
+ # rapidly, but doesn't want to tie up threads waiting to send data
+ # to low-bandwidth connections]
+
+ # The other major feature provided by this class is the ability to
+ # move work back into the main thread: if you call pull_trigger()
+ # with a thunk argument, when select() wakes up and receives the
+ # event it will call your thunk from within that thread. The main
+ # purpose of this is to remove the need to wrap thread locks around
+ # Medusa's data structures, which normally do not need them. [To see
+ # why this is true, imagine this scenario: A thread tries to push some
+ # new data onto a channel's outgoing data queue at the same time that
+ # the main thread is trying to remove some]
+
+ def __init__ (self):
+ r, w = self._fds = os.pipe()
+ self.trigger = w
+ asyncore.file_dispatcher.__init__(self, r)
+ self.lock = thread.allocate_lock()
+ self.thunks = []
+ self._closed = 0
+
+ # Override the asyncore close() method, because it seems that
+ # it would only close the r file descriptor and not w. The
+ # constructor calls file_dispatcher.__init__ and passes r,
+ # which would get stored in a file_wrapper and get closed by
+ # the default close. But that would leave w open...
+
+ def close(self):
+ if not self._closed:
+ self._closed = 1
+ self.del_channel()
+ for fd in self._fds:
+ os.close(fd)
+ self._fds = []
+
+ def __repr__ (self):
+ return '<select-trigger (pipe) at %x>' % id(self)
+
+ def readable (self):
+ return 1
+
+ def writable (self):
+ return 0
+
+ def handle_connect (self):
+ pass
+
+ def handle_close(self):
+ self.close()
+
+ def pull_trigger (self, thunk=None):
+ # print 'PULL_TRIGGER: ', len(self.thunks)
+ if thunk:
+ self.lock.acquire()
+ try:
+ self.thunks.append(thunk)
+ finally:
+ self.lock.release()
+ os.write(self.trigger, 'x')
+
+ def handle_read (self):
+ try:
+ self.recv(8192)
+ except socket.error:
+ return
+ self.lock.acquire()
+ try:
+ for thunk in self.thunks:
+ try:
+ thunk()
+ except:
+ nil, t, v, tbinfo = asyncore.compact_traceback()
+ print ('exception in trigger thunk:'
+ ' (%s:%s %s)' % (t, v, tbinfo))
+ self.thunks = []
+ finally:
+ self.lock.release()
+
+else:
+
+ # win32-safe version
+
+ # XXX Should define a base class that has the common methods and
+ # then put the platform-specific in a subclass named trigger.
+
+ HOST = '127.0.0.1'
+ MINPORT = 19950
+ NPORTS = 50
+
+ class trigger (asyncore.dispatcher):
+ portoffset = 0
+
+ def __init__ (self):
+ a = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ w = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+
+ # set TCP_NODELAY to true to avoid buffering
+ w.setsockopt(socket.IPPROTO_TCP, 1, 1)
+
+ # tricky: get a pair of connected sockets
+ for i in range(NPORTS):
+ trigger.portoffset = (trigger.portoffset + 1) % NPORTS
+ port = MINPORT + trigger.portoffset
+ address = (HOST, port)
+ try:
+ a.bind(address)
+ except socket.error:
+ continue
+ else:
+ break
+ else:
+ raise RuntimeError, 'Cannot bind trigger!'
+
+ a.listen(1)
+ w.setblocking(0)
+ try:
+ w.connect(address)
+ except:
+ pass
+ r, addr = a.accept()
+ a.close()
+ w.setblocking(1)
+ self.trigger = w
+
+ asyncore.dispatcher.__init__(self, r)
+ self.lock = thread.allocate_lock()
+ self.thunks = []
+ self._trigger_connected = 0
+
+ def __repr__ (self):
+ return '<select-trigger (loopback) at %x>' % id(self)
+
+ def readable (self):
+ return 1
+
+ def writable (self):
+ return 0
+
+ def handle_connect (self):
+ pass
+
+ def pull_trigger (self, thunk=None):
+ if thunk:
+ self.lock.acquire()
+ try:
+ self.thunks.append(thunk)
+ finally:
+ self.lock.release()
+ self.trigger.send('x')
+
+ def handle_read (self):
+ try:
+ self.recv(8192)
+ except socket.error:
+ return
+ self.lock.acquire()
+ try:
+ for thunk in self.thunks:
+ try:
+ thunk()
+ except:
+ nil, t, v, tbinfo = asyncore.compact_traceback()
+ print ('exception in trigger thunk:'
+ ' (%s:%s %s)' % (t, v, tbinfo))
+ self.thunks = []
+ finally:
+ self.lock.release()
+
+
+the_trigger = None
+
+class trigger_file:
+
+ "A 'triggered' file object"
+
+ buffer_size = 4096
+
+ def __init__ (self, parent):
+ global the_trigger
+ if the_trigger is None:
+ the_trigger = trigger()
+ self.parent = parent
+ self.buffer = ''
+
+ def write (self, data):
+ self.buffer = self.buffer + data
+ if len(self.buffer) > self.buffer_size:
+ d, self.buffer = self.buffer, ''
+ the_trigger.pull_trigger (
+ lambda d=d,p=self.parent: p.push (d)
+ )
+
+ def writeline (self, line):
+ self.write (line+'\r\n')
+
+ def writelines (self, lines):
+ self.write (
+ string.joinfields (
+ lines,
+ '\r\n'
+ ) + '\r\n'
+ )
+
+ def flush (self):
+ if self.buffer:
+ d, self.buffer = self.buffer, ''
+ the_trigger.pull_trigger (
+ lambda p=self.parent,d=d: p.push (d)
+ )
+
+ def softspace (self, *args):
+ pass
+
+ def close (self):
+ # in a derived class, you may want to call trigger_close() instead.
+ self.flush()
+ self.parent = None
+
+ def trigger_close (self):
+ d, self.buffer = self.buffer, ''
+ p, self.parent = self.parent, None
+ the_trigger.pull_trigger (
+ lambda p=p,d=d: (p.push(d), p.close_when_done())
+ )
+
+if __name__ == '__main__':
+
+ import time
+
+ def thread_function (output_file, i, n):
+ print 'entering thread_function'
+ while n:
+ time.sleep (5)
+ output_file.write ('%2d.%2d %s\r\n' % (i, n, output_file))
+ output_file.flush()
+ n = n - 1
+ output_file.close()
+ print 'exiting thread_function'
+
+ class thread_parent (asynchat.async_chat):
+
+ def __init__ (self, conn, addr):
+ self.addr = addr
+ asynchat.async_chat.__init__ (self, conn)
+ self.set_terminator ('\r\n')
+ self.buffer = ''
+ self.count = 0
+
+ def collect_incoming_data (self, data):
+ self.buffer = self.buffer + data
+
+ def found_terminator (self):
+ data, self.buffer = self.buffer, ''
+ if not data:
+ asyncore.close_all()
+ print "done"
+ return
+ n = string.atoi (string.split (data)[0])
+ tf = trigger_file (self)
+ self.count = self.count + 1
+ thread.start_new_thread (thread_function, (tf, self.count, n))
+
+ class thread_server (asyncore.dispatcher):
+
+ def __init__ (self, family=socket.AF_INET, address=('', 9003)):
+ asyncore.dispatcher.__init__ (self)
+ self.create_socket (family, socket.SOCK_STREAM)
+ self.set_reuse_addr()
+ self.bind (address)
+ self.listen (5)
+
+ def handle_accept (self):
+ conn, addr = self.accept()
+ tp = thread_parent (conn, addr)
+
+ thread_server()
+ #asyncore.loop(1.0, use_poll=1)
+ try:
+ asyncore.loop ()
+ except:
+ asyncore.close_all()
Added: supervisor/trunk/src/medusa/thread/test_module.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/thread/test_module.py Fri May 22 23:51:54 2009
@@ -0,0 +1,12 @@
+# -*- Mode: Python -*-
+
+import pprint
+
+def main (env, stdin, stdout):
+
+ stdout.write (
+ '<html><body><h1>Test CGI Module</h1>\r\n'
+ '<br>The Environment:<pre>\r\n'
+ )
+ pprint.pprint (env, stdout)
+ stdout.write ('</pre></body></html>\r\n')
Added: supervisor/trunk/src/medusa/thread/thread_channel.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/thread/thread_channel.py Fri May 22 23:51:54 2009
@@ -0,0 +1,125 @@
+# -*- Mode: Python -*-
+
+VERSION_STRING = "$Id: thread_channel.py,v 1.3 2002/03/19 22:49:40 amk Exp $"
+
+# This will probably only work on Unix.
+
+# The disadvantage to this technique is that it wastes file
+# descriptors (especially when compared to select_trigger.py)
+
+# May be possible to do it on Win32, using TCP localhost sockets.
+# [does winsock support 'socketpair'?]
+
+import asyncore_25 as asyncore
+import asynchat_25 as asynchat
+
+import fcntl
+import FCNTL
+import os
+import socket
+import string
+import thread
+
+# this channel slaves off of another one. it starts a thread which
+# pumps its output through the 'write' side of the pipe. The 'read'
+# side of the pipe will then notify us when data is ready. We push
+# this data on the owning data channel's output queue.
+
+class thread_channel (asyncore.file_dispatcher):
+
+ buffer_size = 8192
+
+ def __init__ (self, channel, function, *args):
+ self.parent = channel
+ self.function = function
+ self.args = args
+ self.pipe = rfd, wfd = os.pipe()
+ asyncore.file_dispatcher.__init__ (self, rfd)
+
+ def start (self):
+ rfd, wfd = self.pipe
+
+ # The read side of the pipe is set to non-blocking I/O; it is
+ # 'owned' by medusa.
+
+ flags = fcntl.fcntl (rfd, FCNTL.F_GETFL, 0)
+ fcntl.fcntl (rfd, FCNTL.F_SETFL, flags | FCNTL.O_NDELAY)
+
+ # The write side of the pipe is left in blocking mode; it is
+ # 'owned' by the thread. However, we wrap it up as a file object.
+ # [who wants to 'write()' to a number?]
+
+ of = os.fdopen (wfd, 'w')
+
+ thread.start_new_thread (
+ self.function,
+ # put the output file in front of the other arguments
+ (of,) + self.args
+ )
+
+ def writable (self):
+ return 0
+
+ def readable (self):
+ return 1
+
+ def handle_read (self):
+ data = self.recv (self.buffer_size)
+ self.parent.push (data)
+
+ def handle_close (self):
+ # Depending on your intentions, you may want to close
+ # the parent channel here.
+ self.close()
+
+# Yeah, it's bad when the test code is bigger than the library code.
+
+if __name__ == '__main__':
+
+ import time
+
+ def thread_function (output_file, i, n):
+ print 'entering thread_function'
+ while n:
+ time.sleep (5)
+ output_file.write ('%2d.%2d %s\r\n' % (i, n, output_file))
+ output_file.flush()
+ n = n - 1
+ output_file.close()
+ print 'exiting thread_function'
+
+ class thread_parent (asynchat.async_chat):
+
+ def __init__ (self, conn, addr):
+ self.addr = addr
+ asynchat.async_chat.__init__ (self, conn)
+ self.set_terminator ('\r\n')
+ self.buffer = ''
+ self.count = 0
+
+ def collect_incoming_data (self, data):
+ self.buffer = self.buffer + data
+
+ def found_terminator (self):
+ data, self.buffer = self.buffer, ''
+ n = string.atoi (string.split (data)[0])
+ tc = thread_channel (self, thread_function, self.count, n)
+ self.count = self.count + 1
+ tc.start()
+
+ class thread_server (asyncore.dispatcher):
+
+ def __init__ (self, family=socket.AF_INET, address=('127.0.0.1', 9003)):
+ asyncore.dispatcher.__init__ (self)
+ self.create_socket (family, socket.SOCK_STREAM)
+ self.set_reuse_addr()
+ self.bind (address)
+ self.listen (5)
+
+ def handle_accept (self):
+ conn, addr = self.accept()
+ tp = thread_parent (conn, addr)
+
+ thread_server()
+ #asyncore.loop(1.0, use_poll=1)
+ asyncore.loop ()
Added: supervisor/trunk/src/medusa/thread/thread_handler.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/thread/thread_handler.py Fri May 22 23:51:54 2009
@@ -0,0 +1,362 @@
+# -*- Mode: Python -*-
+
+import re
+import string
+import StringIO
+import sys
+
+import os
+import sys
+import time
+
+import select_trigger
+from medusa import counter
+from medusa import producers
+
+from medusa.default_handler import unquote, get_header
+
+import threading
+
+class request_queue:
+
+ def __init__ (self):
+ self.mon = threading.RLock()
+ self.cv = threading.Condition (self.mon)
+ self.queue = []
+
+ def put (self, item):
+ self.cv.acquire()
+ self.queue.append(item)
+ self.cv.notify()
+ self.cv.release()
+
+ def get(self):
+ self.cv.acquire()
+ while not self.queue:
+ self.cv.wait()
+ result = self.queue.pop(0)
+ self.cv.release()
+ return result
+
+header2env= {
+ 'Content-Length' : 'CONTENT_LENGTH',
+ 'Content-Type' : 'CONTENT_TYPE',
+ 'Referer' : 'HTTP_REFERER',
+ 'User-Agent' : 'HTTP_USER_AGENT',
+ 'Accept' : 'HTTP_ACCEPT',
+ 'Accept-Charset' : 'HTTP_ACCEPT_CHARSET',
+ 'Accept-Language' : 'HTTP_ACCEPT_LANGUAGE',
+ 'Host' : 'HTTP_HOST',
+ 'Connection' : 'CONNECTION_TYPE',
+ 'Authorization' : 'HTTP_AUTHORIZATION',
+ 'Cookie' : 'HTTP_COOKIE',
+ }
+
+# convert keys to lower case for case-insensitive matching
+for (key,value) in header2env.items():
+ del header2env[key]
+ key=string.lower(key)
+ header2env[key]=value
+
+class thread_output_file (select_trigger.trigger_file):
+
+ def close (self):
+ self.trigger_close()
+
+class script_handler:
+
+ def __init__ (self, queue, document_root=""):
+ self.modules = {}
+ self.document_root = document_root
+ self.queue = queue
+
+ def add_module (self, module, *names):
+ if not names:
+ names = ["/%s" % module.__name__]
+ for name in names:
+ self.modules['/'+name] = module
+
+ def match (self, request):
+ uri = request.uri
+
+ i = string.find(uri, "/", 1)
+ if i != -1:
+ uri = uri[:i]
+
+ i = string.find(uri, "?", 1)
+ if i != -1:
+ uri = uri[:i]
+
+ if self.modules.has_key (uri):
+ request.module = self.modules[uri]
+ return 1
+ else:
+ return 0
+
+ def handle_request (self, request):
+
+ [path, params, query, fragment] = request.split_uri()
+
+ while path and path[0] == '/':
+ path = path[1:]
+
+ if '%' in path:
+ path = unquote (path)
+
+ env = {}
+
+ env['REQUEST_URI'] = "/" + path
+ env['REQUEST_METHOD'] = string.upper(request.command)
+ env['SERVER_PORT'] = str(request.channel.server.port)
+ env['SERVER_NAME'] = request.channel.server.server_name
+ env['SERVER_SOFTWARE'] = request['Server']
+ env['DOCUMENT_ROOT'] = self.document_root
+
+ parts = string.split(path, "/")
+
+ # are script_name and path_info ok?
+
+ env['SCRIPT_NAME'] = "/" + parts[0]
+
+ if query and query[0] == "?":
+ query = query[1:]
+
+ env['QUERY_STRING'] = query
+
+ try:
+ path_info = "/" + string.join(parts[1:], "/")
+ except:
+ path_info = ''
+
+ env['PATH_INFO'] = path_info
+ env['GATEWAY_INTERFACE']='CGI/1.1' # what should this really be?
+ env['REMOTE_ADDR'] =request.channel.addr[0]
+ env['REMOTE_HOST'] =request.channel.addr[0] # TODO: connect to resolver
+
+ for header in request.header:
+ [key,value]=string.split(header,": ",1)
+ key=string.lower(key)
+
+ if header2env.has_key(key):
+ if header2env[key]:
+ env[header2env[key]]=value
+ else:
+ key = 'HTTP_' + string.upper(
+ string.join(
+ string.split (key,"-"),
+ "_"
+ )
+ )
+ env[key]=value
+
+ ## remove empty environment variables
+ for key in env.keys():
+ if env[key]=="" or env[key]==None:
+ del env[key]
+
+ try:
+ httphost = env['HTTP_HOST']
+ parts = string.split(httphost,":")
+ env['HTTP_HOST'] = parts[0]
+ except KeyError:
+ pass
+
+ if request.command in ('put', 'post'):
+ # PUT data requires a correct Content-Length: header
+ # (though I bet with http/1.1 we can expect chunked encoding)
+ request.collector = collector (self, request, env)
+ request.channel.set_terminator (None)
+ else:
+ sin = StringIO.StringIO ('')
+ self.continue_request (sin, request, env)
+
+ def continue_request (self, stdin, request, env):
+ stdout = header_scanning_file (
+ request,
+ thread_output_file (request.channel)
+ )
+ self.queue.put (
+ (request.module.main, (env, stdin, stdout))
+ )
+
+HEADER_LINE = re.compile ('([A-Za-z0-9-]+): ([^\r\n]+)')
+
+# A file wrapper that handles the CGI 'Status:' header hack
+# by scanning the output.
+
+class header_scanning_file:
+
+ def __init__ (self, request, file):
+ self.buffer = ''
+ self.request = request
+ self.file = file
+ self.got_header = 0
+ self.bytes_out = counter.counter()
+
+ def write (self, data):
+ if self.got_header:
+ self._write (data)
+ else:
+ # CGI scripts may optionally provide extra headers.
+ #
+ # If they do not, then the output is assumed to be
+ # text/html, with an HTTP reply code of '200 OK'.
+ #
+ # If they do, we need to scan those headers for one in
+ # particular: the 'Status:' header, which will tell us
+ # to use a different HTTP reply code [like '302 Moved']
+ #
+ self.buffer = self.buffer + data
+ lines = string.split (self.buffer, '\n')
+ # ignore the last piece, it is either empty, or a partial line
+ lines = lines[:-1]
+ # look for something un-header-like
+ for i in range(len(lines)):
+ li = lines[i]
+ if (not li) or (HEADER_LINE.match (li) is None):
+ # this is either the header separator, or it
+ # is not a header line.
+ self.got_header = 1
+ h = self.build_header (lines[:i])
+ self._write (h)
+ # rejoin the rest of the data
+ d = string.join (lines[i:], '\n')
+ self._write (d)
+ self.buffer = ''
+ break
+
+ def build_header (self, lines):
+ status = '200 OK'
+ saw_content_type = 0
+ hl = HEADER_LINE
+ for line in lines:
+ mo = hl.match (line)
+ if mo is not None:
+ h = string.lower (mo.group(1))
+ if h == 'status':
+ status = mo.group(2)
+ elif h == 'content-type':
+ saw_content_type = 1
+ lines.insert (0, 'HTTP/1.0 %s' % status)
+ lines.append ('Server: ' + self.request['Server'])
+ lines.append ('Date: ' + self.request['Date'])
+ if not saw_content_type:
+ lines.append ('Content-Type: text/html')
+ lines.append ('Connection: close')
+ return string.join (lines, '\r\n')+'\r\n\r\n'
+
+ def _write (self, data):
+ self.bytes_out.increment (len(data))
+ self.file.write (data)
+
+ def writelines(self, list):
+ self.write (string.join (list, ''))
+
+ def flush(self):
+ pass
+
+ def close (self):
+ if not self.got_header:
+ # managed to slip through our header detectors
+ self._write (self.build_header (['Status: 502', 'Content-Type: text/html']))
+ self._write (
+ '<html><h1>Server Error</h1>\r\n'
+ '<b>Bad Gateway:</b> No Header from CGI Script\r\n'
+ '<pre>Data: %s</pre>'
+ '</html>\r\n' % (repr(self.buffer))
+ )
+ self.request.log (int(self.bytes_out.as_long()))
+ self.file.close()
+ self.request.channel.current_request = None
+
+
+class collector:
+
+ "gathers input for PUT requests"
+
+ def __init__ (self, handler, request, env):
+ self.handler = handler
+ self.env = env
+ self.request = request
+ self.data = StringIO.StringIO()
+
+ # make sure there's a content-length header
+ self.cl = request.get_header ('content-length')
+
+ if not self.cl:
+ request.error (411)
+ return
+ else:
+ self.cl = string.atoi(self.cl)
+
+ def collect_incoming_data (self, data):
+ self.data.write (data)
+ if self.data.tell() >= self.cl:
+ self.data.seek(0)
+
+ h=self.handler
+ r=self.request
+
+ # set the terminator back to the default
+ self.request.channel.set_terminator ('\r\n\r\n')
+ del self.handler
+ del self.request
+
+ h.continue_request (self.data, r, self.env)
+
+
+class request_loop_thread (threading.Thread):
+
+ def __init__ (self, queue):
+ threading.Thread.__init__ (self)
+ self.setDaemon(1)
+ self.queue = queue
+
+ def run (self):
+ while 1:
+ function, (env, stdin, stdout) = self.queue.get()
+ function (env, stdin, stdout)
+ stdout.close()
+
+# ===========================================================================
+# Testing
+# ===========================================================================
+
+if __name__ == '__main__':
+
+ import sys
+
+ if len(sys.argv) < 2:
+ print 'Usage: %s <worker_threads>' % sys.argv[0]
+ else:
+ nthreads = string.atoi (sys.argv[1])
+
+ import asyncore_25 as asyncore
+ from medusa import http_server
+ # create a generic web server
+ hs = http_server.http_server ('', 7080)
+
+ # create a request queue
+ q = request_queue()
+
+ # create a script handler
+ sh = script_handler (q)
+
+ # install the script handler on the web server
+ hs.install_handler (sh)
+
+ # get a couple of CGI modules
+ import test_module
+ import pi_module
+
+ # install the module on the script handler
+ sh.add_module (test_module, 'test')
+ sh.add_module (pi_module, 'pi')
+
+ # fire up the worker threads
+ for i in range (nthreads):
+ rt = request_loop_thread (q)
+ rt.start()
+
+ # start the main event loop
+ asyncore.loop()
Added: supervisor/trunk/src/medusa/unix_user_handler.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/unix_user_handler.py Fri May 22 23:51:54 2009
@@ -0,0 +1,78 @@
+# -*- Mode: Python -*-
+#
+# Author: Sam Rushing <rushing at nightmare.com>
+# Copyright 1996, 1997 by Sam Rushing
+# All Rights Reserved.
+#
+
+RCS_ID = '$Id: unix_user_handler.py,v 1.4 2002/11/25 00:09:23 akuchling Exp $'
+
+# support for `~user/public_html'.
+
+import re
+import string
+import default_handler
+import filesys
+import os
+import pwd
+
+get_header = default_handler.get_header
+
+user_dir = re.compile ('/~([^/]+)(.*)')
+
+class unix_user_handler (default_handler.default_handler):
+
+ def __init__ (self, public_html = 'public_html'):
+ self.public_html = public_html
+ default_handler.default_handler.__init__ (self, None)
+
+ # cache userdir-filesystem objects
+ fs_cache = {}
+
+ def match (self, request):
+ m = user_dir.match (request.uri)
+ return m and (m.end() == len (request.uri))
+
+ def handle_request (self, request):
+ # get the user name
+ m = user_dir.match (request.uri)
+ user = m.group(1)
+ rest = m.group(2)
+
+ # special hack to catch those lazy URL typers
+ if not rest:
+ request['Location'] = '/~%s/' % user
+ request.error (301)
+ return
+
+ # have we already built a userdir fs for this user?
+ if self.fs_cache.has_key (user):
+ fs = self.fs_cache[user]
+ else:
+ # no, well then, let's build one.
+ # first, find out where the user directory is
+ try:
+ info = pwd.getpwnam (user)
+ except KeyError:
+ request.error (404)
+ return
+ ud = info[5] + '/' + self.public_html
+ if os.path.isdir (ud):
+ fs = filesys.os_filesystem (ud)
+ self.fs_cache[user] = fs
+ else:
+ request.error (404)
+ return
+
+ # fake out default_handler
+ self.filesystem = fs
+ # massage the request URI
+ request.uri = '/' + rest
+ return default_handler.default_handler.handle_request (self, request)
+
+ def __repr__ (self):
+ return '<Unix User Directory Handler at %08x [~user/%s, %d filesystems loaded]>' % (
+ id(self),
+ self.public_html,
+ len(self.fs_cache)
+ )
Added: supervisor/trunk/src/medusa/virtual_handler.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/virtual_handler.py Fri May 22 23:51:54 2009
@@ -0,0 +1,59 @@
+# -*- Mode: Python -*-
+
+import socket
+import default_handler
+import re
+
+HOST = re.compile ('Host: ([^:/]+).*', re.IGNORECASE)
+
+get_header = default_handler.get_header
+
+class virtual_handler:
+
+ """HTTP request handler for an HTTP/1.0-style virtual host. Each
+ Virtual host must have a different IP"""
+
+ def __init__ (self, handler, hostname):
+ self.handler = handler
+ self.hostname = hostname
+ try:
+ self.ip = socket.gethostbyname (hostname)
+ except socket.error:
+ raise ValueError, "Virtual Hostname %s does not appear to be registered in the DNS" % hostname
+
+ def match (self, request):
+ if (request.channel.addr[0] == self.ip):
+ return 1
+ else:
+ return 0
+
+ def handle_request (self, request):
+ return self.handler.handle_request (request)
+
+ def __repr__ (self):
+ return '<virtual request handler for %s>' % self.hostname
+
+
+class virtual_handler_with_host:
+
+ """HTTP request handler for HTTP/1.1-style virtual hosts. This
+ matches by checking the value of the 'Host' header in the request.
+ You actually don't _have_ to support HTTP/1.1 to use this, since
+ many browsers now send the 'Host' header. This is a Good Thing."""
+
+ def __init__ (self, handler, hostname):
+ self.handler = handler
+ self.hostname = hostname
+
+ def match (self, request):
+ host = get_header (HOST, request.header)
+ if host == self.hostname:
+ return 1
+ else:
+ return 0
+
+ def handle_request (self, request):
+ return self.handler.handle_request (request)
+
+ def __repr__ (self):
+ return '<virtual request handler for %s>' % self.hostname
Added: supervisor/trunk/src/medusa/xmlrpc_handler.py
==============================================================================
--- (empty file)
+++ supervisor/trunk/src/medusa/xmlrpc_handler.py Fri May 22 23:51:54 2009
@@ -0,0 +1,103 @@
+# -*- Mode: Python -*-
+
+# See http://www.xml-rpc.com/
+# http://www.pythonware.com/products/xmlrpc/
+
+# Based on "xmlrpcserver.py" by Fredrik Lundh (fredrik at pythonware.com)
+
+VERSION = "$Id: xmlrpc_handler.py,v 1.6 2004/04/21 14:09:24 akuchling Exp $"
+
+import http_server
+import xmlrpclib
+
+import string
+import sys
+
+class xmlrpc_handler:
+
+ def match (self, request):
+ # Note: /RPC2 is not required by the spec, so you may override this method.
+ if request.uri[:5] == '/RPC2':
+ return 1
+ else:
+ return 0
+
+ def handle_request (self, request):
+ [path, params, query, fragment] = request.split_uri()
+
+ if request.command == 'POST':
+ request.collector = collector (self, request)
+ else:
+ request.error (400)
+
+ def continue_request (self, data, request):
+ params, method = xmlrpclib.loads (data)
+ try:
+ # generate response
+ try:
+ response = self.call (method, params)
+ if type(response) != type(()):
+ response = (response,)
+ except:
+ # report exception back to server
+ response = xmlrpclib.dumps (
+ xmlrpclib.Fault (1, "%s:%s" % (sys.exc_type, sys.exc_value))
+ )
+ else:
+ response = xmlrpclib.dumps (response, methodresponse=1)
+ except:
+ # internal error, report as HTTP server error
+ request.error (500)
+ else:
+ # got a valid XML RPC response
+ request['Content-Type'] = 'text/xml'
+ request.push (response)
+ request.done()
+
+ def call (self, method, params):
+ # override this method to implement RPC methods
+ raise "NotYetImplemented"
+
+class collector:
+
+ "gathers input for POST and PUT requests"
+
+ def __init__ (self, handler, request):
+
+ self.handler = handler
+ self.request = request
+ self.data = []
+
+ # make sure there's a content-length header
+ cl = request.get_header ('content-length')
+
+ if not cl:
+ request.error (411)
+ else:
+ cl = string.atoi (cl)
+ # using a 'numeric' terminator
+ self.request.channel.set_terminator (cl)
+
+ def collect_incoming_data (self, data):
+ self.data.append(data)
+
+ def found_terminator (self):
+ # set the terminator back to the default
+ self.request.channel.set_terminator ('\r\n\r\n')
+ self.handler.continue_request ("".join(self.data), self.request)
+
+if __name__ == '__main__':
+
+ class rpc_demo (xmlrpc_handler):
+
+ def call (self, method, params):
+ print 'method="%s" params=%s' % (method, params)
+ return "Sure, that works"
+
+ import asyncore_25 as asyncore
+
+ hs = http_server.http_server ('', 8000)
+ rpc = rpc_demo()
+ hs.install_handler (rpc)
+
+ asyncore.loop()
Modified: supervisor/trunk/src/supervisor/http.py
==============================================================================
--- supervisor/trunk/src/supervisor/http.py (original)
+++ supervisor/trunk/src/supervisor/http.py Fri May 22 23:51:54 2009
@@ -12,7 +12,6 @@
#
##############################################################################
-import asyncore
import os
import stat
import time
@@ -22,6 +21,7 @@
import pwd
import urllib
+from medusa import asyncore_25 as asyncore
from medusa import http_date
from medusa import http_server
from medusa import producers
Modified: supervisor/trunk/src/supervisor/http_client.py
==============================================================================
--- supervisor/trunk/src/supervisor/http_client.py (original)
+++ supervisor/trunk/src/supervisor/http_client.py Fri May 22 23:51:54 2009
@@ -2,11 +2,12 @@
import sys
import socket
-import asyncore
-import asynchat
import base64
from urlparse import urlparse
+from medusa import asyncore_25 as aysncore
+from medusa import asynchat_25 as asynchat
+
CR="\x0d"
LF="\x0a"
CRLF=CR+LF
Modified: supervisor/trunk/src/supervisor/options.py
==============================================================================
--- supervisor/trunk/src/supervisor/options.py (original)
+++ supervisor/trunk/src/supervisor/options.py Fri May 22 23:51:54 2009
@@ -13,7 +13,6 @@
##############################################################################
import ConfigParser
-import asyncore
import socket
import getopt
import os
@@ -34,6 +33,8 @@
from fcntl import fcntl
from fcntl import F_SETFL, F_GETFL
+from medusa import asyncore_25 as asyncore
+
from supervisor.datatypes import boolean
from supervisor.datatypes import integer
from supervisor.datatypes import name_to_uid
Modified: supervisor/trunk/src/supervisor/process.py
==============================================================================
--- supervisor/trunk/src/supervisor/process.py (original)
+++ supervisor/trunk/src/supervisor/process.py Fri May 22 23:51:54 2009
@@ -12,7 +12,6 @@
#
##############################################################################
-import asyncore
import os
import sys
import time
@@ -22,6 +21,8 @@
import traceback
import signal
+from medusa import asyncore_25 as asyncore
+
from supervisor.states import ProcessStates
from supervisor.states import SupervisorStates
from supervisor.states import getProcessStateDescription
Modified: supervisor/trunk/src/supervisor/supervisorctl.py
==============================================================================
--- supervisor/trunk/src/supervisor/supervisorctl.py (original)
+++ supervisor/trunk/src/supervisor/supervisorctl.py Fri May 22 23:51:54 2009
@@ -39,11 +39,12 @@
import getpass
import xmlrpclib
import socket
-import asyncore
import errno
import urlparse
import threading
+from medusa import asyncore_25 as asyncore
+
from supervisor.options import ClientOptions
from supervisor.options import split_namespec
from supervisor import xmlrpc
Modified: supervisor/trunk/src/supervisor/supervisord.py
==============================================================================
--- supervisor/trunk/src/supervisor/supervisord.py (original)
+++ supervisor/trunk/src/supervisor/supervisord.py Fri May 22 23:51:54 2009
@@ -46,7 +46,8 @@
import errno
import select
import signal
-import asyncore
+
+from medusa import asyncore_25 as asyncore
from supervisor.options import ServerOptions
from supervisor.options import signame
Modified: supervisor/trunk/src/supervisor/tests/test_http.py
==============================================================================
--- supervisor/trunk/src/supervisor/tests/test_http.py (original)
+++ supervisor/trunk/src/supervisor/tests/test_http.py Fri May 22 23:51:54 2009
@@ -271,14 +271,20 @@
self.assertEqual(authorizer.authorize(('foo', 'password')), True)
def test_authorize_gooduser_badpassword_sha(self):
- import sha
- password = '{SHA}' + sha.new('password').hexdigest()
+ try:
+ from hashlib import sha1
+ except ImportError:
+ from sha import new as sha1
+ password = '{SHA}' + sha1('password').hexdigest()
authorizer = self._makeOne({'foo':password})
self.assertEqual(authorizer.authorize(('foo', 'bar')), False)
def test_authorize_gooduser_goodpassword_sha(self):
- import sha
- password = '{SHA}' + sha.new('password').hexdigest()
+ try:
+ from hashlib import sha1
+ except ImportError:
+ from sha import new as sha1
+ password = '{SHA}' + sha1('password').hexdigest()
authorizer = self._makeOne({'foo':password})
self.assertEqual(authorizer.authorize(('foo', 'password')), True)
Modified: supervisor/trunk/src/supervisor/tests/test_supervisord.py
==============================================================================
--- supervisor/trunk/src/supervisor/tests/test_supervisord.py (original)
+++ supervisor/trunk/src/supervisor/tests/test_supervisord.py Fri May 22 23:51:54 2009
@@ -390,7 +390,7 @@
process = DummyProcess(pconfig)
gconfig = DummyPGroupConfig(options, pconfigs=[pconfig])
pgroup = DummyProcessGroup(gconfig)
- import asyncore
+ from medusa import asyncore_25 as asyncore
exitnow = DummyDispatcher(readable=True, error=asyncore.ExitNow)
pgroup.dispatchers = {6:exitnow}
supervisord.process_groups = {'foo': pgroup}
@@ -410,7 +410,7 @@
L.append(event)
from supervisor import events
events.subscribe(events.SupervisorStateChangeEvent, callback)
- import asyncore
+ from medusa import asyncore_25 as asyncore
options.test = True
self.assertRaises(asyncore.ExitNow, supervisord.runforever)
self.assertTrue(pgroup.all_stopped)
@@ -432,7 +432,7 @@
supervisord.process_groups = {'foo': pgroup}
supervisord.options.mood = 0
supervisord.options.test = True
- import asyncore
+ from medusa import asyncore_25 as asyncore
self.assertRaises(asyncore.ExitNow, supervisord.runforever)
self.assertEqual(pgroup.all_stopped, True)
More information about the Supervisor-checkins
mailing list