Merge branch 'release/4.3.1'

Author: GrahamDumpleton

Makefile.in

@@ -15,7 +15,7 @@
 APXS = @APXS@
 PYTHON = @PYTHON@
 
-DESTDIR =
+DESTDIR = @DESTDIR@
 LIBEXECDIR = @LIBEXECDIR@
 
 CPPFLAGS = @CPPFLAGS@

configure

@@ -585,6 +585,7 @@ ac_unique_file="src/server/mod_wsgi.c"
 ac_subst_vars='LTLIBOBJS
 LIBOBJS
 LIBEXECDIR
+DESTDIR
 LDLIBS
 PYTHON
 OBJEXT
@@ -1932,15 +1933,24 @@ fi
 
 XCODE_PREFIX="/Applications/Xcode.app/Contents/Developer/Toolchains/"
 XCODE_CC="${XCODE_PREFIX}XcodeDefault.xctoolchain/usr/bin/cc"
+XCODE_BIN_PATTERN="${XCODE_PREFIX}.*/usr/bin/"
 
 if test -x "${APXS}"; then
     APXS_CC=`${APXS} -q CC`
     APXS_LIBTOOL=`${APXS} -q LIBTOOL | sed -e "s/ .*$//"`
     if [[ ${APXS_CC} = ${XCODE_PREFIX}* ]]; then
-        cat "${APXS_LIBTOOL}" | sed -e \
-            "s/OSX10.[89].xctoolchain/XcodeDefault.xctoolchain/" > ./libtool
-        cat "${APXS}" | sed -e "s%get_vars(\"CC\")%\"${XCODE_CC}\"%" \
-            -e 's%^my $libtool = .*;%my $libtool = \"./libtool\";%' > ./apxs
+        if test ! -x ${XCODE_CC}; then
+            cat "${APXS_LIBTOOL}" | sed -e \
+                "s%${XCODE_BIN_PATTERN}%/usr/bin/%" > ./libtool
+            cat "${APXS}" | sed -e "s%get_vars(\"CC\")%\"/usr/bin/cc\"%" \
+                -e 's%^my $libtool = .*;%my $libtool = \"./libtool\";%' > ./apxs
+        else
+            cat "${APXS_LIBTOOL}" | sed -e \
+                "s%OSX10.[0-9][0-9]*.xctoolchain%XcodeDefault.xctoolchain%" > ./libtool
+            cat "${APXS}" | sed -e "s%get_vars(\"CC\")%\"${XCODE_CC}\"%" \
+                -e 's%^my $libtool = .*;%my $libtool = \"./libtool\";%' > ./apxs
+        fi
+
         chmod +x ./apxs ./libtool
         APXS=./apxs
     fi
@@ -2950,6 +2960,7 @@ LDLIBS="${LDLIBS} ${LDLIBS1} ${LDLIBS2} ${LDLIBS3}"
 
 
 
+
 LIBEXECDIR="`${APXS} -q LIBEXECDIR`"
 
 

configure.ac

@@ -39,15 +39,24 @@ fi
 
 XCODE_PREFIX="/Applications/Xcode.app/Contents/Developer/Toolchains/"
 XCODE_CC="${XCODE_PREFIX}XcodeDefault.xctoolchain/usr/bin/cc"
+XCODE_BIN_PATTERN="${XCODE_PREFIX}.*/usr/bin/"
 
 if test -x "${APXS}"; then
     APXS_CC=`${APXS} -q CC`
     APXS_LIBTOOL=`${APXS} -q LIBTOOL | sed -e "s/ .*$//"`
     if [[[ ${APXS_CC} = ${XCODE_PREFIX}* ]]]; then
-        cat "${APXS_LIBTOOL}" | sed -e \
-            "s/OSX10.[[89]].xctoolchain/XcodeDefault.xctoolchain/" > ./libtool
-        cat "${APXS}" | sed -e "s%get_vars(\"CC\")%\"${XCODE_CC}\"%" \
-            -e 's%^my $libtool = .*;%my $libtool = \"./libtool\";%' > ./apxs
+        if test ! -x ${XCODE_CC}; then
+            cat "${APXS_LIBTOOL}" | sed -e \
+                "s%${XCODE_BIN_PATTERN}%/usr/bin/%" > ./libtool
+            cat "${APXS}" | sed -e "s%get_vars(\"CC\")%\"/usr/bin/cc\"%" \
+                -e 's%^my $libtool = .*;%my $libtool = \"./libtool\";%' > ./apxs
+        else
+            cat "${APXS_LIBTOOL}" | sed -e \
+                "s%OSX10.[[0-9]][[0-9]]*.xctoolchain%XcodeDefault.xctoolchain%" > ./libtool
+            cat "${APXS}" | sed -e "s%get_vars(\"CC\")%\"${XCODE_CC}\"%" \
+                -e 's%^my $libtool = .*;%my $libtool = \"./libtool\";%' > ./apxs
+        fi
+
         chmod +x ./apxs ./libtool
         APXS=./apxs
     fi
@@ -206,6 +215,7 @@ LDLIBS="${LDLIBS} ${LDLIBS1} ${LDLIBS2} ${LDLIBS3}"
 AC_SUBST(CFLAGS)
 AC_SUBST(LDFLAGS)
 AC_SUBST(LDLIBS)
+AC_SUBST(DESTDIR)
 
 LIBEXECDIR="`${APXS} -q LIBEXECDIR`"
 AC_SUBST(LIBEXECDIR)

docs/release-notes/index.rst

@@ -5,6 +5,7 @@ Release Notes
 .. toctree::
    :maxdepth: 2
 
+   version-4.3.1.rst
    version-4.3.0.rst
 
    version-4.2.8.rst

docs/release-notes/version-4.2.8.rst

@@ -26,3 +26,21 @@ This needs to be inforced as Apache Runtime library has a definition in
 header files which changes sizes from 10.7 to 10.8 and trying to compile
 for compatability back to 10.6 as Python 3.4 tries to enforce, will cause
 mod_wsgi daemon mode processes to crash at runtime.
+
+3. Python 3.3+ pyvenv style virtual environments would not work with
+mod_wsgi via the ``WSGIPythonHome`` directive or the ``home`` option to the
+``WSGIDaemonProcess`` directive. This is because the support in Python for
+pyvenv will not work with embedded systems which set the equivalent of
+``PYTHONHOME`` via the Python C API.
+
+The underlying problem in Python is described in issue:
+
+  * http://bugs.python.org/issue22213
+
+of the Python issue tracer.
+
+To support both normal virtualenv style virtual environments and pyvenv
+style virtual environments, the manner in which virtual environments are
+setup by mod_wsgi has been changed. This has at this point only been done
+on UNIX systems however, as it isn't known at this point whether the same
+trick will work on Windows systems.

docs/release-notes/version-4.3.1.rst

@@ -0,0 +1,56 @@
+=============
+Version 4.3.1
+=============
+
+Version 4.3.1 of mod_wsgi can be obtained from:
+
+  https://github.com/GrahamDumpleton/mod_wsgi/archive/4.3.1.tar.gz
+
+Known Issues
+------------
+
+1. The makefiles for building mod_wsgi on Windows are currently broken and
+need updating. As most new changes relate to mod_wsgi daemon mode, which is
+not supported under Windows, you should keep using the last available
+binary for version 3.X on Windows instead.
+
+Bugs Fixed
+----------
+
+1. The ``install-module`` sub command of ``mod_wsgi-express`` was incorrectly
+trying to install the mod_wsgi ``.so`` file onto itself rather than into
+the Apache modules directory.
+
+2. The workaround for the broken MacOS X Apache build scripts as implemented
+by the ``configure`` script used when building using the traditional make
+command wasn't working correctly for MacOS X 10.10 (Yosemite).
+
+In fixing this issue, the ``configure`` script has been enhanced such that
+it is now no longer to have the whole of the Xcode package installed on
+MacOS X. Instead the minimum required now is the developer command line
+tools. If using Python and you wanted to be able to install Python packages
+which has a source code component you would have already likely installed
+the developer command line tools.
+
+New Features
+------------
+
+1. Added the ``--add-handler`` option to ``mod_wsgi-express`` to allow a
+WSGI application script file to be provided which is to handle any requests
+against static resources in the document root directory matching a specific
+extension type.
+
+2. Added a mechanism to limit the amount of response content that can
+buffered in the Apache child worker processes when proxying back the response
+from a request which had been handled in a mod_wsgi daemon process.
+
+This is to combat a lack of flow control within Apache 2.2 which can cause
+excessive amounts of memory usage as a result of such buffered content.
+This issue in Apache 2.2 was fixed in Apache 2.4, but the new mechanism is
+applied to both versions for consistency.
+
+The default maximum on the amount of buffered content is 65536 bytes. This
+can be increased by using the ``proxy-buffer-size`` option to the
+``WSGIDaemonProcess`` directive or the ``--proxy-buffer-size`` option to
+``mod_wsgi-express``. If using Apache 2.4, its own flow control mechanism
+may override the value in increasing the buffer size.

src/server/__init__.py

@@ -12,6 +12,7 @@
 import imp
 import pwd
 import grp
+import re
 
 try:
     import Queue as queue
@@ -243,6 +244,7 @@ def find_mimetypes():
    send-buffer-size=%(send_buffer_size)s \\
    receive-buffer-size=%(receive_buffer_size)s \\
    header-buffer-size=%(header_buffer_size)s \\
+   proxy-buffer-size=%(proxy_buffer_size)s \\
    server-metrics=%(daemon_server_metrics_flag)s
 </IfDefine>
 <IfDefine !WSGI_MULTIPROCESS>
@@ -265,7 +267,7 @@ def find_mimetypes():
    shutdown-timeout=%(shutdown_timeout)s \\
    send-buffer-size=%(send_buffer_size)s \\
    receive-buffer-size=%(receive_buffer_size)s \\
-   header-buffer-size=%(header_buffer_size)s \\
+   proxy-buffer-size=%(proxy_buffer_size)s \\
    server-metrics=%(daemon_server_metrics_flag)s
 </IfDefine>
 </IfDefine>
@@ -618,6 +620,15 @@ def find_mimetypes():
 PassEnv '%(name)s'
 """
 
+APACHE_HANDLERS_CONFIG = """
+WSGIHandlerScript wsgi-resource '%(server_root)s/resource.wsgi' \\
+    process-group='%(host)s:%(port)s' application-group=%%{GLOBAL}
+"""
+
+APACHE_EXTENSION_CONFIG = """
+AddHandler wsgi-resource %(extension)s
+"""
+
 APACHE_INCLUDE_CONFIG = """
 Include '%(filename)s'
 """
@@ -677,6 +688,13 @@ def generate_apache_config(options):
             for name in options['passenv_variables']:
                 print(APACHE_PASSENV_CONFIG % dict(name=name), file=fp)
 
+        if options['handler_scripts']:
+            print(APACHE_HANDLERS_CONFIG % options, file=fp)
+
+            for extension, script in options['handler_scripts']:
+                print(APACHE_EXTENSION_CONFIG % dict(extension=extension),
+                        file=fp)
+
         if options['include_files']:
             for filename in options['include_files']:
                 filename = os.path.abspath(filename)
@@ -902,6 +920,48 @@ def handle_request(self, environ, start_response):
     def __call__(self, environ, start_response):
         return self.handle_request(environ, start_response)
 
+class ResourceHandler(object):
+
+    def __init__(self, resources):
+        self.resources = {}
+
+        for extension, script in resources:
+            extension_name = re.sub('[^\w]{1}', '_', extension)
+            module_name = '__wsgi_resource%s__' % extension_name
+            module = imp.new_module(module_name)
+            module.__file__ = script
+
+            with open(script, 'r') as fp:
+                code = compile(fp.read(), script, 'exec',
+                        dont_inherit=True)
+                exec(code, module.__dict__)
+
+            sys.modules[module_name] = module
+            self.resources[extension] = module
+
+    def resource_extension(self, resource):
+        return os.path.splitext(resource)[-1]
+
+    def reload_required(self, resource):
+        extension = self.resource_extension(resource)
+        function = getattr(self.resources[extension], 'reload_required', None)
+        if function is not None:
+            return function(environ)
+        return False
+
+    def handle_request(self, environ, start_response):
+        resource = environ['SCRIPT_NAME']
+        extension = self.resource_extension(resource)
+        module = self.resources[extension]
+        function = getattr(module, 'handle_request', None)
+        if function is not None:
+            return function(environ, start_response)
+        function = getattr(module, 'application')
+        return function(environ, start_response)
+
+    def __call__(self, environ, start_response):
+        return self.handle_request(environ, start_response)
+
 WSGI_HANDLER_SCRIPT = """
 import mod_wsgi.server
 
@@ -926,6 +986,17 @@ def __call__(self, environ, start_response):
     mod_wsgi.server.start_reloader()
 """
 
+WSGI_RESOURCE_SCRIPT = """
+import mod_wsgi.server
+
+resources = %(resources)s
+
+handler = mod_wsgi.server.ResourceHandler(resources)
+
+reload_required = handler.reload_required
+handle_request = handler.handle_request
+"""
+
 WSGI_DEFAULT_SCRIPT = """
 CONTENT = b'''
 <html>
@@ -975,6 +1046,11 @@ def generate_wsgi_handler_script(options):
     with open(path, 'w') as fp:
         print(WSGI_HANDLER_SCRIPT % options, file=fp)
 
+    path = os.path.join(options['server_root'], 'resource.wsgi')
+    with open(path, 'w') as fp:
+        print(WSGI_RESOURCE_SCRIPT % dict(resources=repr(
+                options['handler_scripts'])), file=fp)
+
     path = os.path.join(options['server_root'], 'default.wsgi')
     with open(path, 'w') as fp:
         print(WSGI_DEFAULT_SCRIPT % options, file=fp)
@@ -1275,6 +1351,12 @@ def check_percentage(option, opt_str, value, parser):
             metavar='NUMBER', help='Size of buffer used for reading '
             'response headers from daemon processes. Defaults to 0, '
             'indicating internal default of 32768 bytes is used.'),
+    optparse.make_option('--proxy-buffer-size', type='int', default=0,
+            metavar='NUMBER', help='Maximum amount of response content '
+            'that will be allowed to be buffered in the Apache child '
+            'worker process when proxying the response from a daemon '
+            'process. Defaults to 0, indicating internal default of '
+            '65536 bytes is used.'),
 
     optparse.make_option('--reload-on-changes', action='store_true',
             default=False, help='Flag indicating whether worker processes '
@@ -1362,7 +1444,7 @@ def check_percentage(option, opt_str, value, parser):
             default=None, help='Specify a Python script file for '
             'performing group based authorization in conjunction with '
             'a user authentication script.'),
-    optparse.make_option('--auth-group', metavar='SCRIPT-PATH',
+    optparse.make_option('--auth-group', metavar='NAME',
             default='wsgi', help='Specify the group which users should '
             'be a member of when using a group based authorization script. '
             'Defaults to \'wsgi\' as a place holder but should be '
@@ -1450,6 +1532,12 @@ def check_percentage(option, opt_str, value, parser):
             metavar='FILE-PATH', help='Override the path to the mime types '
             'file used by the web server.'),
 
+    optparse.make_option('--add-handler', action='append', nargs=2,
+            dest='handler_scripts', metavar='EXTENSION SCRIPT-PATH',
+            help='Specify a WSGI application to be used as a special '
+            'handler for any resources matched from the document root '
+            'directory with a specific extension type.'),
+
     optparse.make_option('--with-newrelic', action='store_true',
             default=False, help='Flag indicating whether all New Relic '
             'performance monitoring features should be enabled.'),
@@ -1644,6 +1732,14 @@ def _cmd_setup_server(command, args, options):
     else:
         options['daemon_server_metrics_flag'] = 'Off'
 
+    if options['handler_scripts']:
+        handler_scripts = []
+        for extension, script in options['handler_scripts']:
+            if not os.path.isabs(script):
+                script = os.path.abspath(script)
+            handler_scripts.append((extension, script))
+        options['handler_scripts'] = handler_scripts
+
     if options['with_newrelic']:
         options['with_newrelic_agent'] = True
         options['with_newrelic_platform'] = True
@@ -1902,7 +1998,7 @@ def cmd_install_module(params):
         parser.error('Incorrect number of arguments.')
 
     target = os.path.abspath(os.path.join(options.modules_directory,
-            MOD_WSGI_SO))
+            os.path.basename(MOD_WSGI_SO)))
 
     shutil.copyfile(where(), target)