Add Monkey Patch for Gevent (#218)

harrryr · web-flow · commit 7f25ce2aacad · 2024-07-05T16:40:12.000-07:00
*Issue #, if available:* Some customers are using gevent in their Python application with opentelemetry, and this causes the application to crash due to `maximum recursion depth exceeded` error. This is because `gevent` uses a specific version of the `ssl` module and runs `monkey.patch_all()` to patch the ssl module to the required version. The `monkey` command patches already existing modules to the required version, which may cause problems if there are other applications already using the module before monkey is ran. Because Opentelemetry is also using the `ssl` module , it creates a conflict and causes the application to crash. The recommended solution is to run `moneky.patch_all()` in opentelemetry beforehand so that such conflicts do not occur. *Description of changes:* The issue experienced by customers is specifically due to the `ssl` module, therefore, we can run monkey.patch_ssl() command to only monkey the `ssl` module and reduce the scope. The distro will check if the customer application has the `gevent` module installed, and if so, run the `monkey.patch_ssl()`. *Test* Built opentelemetry docker image with the change, modified the operator to install opentelemetry on sample python application with the image. Checked that the sample application has opentelemetry running and doesn't crash when APIs are called. Test run: https://github.com/aws-observability/aws-otel-python-instrumentation/actions/runs/9813426624 By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.
diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_instrumentation_patch.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_instrumentation_patch.py
@@ -1,13 +1,18 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 # Modifications Copyright The OpenTelemetry Authors. Licensed under the Apache License 2.0 License.
+import os
 import sys
 from logging import Logger, getLogger
 
 import pkg_resources
 
 from amazon.opentelemetry.distro.patches._resource_detector_patches import _apply_resource_detector_patches
 
+# Env variable for determining whether we want to monkey patch gevent modules. Possible values are 'all', 'none', and
+# comma separated list 'os, thread, time, sys, socket, select, ssl, subprocess, builtins, signal, queue, contextvars'
+AWS_GEVENT_PATCH_MODULES = "AWS_GEVENT_PATCH_MODULES"
+
 _logger: Logger = getLogger(__name__)
 
 
@@ -20,6 +25,37 @@ def apply_instrumentation_patches() -> None:
 
     Where possible, automated testing should be run to catch upstream changes resulting in broken patches
     """
+    if _is_installed("gevent"):
+        try:
+            gevent_patch_module = os.environ.get(AWS_GEVENT_PATCH_MODULES, "all")
+
+            if gevent_patch_module != "none":
+                # pylint: disable=import-outside-toplevel
+                # Delay import to only occur if monkey patch is needed (e.g. gevent is used to run application).
+                from gevent import monkey
+
+                if gevent_patch_module == "all":
+
+                    monkey.patch_all()
+                else:
+                    module_list = [module.strip() for module in gevent_patch_module.split(",")]
+
+                    monkey.patch_all(
+                        socket="socket" in module_list,
+                        time="time" in module_list,
+                        select="select" in module_list,
+                        thread="thread" in module_list,
+                        os="os" in module_list,
+                        ssl="ssl" in module_list,
+                        subprocess="subprocess" in module_list,
+                        sys="sys" in module_list,
+                        builtins="builtins" in module_list,
+                        signal="signal" in module_list,
+                        queue="queue" in module_list,
+                        contextvars="contextvars" in module_list,
+                    )
+        except Exception as exc:  # pylint: disable=broad-except
+            _logger.info("Failed to monkey patch gevent, exception: %s", exc)
 
     if _is_installed("botocore ~= 1.0"):
         # pylint: disable=import-outside-toplevel
diff --git a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_instrumentation_patch.py b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_instrumentation_patch.py
@@ -1,12 +1,17 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+import os
 from typing import Dict
 from unittest import TestCase
 from unittest.mock import MagicMock, patch
 
+import gevent.monkey
 import pkg_resources
 
-from amazon.opentelemetry.distro.patches._instrumentation_patch import apply_instrumentation_patches
+from amazon.opentelemetry.distro.patches._instrumentation_patch import (
+    AWS_GEVENT_PATCH_MODULES,
+    apply_instrumentation_patches,
+)
 from opentelemetry.instrumentation.botocore.extensions import _KNOWN_EXTENSIONS
 from opentelemetry.semconv.trace import SpanAttributes
 
@@ -54,15 +59,37 @@ def test_instrumentation_patch(self):
     def _run_patch_behaviour_tests(self):
         # Test setup
         self.method_patches[GET_DISTRIBUTION_PATCH].return_value = "CorrectDistributionObject"
+        # Test setup to not patch gevent
+        os.environ[AWS_GEVENT_PATCH_MODULES] = "none"
 
         # Validate unpatched upstream behaviour - important to detect upstream changes that may break instrumentation
         self._test_unpatched_botocore_instrumentation()
+        self._test_unpatched_gevent_instrumentation()
 
         # Apply patches
         apply_instrumentation_patches()
 
         # Validate patched upstream behaviour - important to detect downstream changes that may break instrumentation
         self._test_patched_botocore_instrumentation()
+        self._test_unpatched_gevent_instrumentation()
+
+        # Test setup to check whether only these two modules get patched by gevent monkey
+        os.environ[AWS_GEVENT_PATCH_MODULES] = "os, ssl"
+
+        # Apply patches
+        apply_instrumentation_patches()
+
+        # Validate that os and ssl gevent monkey patch modules were patched
+        self._test_patched_gevent_os_ssl_instrumentation()
+
+        # Set the value to 'all' so that all the remaining gevent monkey patch modules are patched
+        os.environ[AWS_GEVENT_PATCH_MODULES] = "all"
+
+        # Apply patches again.
+        apply_instrumentation_patches()
+
+        # Validate that remaining gevent monkey patch modules were patched
+        self._test_patched_gevent_instrumentation()
 
         # Test teardown
         self._reset_mocks()
@@ -93,6 +120,20 @@ def _test_unpatched_botocore_instrumentation(self):
         self.assertFalse("aws.sqs.queue.url" in attributes)
         self.assertFalse("aws.sqs.queue.name" in attributes)
 
+    def _test_unpatched_gevent_instrumentation(self):
+        self.assertFalse(gevent.monkey.is_module_patched("os"), "gevent os module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("thread"), "gevent thread module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("time"), "gevent time module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("sys"), "gevent sys module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("socket"), "gevent socket module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("select"), "gevent select module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("ssl"), "gevent ssl module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("subprocess"), "gevent subprocess module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("builtins"), "gevent builtins module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("signal"), "gevent signal module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("queue"), "gevent queue module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("contextvars"), "gevent contextvars module has been patched")
+
     def _test_patched_botocore_instrumentation(self):
         # Kinesis
         self.assertTrue("kinesis" in _KNOWN_EXTENSIONS)
@@ -115,6 +156,38 @@ def _test_patched_botocore_instrumentation(self):
         self.assertTrue("aws.sqs.queue.name" in sqs_attributes)
         self.assertEqual(sqs_attributes["aws.sqs.queue.name"], _QUEUE_NAME)
 
+    def _test_patched_gevent_os_ssl_instrumentation(self):
+        # Only ssl and os module should have been patched since the environment variable was set to 'os, ssl'
+        self.assertTrue(gevent.monkey.is_module_patched("ssl"), "gevent ssl module has not been patched")
+        self.assertTrue(gevent.monkey.is_module_patched("os"), "gevent os module has not been patched")
+        # Rest should still be unpatched
+        self.assertFalse(gevent.monkey.is_module_patched("thread"), "gevent thread module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("time"), "gevent time module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("sys"), "gevent sys module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("socket"), "gevent socket module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("select"), "gevent select module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("subprocess"), "gevent subprocess module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("builtins"), "gevent builtins module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("signal"), "gevent signal module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("queue"), "gevent queue module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("contextvars"), "gevent contextvars module has been patched")
+
+    def _test_patched_gevent_instrumentation(self):
+        self.assertTrue(gevent.monkey.is_module_patched("os"), "gevent os module has not been patched")
+        self.assertTrue(gevent.monkey.is_module_patched("time"), "gevent time module has not been patched")
+        self.assertTrue(gevent.monkey.is_module_patched("socket"), "gevent socket module has not been patched")
+        self.assertTrue(gevent.monkey.is_module_patched("select"), "gevent select module has not been patched")
+        self.assertTrue(gevent.monkey.is_module_patched("ssl"), "gevent ssl module has not been patched")
+        self.assertTrue(gevent.monkey.is_module_patched("subprocess"), "gevent subprocess module has not been patched")
+        self.assertTrue(gevent.monkey.is_module_patched("signal"), "gevent signal module has not been patched")
+        self.assertTrue(gevent.monkey.is_module_patched("queue"), "gevent queue module has not been patched")
+
+        # Current version of gevent.monkey.patch_all() does not do anything to these modules despite being called
+        self.assertFalse(gevent.monkey.is_module_patched("thread"), "gevent thread module has been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("sys"), "gevent sys module has  been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("builtins"), "gevent builtins module not been patched")
+        self.assertFalse(gevent.monkey.is_module_patched("contextvars"), "gevent contextvars module has been patched")
+
     def _test_botocore_installed_flag(self):
         with patch(
             "amazon.opentelemetry.distro.patches._botocore_patches._apply_botocore_instrumentation_patches"
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -14,4 +14,4 @@ codespell==2.1.0
 requests==2.32.0
 ruamel.yaml==0.17.21
 flaky==3.7.0
-botocore==1.34.67
+botocore==1.34.67
diff --git a/tox.ini b/tox.ini
@@ -30,6 +30,7 @@ commands_pre =
 ; Install common packages for all the tests. These are not needed in all the
 ; cases but it saves a lot of boilerplate in this file.
   test: pip install botocore
+  test: pip install gevent
   test: pip install "opentelemetry-api[test] @ {env:CORE_REPO}#egg=opentelemetry-api&subdirectory=opentelemetry-api"
   test: pip install "opentelemetry-sdk[test] @ {env:CORE_REPO}#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk"
   test: pip install "opentelemetry-instrumentation[test] @ {env:CONTRIB_REPO}#egg=opentelemetry-instrumentation&subdirectory=opentelemetry-instrumentation"
