Feature #65: track metrics of failed tests

.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+    "python.testing.pytestArgs": [
+        "tests"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
+}

AUTHORS

@@ -4,3 +4,4 @@ Contributors include:
  - Raymond Gauthier (jraygauthier) added Python 3.5 support.
  - Kyle Altendorf (altendky) fixed bugs on session teardown
  - Hannes Engelhardt (veritogen) added Bitbucket CI support.
+ - Lucas Haupt (lhpt2) added profiler.py and track failed test support.

docs/sources/changelog.rst

@@ -3,6 +3,8 @@ Changelog
 =========
 
 * :release:`to be discussed`
+* :feature: `#65` Also monitor failed test as default and add flag ``--no-failed`` to turn monitoring failed tests off.
+* :bug: `#79` Fix a bug concerning commandline flag ``--no-monitor`` causing tests that are supposed to fail to pass instead
 * :feature:`#75` Automatically gather CI build information for Bitbucket CI.
 
 * :release:`1.6.6 <2023-05-06>`

docs/sources/configuration.rst

@@ -79,6 +79,14 @@ Disable monitoring
 If you need for some reason to disable the monitoring, pass the *\-\-no-monitor* option.
 
 
+Disable failed tests
+--------------------
+
+By default failing tests are monitored in the database. The database has an additional column that
+indicates if a test passed (boolean value). If you only need to monitor successful tests, pass
+the *\-\-no-failed* option.
+
+
 Describing a run
 ----------------
 

docs/sources/operating.rst

@@ -129,5 +129,7 @@ CPU_USAGE (FLOAT)
     System-wide CPU usage as a percentage (100 % is equivalent to one core).
 MEM_USAGE (FLOAT)
     Maximum resident memory used during the test execution (in megabytes).
+TEST_PASSED (BOOLEAN)
+    Boolean Value indicating if a test passed.
 
 In the local database, these Metrics are stored in table `TEST_METRICS`.

docs/sources/remote.rst

@@ -115,7 +115,8 @@ POST /metrics/
             user_time: float,
             kernel_time: float,
             cpu_usage: float,
-            mem_usage: float
+            mem_usage: float,
+            passed: bool,
         }
 
     **Return Codes**: Must return *201* (*CREATED*) if the **Metrics** has been created

pytest_monitor/handler.py

@@ -6,18 +6,41 @@ def __init__(self, db_path):
         self.__db = db_path
         self.__cnx = sqlite3.connect(self.__db) if db_path else None
         self.prepare()
+        # check if new table column is existent, if not create it
+        self.check_create_test_passed_column()
+
+    def close(self):
+        self.__cnx.close()
+
+    def __del__(self):
+        self.__cnx.close()
+
+    def check_create_test_passed_column(self):
+        cursor = self.__cnx.cursor()
+        # check for test_passed column,
+        # table exists bc call happens after prepare()
+        cursor.execute("PRAGMA table_info(TEST_METRICS)")
+        has_test_column = any(
+            column[1] == "TEST_PASSED" for column in cursor.fetchall()
+        )
+        if not has_test_column:
+            cursor.execute(
+                "ALTER TABLE TEST_METRICS ADD COLUMN TEST_PASSED BOOLEAN DEFAULT TRUE;"
+            )
+            self.__cnx.commit()
 
     def query(self, what, bind_to, many=False):
         cursor = self.__cnx.cursor()
         cursor.execute(what, bind_to)
         return cursor.fetchall() if many else cursor.fetchone()
 
     def insert_session(self, h, run_date, scm_id, description):
-        with self.__cnx:
-            self.__cnx.execute(
-                "insert into TEST_SESSIONS(SESSION_H, RUN_DATE, SCM_ID, RUN_DESCRIPTION)" " values (?,?,?,?)",
-                (h, run_date, scm_id, description),
-            )
+        self.__cnx.execute(
+            "insert into TEST_SESSIONS(SESSION_H, RUN_DATE, SCM_ID, RUN_DESCRIPTION)"
+            " values (?,?,?,?)",
+            (h, run_date, scm_id, description),
+        )
+        self.__cnx.commit()
 
     def insert_metric(
         self,
@@ -35,51 +58,53 @@ def insert_metric(
         kernel_time,
         cpu_usage,
         mem_usage,
+        passed: bool,
     ):
-        with self.__cnx:
-            self.__cnx.execute(
-                "insert into TEST_METRICS(SESSION_H,ENV_H,ITEM_START_TIME,ITEM,"
-                "ITEM_PATH,ITEM_VARIANT,ITEM_FS_LOC,KIND,COMPONENT,TOTAL_TIME,"
-                "USER_TIME,KERNEL_TIME,CPU_USAGE,MEM_USAGE) "
-                "values (?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
-                (
-                    session_id,
-                    env_id,
-                    item_start_date,
-                    item,
-                    item_path,
-                    item_variant,
-                    item_loc,
-                    kind,
-                    component,
-                    total_time,
-                    user_time,
-                    kernel_time,
-                    cpu_usage,
-                    mem_usage,
-                ),
-            )
+        self.__cnx.execute(
+            "insert into TEST_METRICS(SESSION_H,ENV_H,ITEM_START_TIME,ITEM,"
+            "ITEM_PATH,ITEM_VARIANT,ITEM_FS_LOC,KIND,COMPONENT,TOTAL_TIME,"
+            "USER_TIME,KERNEL_TIME,CPU_USAGE,MEM_USAGE,TEST_PASSED) "
+            "values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+            (
+                session_id,
+                env_id,
+                item_start_date,
+                item,
+                item_path,
+                item_variant,
+                item_loc,
+                kind,
+                component,
+                total_time,
+                user_time,
+                kernel_time,
+                cpu_usage,
+                mem_usage,
+                passed,
+            ),
+        )
+        self.__cnx.commit()
 
     def insert_execution_context(self, exc_context):
-        with self.__cnx:
-            self.__cnx.execute(
-                "insert into EXECUTION_CONTEXTS(CPU_COUNT,CPU_FREQUENCY_MHZ,CPU_TYPE,CPU_VENDOR,"
-                "RAM_TOTAL_MB,MACHINE_NODE,MACHINE_TYPE,MACHINE_ARCH,SYSTEM_INFO,"
-                "PYTHON_INFO,ENV_H) values (?,?,?,?,?,?,?,?,?,?,?)",
-                (
-                    exc_context.cpu_count,
-                    exc_context.cpu_frequency,
-                    exc_context.cpu_type,
-                    exc_context.cpu_vendor,
-                    exc_context.ram_total,
-                    exc_context.fqdn,
-                    exc_context.machine,
-                    exc_context.architecture,
-                    exc_context.system_info,
-                    exc_context.python_info,
-                    exc_context.compute_hash(),
-                ),
-            )
+        self.__cnx.execute(
+            "insert into EXECUTION_CONTEXTS(CPU_COUNT,CPU_FREQUENCY_MHZ,CPU_TYPE,CPU_VENDOR,"
+            "RAM_TOTAL_MB,MACHINE_NODE,MACHINE_TYPE,MACHINE_ARCH,SYSTEM_INFO,"
+            "PYTHON_INFO,ENV_H) values (?,?,?,?,?,?,?,?,?,?,?)",
+            (
+                exc_context.cpu_count,
+                exc_context.cpu_frequency,
+                exc_context.cpu_type,
+                exc_context.cpu_vendor,
+                exc_context.ram_total,
+                exc_context.fqdn,
+                exc_context.machine,
+                exc_context.architecture,
+                exc_context.system_info,
+                exc_context.python_info,
+                exc_context.compute_hash(),
+            ),
+        )
+        self.__cnx.commit()
 
     def prepare(self):
         cursor = self.__cnx.cursor()
@@ -109,6 +134,7 @@ def prepare(self):
     KERNEL_TIME float, -- time spent in kernel space
     CPU_USAGE float, -- cpu usage
     MEM_USAGE float, -- Max resident memory used.
+    TEST_PASSED boolean, -- boolean indicating if test passed
     FOREIGN KEY (ENV_H) REFERENCES EXECUTION_CONTEXTS(ENV_H),
     FOREIGN KEY (SESSION_H) REFERENCES TEST_SESSIONS(SESSION_H)
 );"""
@@ -131,3 +157,9 @@ def prepare(self):
 """
         )
         self.__cnx.commit()
+
+    def get_env_id(self, env_hash):
+        query_result = self.query(
+            "SELECT ENV_H FROM EXECUTION_CONTEXTS WHERE ENV_H= ?", (env_hash,)
+        )
+        return query_result[0] if query_result else None