Merge pull request #978 from netenglabs/feature/cert-verify

LanutiEmanuele · web-flow · commit 11201d7bd4f6 · 2025-05-07T13:22:34.000+02:00
New feature cert verify
diff --git a/docs/config_file.md b/docs/config_file.md
@@ -53,6 +53,7 @@ coalescer:
 | rest.logsize                 | maximum size of the REST logfile in bytes                                                                                                                                                                                                  | 10000000                         | no                  |
 | rest.log-stdout              | log everything on the standard output instead of a file                                                                                                                                                                                    | False                            | no                  |
 | rest.no-https                | if True, the REST server doesn't use SSL. Highly discouraged in production.                                                                                                                                                                | False                            | no                  |
+| rest.cert-verify             | if False the certificate will not be verified.<br>If True the certificate will be verified.<br>If a path, that will be checked as CA.                                                                                                                                                                                    | True                            | no                  |
 | poller.logging-level         | logging level for the poller.<br/> Choices: INFO, WARNING, ERROR                                                                                                                                                                           | WARNING                          | no                  |
 | poller.logfile               | log file for poller                                                                                                                                                                                                                        | /tmp/sq-poller.log               | no                  |
 | poller.log-stdout            | log on standard output instead of file                                                                                                                                                                                                     | False                            | no                  |
diff --git a/suzieq/engines/rest/engineobj.py b/suzieq/engines/rest/engineobj.py
@@ -1,9 +1,10 @@
-from typing import Type, Dict
+import os
 import urllib
-import requests
-import urllib3
+from typing import Dict, Type
 
 import pandas as pd
+import requests
+import urllib3
 
 from suzieq.engines.base_engine import SqEngineObj
 
@@ -105,7 +106,15 @@ def _get_response(self, verb: str, **kwargs) -> pd.DataFrame:
             f'{query_params}')
 
         # pylint: disable=missing-timeout
-        response = requests.get(url, verify=None)
+        cert_verify = self.ctxt.cfg.get('rest', {}).get('cert-verify', True)
+
+        if isinstance(cert_verify, (str, bool)) is False:
+            raise TypeError('cert_verify must be a boolean or a string')
+        if isinstance(cert_verify, str):
+            if not os.path.exists(os.path.dirname(cert_verify)):
+                raise ValueError('cert_verify path does not exist')
+
+        response = requests.get(url, verify=cert_verify)
         if response.status_code != 200:
             if response.text:
                 msg = response.json().get("detail", str(response.status_code))
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -23,6 +23,8 @@
 from suzieq.shared.schema import Schema
 from suzieq.shared.utils import load_sq_config
 from suzieq.sqobjects import get_sqobject, get_tables
+import socket
+from contextlib import closing
 
 suzieq_cli_path = './suzieq/cli/sq_cli.py'
 suzieq_rest_server_path = './suzieq/restServer/sq_rest_server.py'
@@ -286,3 +288,15 @@ def validate_host_shape(df: pd.DataFrame, ns_dict: dict):
         if ns in df.namespace.unique():
             assert df.query(
                 f'namespace == "{ns}"').hostname.nunique() == ns_dict[ns]
+
+
+def get_free_port() -> int:
+    """Return a free port
+
+    Returns:
+        int: free port
+    """
+    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+        s.bind(('', 0))
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        return s.getsockname()[1]
diff --git a/tests/test_cert_CA/ca-cert.pem b/tests/test_cert_CA/ca-cert.pem
@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIID7TCCAtWgAwIBAgIUe0bXUvllC1Ocgp1/Njwis+rb+DkwDQYJKoZIhvcNAQEL
+BQAwgYQxCzAJBgNVBAYTAklUMQ4wDAYDVQQIDAVJdGFseTENMAsGA1UEBwwEcm9t
+ZTENMAsGA1UECgwEUm9vdDENMAsGA1UECwwEUm9vdDEQMA4GA1UEAwwHMC4wLjAu
+MDEmMCQGCSqGSIb3DQEJARYXc3RhcmR1c3Qucm9vdEBnbWFpbC5jb20wIBcNMjUw
+MjIwMTQ0MzQ4WhgPMzAyNDA2MjMxNDQzNDhaMIGEMQswCQYDVQQGEwJJVDEOMAwG
+A1UECAwFSXRhbHkxDTALBgNVBAcMBHJvbWUxDTALBgNVBAoMBFJvb3QxDTALBgNV
+BAsMBFJvb3QxEDAOBgNVBAMMBzAuMC4wLjAxJjAkBgkqhkiG9w0BCQEWF3N0YXJk
+dXN0LnJvb3RAZ21haWwuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
+AQEApph5CxSlWFkxR94QKfcOgh9OpzXh8HTn+9vZpDv4t3w/pGUGYysoIWx2B7xo
+evzFroGJ6CQEGSG23gyixipP1sd5hmKHbapk+G286X21kOSUYTZ50E+8bZgJkUof
+xYlUQUJf+MmHdlwJXEDDH4I0uuXNrddpNTRmsxH6tqBDNhD5SBulzrUfeLba/Mdw
+H0+He720e/KNhtGgi1z97uBnb85YcCE+uSi+zhWunLHYHfc61W1B+5CkRWLxD2Ag
+V84g0E4O5pCSlwBHwHRiykWbvTIQD1aMVyhSQxbA6U519oIauR7ajbtTWbqJmxe/
+2iJnEPXpSj90LH0lWs926E6IKwIDAQABo1MwUTAdBgNVHQ4EFgQUTr1dzVayV602
+zsi05K4JBQGe2QgwHwYDVR0jBBgwFoAUTr1dzVayV602zsi05K4JBQGe2QgwDwYD
+VR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAYJYFdPw4MlOJRcMNHSNF
+34E5Oh2yRGqUZtrL9F3Acc3UV2BKHRy30w/PCdBLoynaUu9khxmROFFO9mYuolOQ
+UP8LoK1lVg1e7JkrQmA53CRrCSeES8dqZvsOdOdhKRmFKYftzefjbk3A+9J1M3wX
+IiMlq3lAzj2ts8S9HQLYrhDdwKxbfuw+e0HjcG4r4QbjkSA/A+5KLIolyjuPKHcO
+6RADRargWA1kwMxzmNdt0ySHlQ2Lal0fYNdpYlp428eIVRpolsw2SjsC2XZ62Ca+
+6+Xnns2MkTnkbaEhTl6erqsql1lVIT/ramHSuZ+/oCcEvSjvXjp+jUAGi5raESi1
+Zw==
+-----END CERTIFICATE-----
diff --git a/tests/test_cert_CA/server-cert.pem b/tests/test_cert_CA/server-cert.pem
@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDjzCCAncCAQEwDQYJKoZIhvcNAQELBQAwgYQxCzAJBgNVBAYTAklUMQ4wDAYD
+VQQIDAVJdGFseTENMAsGA1UEBwwEcm9tZTENMAsGA1UECgwEUm9vdDENMAsGA1UE
+CwwEUm9vdDEQMA4GA1UEAwwHMC4wLjAuMDEmMCQGCSqGSIb3DQEJARYXc3RhcmR1
+c3Qucm9vdEBnbWFpbC5jb20wIBcNMjUwMjIwMTQ1MDA1WhgPMzAyNDA2MjMxNDUw
+MDVaMIGTMQswCQYDVQQGEwJJVDEOMAwGA1UECAwFSXRhbHkxDTALBgNVBAcMBHJv
+bWUxGDAWBgNVBAoMD1N0YXJkdXN0IHNlcnZlcjEPMA0GA1UECwwGc2VydmVyMRAw
+DgYDVQQDDAcwLjAuMC4wMSgwJgYJKoZIhvcNAQkBFhlzdGFyZHVzdC5zZXJ2ZXJA
+Z21haWwuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4tzYPsL3
+jxpt9zAsHEGvPMZ/3CjklxEXCQKXG2KhJ/RznoW3mBcVRdrGrO2MLgPPM9nZegY1
+H8JJ6ZLvVUGO8E1Yjt3LRX3n3/gS2RYXcSSQogoSdmZciJIyouwgbh5++jvN/DjE
+8fLuRDjrbdPD94rTFkiodenqsLKT9iHhriJwItuT/F4E+w3ZgVAG/5Hi1wLez+Jk
+GdDIY8Hh3D+6FpGmTK3zBEFcYN2QW4oy9rx+IihZfAtvYxpezYh9iRSPxl6PpKY1
+81PSBO84Xvf5MblhN8rnKL72YH4Mxpa0OI/As0liug2517meZNrftc7S5hrBbvEC
+C12RVeoSPHltDQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQCerhsig6eDMNFgRyhh
+WTFwKf0RQ/evcAPmPJ3GBzAlPwvkicj0oa0gKga/0Enot0Y7Ookz5sxi+M1Etyl2
+vd1ch77/xRpUfEzQRu7l5G0UQ+21CaQJBPDlibLG57dqycfQ0xwEajrxf7RAS8EO
+CkasLRps/y7Iq8DeFP5y4Rxs9V/VC0iEQHwqeuqFMgpnS6XpVUPXcMMkmHlUCKcC
+5OTSv0JuyymzYAHIstrm91ZzBrb8yuo9t1riOJqGtqvFKuoPpi5VAH8Twtqymqry
+KAsysWaoSMI45tC0RUJ+817UbMmi8IE56l+ajOAceuy6oXfjv8nU/lxQ8crJba+1
+ck52
+-----END CERTIFICATE-----
diff --git a/tests/test_cert_CA/server-key.pem b/tests/test_cert_CA/server-key.pem
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDi3Ng+wvePGm33
+MCwcQa88xn/cKOSXERcJApcbYqEn9HOehbeYFxVF2sas7YwuA88z2dl6BjUfwknp
+ku9VQY7wTViO3ctFfeff+BLZFhdxJJCiChJ2ZlyIkjKi7CBuHn76O838OMTx8u5E
+OOtt08P3itMWSKh16eqwspP2IeGuInAi25P8XgT7DdmBUAb/keLXAt7P4mQZ0Mhj
+weHcP7oWkaZMrfMEQVxg3ZBbijL2vH4iKFl8C29jGl7NiH2JFI/GXo+kpjXzU9IE
+7zhe9/kxuWE3yucovvZgfgzGlrQ4j8CzSWK6DbnXuZ5k2t+1ztLmGsFu8QILXZFV
+6hI8eW0NAgMBAAECggEAAX+t7+fsemhML1OOCeebnSh+s0Ac6RC6BFbUJhcz5ZHI
+5i/PwBaCqoDenMla07DaiZ3oAN56s3BnnoeGFXdtqP9l0KCq16eUZIwBed8IK4sT
+1WSA/8rjr2Umb4+cAarORE3foInEwnYDFAgW8vDVdhAXefv2HTxzxbi2llCUxhGL
+Dx7HxqrrhjHAhhLRoa4QNbhPn+wXT2ArS+0rKXs2yzVFewPWZ0eR/n2gvyPi3CKS
+puDYzs2ewc8AWW7n6p3ux/DrvP++R9+/UmC+5kjMn5xvQYmPnkUDS24A/BqFPvxh
+NLqHZrWy3bmr8Lttq/He3Ol2K5s+ZrSlRgDi5qb+gQKBgQD8k6VThmcUVU4lXw9D
+1+cSzxVG0lhDNTnV3g6GMPDEuhI93EsIXUH8p4vkM3jc34ZEXStCyFuHfNYXSGWP
+1FHiJQ6NaaA62fF0stv1H+2mQdQqzxkEitEOva9vISnfQzd4t2Alk6WGyNyg2wgQ
+JBn2URMUVkp7YjhQ56kLqyvtgQKBgQDl7/refPGnKAuuEiDWy1sOU8SB9Eoc0OD3
+PPk5SWY8S/eDzlvI/HO2xhto2KcugUzyXCTeUManxHQfDnpAUq1eSLwKzyXpwZWw
+gqrjBRrOsVrkHKfTeH77agmPtXiI+d+8+IqUXpUMkI/bqRHRL4e+MmQj/IKfcIuz
+uacDa2AdjQKBgQCtSwvim9N7gu/j+i26CZcUM5rQhZ9jNVCiKQHkFg4Lm/LKGKwu
+Z/XPSJFVl+8z8/TmUNpOrrMF6aPmQ5jTLwSjWXN7mN4Doubkf5ckvqxKJt5QJNlw
+YWIAcCq+340gDrkvjPldrsiiCow9nSoSEQLzGjsx9+aQcxpagCdexymTgQKBgQCn
+FDLhWj6p7LJYATo1ebync3z1xRHZUHo3jOm3k7sjEzw+XUNajv5yEA+4pr0MUM4d
+yZDMrjs7iseqDXYNqUXqncVtwUnWSmE/yiLsJThuencGDEByrDrw6wMZlo6IUbEe
++iaQWw3I/H5b6cVVkEj9jlYvw/sSadBJfxx5optLvQKBgF1c6DlLvsqJHKP10RDR
+mzjFBG15GYDeN+62EgIrAxFGPGlPMEnm1Tw7Ab69yp/c0gysZ1DmMLp3h94pzmte
+RKioxfV48GjGOaIBmezD+SRze3GH6JudmzDZxAbh1qq3ptWtT1Ih2PQB8g0ED7Ww
+F+QnzMIIw3wJGJhN87EY6Zh9
+-----END PRIVATE KEY-----
diff --git a/tests/unit/engine/rest/test_rest_engine.py b/tests/unit/engine/rest/test_rest_engine.py
@@ -1,17 +1,26 @@
+import os
+import subprocess
 from dataclasses import dataclass
 from itertools import combinations
+from tempfile import mkstemp
+from time import sleep
 from typing import Dict
 from urllib.parse import parse_qs, urlparse
 
 import pytest
+import yaml
 from requests.exceptions import ConnectionError
+
 from suzieq.engines.rest.engineobj import SqRestEngine
+from suzieq.shared.utils import load_sq_config
+from tests.conftest import get_free_port, suzieq_rest_server_path
 
 
 @dataclass
 class SqContextMock:
     """ SqContext rest parameters mock
     """
+    cfg: dict
     rest_api_key: str
     rest_transport: str
     rest_server_ip: str
@@ -70,10 +79,10 @@ def req_error(param: str, exp: str, got: str) -> str:
 
         # check query parameters
         url_query = parse_qs(url.query)
-        for query_param, query_value in url_query.items():
-            assert len(query_value) == 1, \
+        for query_param, query_values in url_query.items():
+            assert len(query_values) == 1, \
                 f'Got more than 1 value for {query_param}'
-            query_value = query_value[0]
+            query_value = query_values[0]
             if query_param == 'access_token':
                 # access_token needs a special validation
                 assert query_value == engine.ctxt.rest_api_key, \
@@ -94,7 +103,7 @@ def req_error(param: str, exp: str, got: str) -> str:
 def test_request_params():
     """This test checks if parameters are set correctly in the request
     """
-    ctxt = SqContextMock('key', 'http', 'rest-ip', 80)
+    ctxt = SqContextMock({}, 'key', 'http', 'rest-ip', 80)
     sqobj = SqObjMock(ctxt, 'default', 'default', 'default',
                       'default', 'default', 'default', 'default')
     engine = SqRestEngine(sqobj)
@@ -109,3 +118,114 @@ def test_request_params():
         for sq_params in combinations(testing_params, n_sq_params):
             req_params = {p: 'override' for p in sq_params}
             validate_args(engine, req_params)
+
+
+@pytest.mark.rest
+@pytest.mark.skipif(not os.environ.get('TEST_SERVER', None),
+                    reason='causes github action hang')
+def test_server_cert():
+    '''Can we can get a valid response with & without certificate'''
+
+    # We need to change the port used to avoid conflicts
+    config = {'data-directory': './tests/data/parquet',
+              'temp-directory': '/tmp/suzieq',
+              'logging-level': 'WARNING',
+              'test_set': 'basic_dual_bgp',  # an extra field for testing
+              'rest': {
+                'rest-certfile': './tests/test_cert_CA/server-cert.pem',
+                'rest-keyfile': './tests/test_cert_CA/server-key.pem',
+                'API_KEY': '496157e6e869ef7f3d6ecb24a6f6d847b224ee4f',
+                'logging-level': 'WARNING',
+                'address': '0.0.0.0',
+                'port': get_free_port(),
+                'no-https': True,
+                'log-stdout': True
+              },
+              'analyzer': {'timezone': 'GMT'},
+              }
+
+    def create_config(config):
+        fd, tmpfname = mkstemp(suffix='.yml')
+        f = os.fdopen(fd, 'w')
+        f.write(yaml.dump(config))
+        f.close()
+
+        cfgfile = tmpfname
+        sqcfg = load_sq_config(config_file=cfgfile)
+
+        print(f'sqcfg: {sqcfg}')
+
+        with open(cfgfile, 'w') as f:
+            f.write(yaml.safe_dump(sqcfg))
+        return sqcfg, cfgfile
+
+    def open_rest_server(cfgfile):
+        server_cmd_args = f'{suzieq_rest_server_path} -c {cfgfile}'.split()
+        # pylint: disable=consider-using-with
+        proc = subprocess.Popen(server_cmd_args)
+        sleep(5)
+        return proc
+
+    def make_get_response_request(sqcfg):
+        ctxt = SqContextMock(
+            rest_api_key=sqcfg['rest']['API_KEY'],
+            rest_transport='http' if sqcfg['rest']['no-https'] is True
+                           else 'https',
+            rest_server_ip=sqcfg['rest']['address'],
+            rest_server_port=sqcfg['rest']['port'],
+            cfg={'rest': sqcfg['rest']})
+
+        sqobj = SqObjMock(ctxt, '', '', 'default',
+                          'default', 'latest', 'device', 'default')
+
+        engine = SqRestEngine(sqobj)
+        try:
+            response = engine._get_response('show')
+            print(f'responsein test: {response}')
+            return 200
+        except Exception:
+            return 400
+
+    def close_session(proc, cfgfile):
+        proc.kill()
+        os.remove(cfgfile)
+
+    # test with http verify None
+    sqcfg, cfgfile = create_config(config)
+    proc = open_rest_server(cfgfile)
+    response = make_get_response_request(sqcfg)
+    assert response == 200
+    close_session(proc, cfgfile)
+
+    # test with https verify None
+    config['rest']['no-https'] = False
+    sqcfg, cfgfile = create_config(config)
+    proc = open_rest_server(cfgfile)
+    response = make_get_response_request(sqcfg)
+    assert response == 400
+    close_session(proc, cfgfile)
+
+    # test with https verify False
+    config['rest']['cert-verify'] = False
+    sqcfg, cfgfile = create_config(config)
+    proc = open_rest_server(cfgfile)
+    response = make_get_response_request(sqcfg)
+    assert response == 200
+    close_session(proc, cfgfile)
+
+    # test with https verify True
+    config['rest']['cert-verify'] = True
+    sqcfg, cfgfile = create_config(config)
+    proc = open_rest_server(cfgfile)
+    response = make_get_response_request(sqcfg)
+    assert response == 400
+    close_session(proc, cfgfile)
+
+    # test with https verify CA
+    config['rest']['cert-verify'] =\
+        './tests/test_cert_CA/ca-cert.pem'
+    sqcfg, cfgfile = create_config(config)
+    proc = open_rest_server(cfgfile)
+    response = make_get_response_request(sqcfg)
+    assert response == 200
+    close_session(proc, cfgfile)
diff --git a/tests/unit/poller/controller/sources/netbox/test_netbox.py b/tests/unit/poller/controller/sources/netbox/test_netbox.py
@@ -14,8 +14,8 @@
     NetboxFaker
 from tests.unit.poller.controller.sources.netbox.netbox_rest_server import \
     NetboxRestApp
-from tests.unit.poller.shared.utils import (get_free_port,
-                                            get_src_sample_config)
+from tests.unit.poller.shared.utils import (get_src_sample_config)
+from tests.conftest import get_free_port
 
 # pylint: disable=redefined-outer-name
 
diff --git a/tests/unit/poller/shared/utils.py b/tests/unit/poller/shared/utils.py
@@ -1,6 +1,4 @@
 import random
-import socket
-from contextlib import closing
 from pathlib import Path
 from typing import Any, Dict, Tuple
 
@@ -119,15 +117,3 @@ def read_yaml_file(path: str) -> Any:
     if not file_path.is_file():
         raise RuntimeError(f'Invalid file to read {path}')
     return yaml.safe_load(open(file_path, 'r'))
-
-
-def get_free_port() -> int:
-    """Return a free port
-
-    Returns:
-        int: free port
-    """
-    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
-        s.bind(('', 0))
-        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-        return s.getsockname()[1]
