Mypy (cowrie#1644)

this is release 2.30 many new typing improvements. deprecate python 3.6.

Author: micheloosterhof

.github/workflows/tox.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "pypy-3.7"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "pypy-3.7"]
     steps:
       - uses: actions/checkout@v1
       - name: Set up Python ${{ matrix.python-version }}

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ repos:
     rev: v2.10.0
     hooks:
       - id: pyupgrade
-        args: ["--py36-plus"]
+        args: ["--py37-plus"]
 
   - repo: https://github.com/psf/black
     rev: 20.8b1

CHANGELOG.rst

@@ -1,6 +1,30 @@
 Release Notes
 #############
 
+Release 2.3.0
+*************
+
+* Deprecate Python 3.6
+* Support Python 3.10
+* Dependency updates
+* MISP Output plugin extension
+* add new public keys ECDSAKeys and ed25519 (#1627)
+* fix userdb.example (#1619)
+* cache url submission to virustotal
+* MySQL connector (#1575) - needs new external dependency mysql-connector-python
+* Fix mysql string expansion (#1565)
+* Rewrite CSIRTG output plugin to use new library version
+* Fixed the Slack output to work with the versions 2.x of slackclient
+* fix MySQL error handling
+* fix tar command
+* limit connections to private address ranges
+* Update GreyNoise Output Script to Use Community API (#1524)
+* Implement getopt-style parsing for uname (#1516)
+* Allow SSLv3 connections for wget and curl
+* Support for 301 redirects in wget
+* Malshare update API (#1472)
+* Remove hpfeeds.py infavour of hpfeeds3.py
+
 Release 2.2.0
 *************
 

README.rst

@@ -73,7 +73,7 @@ Requirements
 
 Software required:
 
-* Python 3.6+
+* Python 3.7+
 * python-virtualenv
 
 For Python dependencies, see `requirements.txt <https://github.com/cowrie/cowrie/blob/master/requirements.txt>`_.

mypy.ini

@@ -0,0 +1,31 @@
+[tool.mypy]
+
+namespace_packages = true
+plugins = [ "mypy_zope:plugin" ]
+
+ignore_missing_imports      = true
+warn_unused_configs         = true
+no_implicit_optional        = true
+show_column_numbers         = true
+show_error_codes            = true
+strict_optional             = true
+warn_no_return              = true
+warn_redundant_casts        = true
+warn_return_any             = true
+warn_unreachable            = true
+disallow_incomplete_defs    = true
+disallow_any_unimported     = true
+strict_equality             = true
+disallow_untyped_decorators = true
+disallow_subclassing_any    = true
+
+# These are too strict for us at the moment
+
+check_untyped_defs          = false
+disallow_untyped_defs       = false
+disallow_any_decorated      = false
+disallow_any_explicit       = false
+disallow_any_expr           = false
+disallow_any_generics       = false
+disallow_untyped_calls      = false
+warn_unused_ignores         = false

pyproject.toml

@@ -1,23 +1,23 @@
 Sphinx==4.2.0
-flake8==3.9.2
-importlib_metadata==4.8.1; python_version<'3.8'
+flake8==4.0.1
+importlib_metadata<4; python_version<'3.8'
 mypy-extensions==0.4.3; platform_python_implementation=='CPython'
 mypy-zope==0.3.2; platform_python_implementation=='CPython'
 mypy==0.910; platform_python_implementation=='CPython'
 pathspec==0.9.0
-pipdeptree==2.1.0
+pipdeptree==2.2.0
 pre-commit==2.15.0
 pur==5.4.2
 pyre-check==0.9.6
-pytype==2021.10.4; platform_python_implementation=='CPython' and python_version<'3.10'
+pytype==2021.10.11; platform_python_implementation=='CPython' and python_version<'3.10'
 pyupgrade==2.29.0
-pyyaml==5.4.1
+pyyaml==6.0
 setuptools==58.2.0
 sphinx-copybutton==0.4.0
 sphinx_rtd_theme==1.0.0
 tox==3.24.4
 twistedchecker==0.7.4
-types-python-dateutil==2.8.0
-types-redis==3.5.11
-types-requests==2.25.9
+types-python-dateutil==2.8.2
+types-redis==3.5.15
+types-requests==2.25.11
 yamllint==1.26.3

requirements-dev.txt

@@ -6,7 +6,7 @@ geoip2
 requests==2.26.0
 
 # elasticsearch
-elasticsearch==7.15.0
+elasticsearch==7.15.1
 
 # hpfeeds
 hpfeeds3==0.9.10
@@ -21,18 +21,14 @@ pymongo==3.12.0
 rethinkdb==2.4.8
 
 # s3
-botocore==1.21.58
+botocore==1.21.63
 
 # slack
 slackclient==2.9.3
 
 # influxdb
 influxdb==5.3.1
 
-# kafka
-afkak==21.5.0.post0
-python-snappy==0.6.0
-
 # dblog_xmpp
 wokkel==18.0.0
 

requirements-output.txt

@@ -19,7 +19,7 @@
     package_dir={"": "src"},
     package_data={"": ["*.md"]},
     use_incremental=True,
-    python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4",
+    python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, <4",
     scripts=["bin/fsctl", "bin/asciinema", "bin/cowrie", "bin/createfs", "bin/playlog"],
     classifiers=[
         "Development Status :: 5 - Production/Stable",

setup.py

@@ -1,6 +1,8 @@
 # Copyright (c) 2019 Guilherme Borges <[email protected]>
 # See the COPYRIGHT file for more information
 
+from __future__ import annotations
+
 import os
 import random
 import sys

src/backend_pool/libvirt/backend_service.py

@@ -1,5 +1,6 @@
 # Copyright (c) 2019 Guilherme Borges <[email protected]>
 # See the COPYRIGHT file for more information
+from __future__ import annotations
 import os
 from configparser import NoOptionError
 

src/backend_pool/libvirt/guest_handler.py

@@ -1,5 +1,6 @@
 # Copyright (c) 2019 Guilherme Borges <[email protected]>
 # See the COPYRIGHT file for more information
+from __future__ import annotations
 import os
 
 from twisted.python import log

src/backend_pool/libvirt/network_handler.py

@@ -1,6 +1,8 @@
 # Copyright (c) 2019 Guilherme Borges <[email protected]>
 # See the COPYRIGHT file for more information
 
+from __future__ import annotations
+
 import getpass
 import shutil
 import subprocess
@@ -16,7 +18,6 @@ def create_disk_snapshot(source_img, destination_img):
     # could use `capture_output=True` instead of `stdout` and `stderr` args in Python 3.7
     out = subprocess.run(
         ["qemu-img", "create", "-f", "qcow2", "-b", source_img, destination_img],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
+        capture_output=True,
     )
     return out.returncode == 0

src/backend_pool/libvirt/snapshot_handler.py

@@ -1,6 +1,8 @@
+from __future__ import annotations
 from threading import Lock
 
-from twisted.internet import protocol, reactor
+from twisted.internet import protocol
+from twisted.internet import reactor  # type: ignore
 
 
 class ClientProtocol(protocol.Protocol):

src/backend_pool/nat.py

@@ -1,6 +1,8 @@
 # Copyright (c) 2019 Guilherme Borges <[email protected]>
 # See the COPYRIGHT file for more information
 
+from __future__ import annotations
+
 import struct
 
 from twisted.internet.protocol import Factory, Protocol
@@ -12,8 +14,8 @@
 
 
 class PoolServer(Protocol):
-    def __init__(self, factory):
-        self.factory = factory
+    def __init__(self, factory: PoolServerFactory) -> None:
+        self.factory: PoolServerFactory = factory
         self.local_pool: bool = (
             CowrieConfig.get("proxy", "pool", fallback="local") == "local"
         )
@@ -25,13 +27,13 @@ def __init__(self, factory):
         )
 
         if self.use_nat:
-            self.nat_public_ip = CowrieConfig.get("backend_pool", "nat_public_ip")
+            self.nat_public_ip: str = CowrieConfig.get("backend_pool", "nat_public_ip")
 
-    def dataReceived(self, data):
-        res_op = struct.unpack("!c", bytes([data[0]]))[
+    def dataReceived(self, data: bytes) -> None:
+        res_op: bytes = struct.unpack("!c", bytes([data[0]]))[
             0
         ]  # yes, this needs to be done to extract the op code correctly
-        response = None
+        response: bytes = b""
 
         if res_op == b"i":
             recv = struct.unpack("!II?", data[1:])
@@ -76,10 +78,10 @@ def dataReceived(self, data):
                     guest_id=guest_id,
                 )
 
-                ssh_port = CowrieConfig.getint(
+                ssh_port: int = CowrieConfig.getint(
                     "backend_pool", "guest_ssh_port", fallback=22
                 )
-                telnet_port = CowrieConfig.getint(
+                telnet_port: int = CowrieConfig.getint(
                     "backend_pool", "guest_telnet_port", fallback=23
                 )
 
@@ -162,31 +164,32 @@ def dataReceived(self, data):
             # free this connection and allow VM to be re-used
             self.factory.pool_service.reuse_vm(guest_id)
 
-        if response:
+        if response and self.transport:
             self.transport.write(response)
 
 
 class PoolServerFactory(Factory):
-    def __init__(self):
-        self.initialised = False
+    def __init__(self) -> None:
+        self.initialised: bool = False
 
         # pool handling
-        self.pool_service = None
+        self.pool_service: PoolService
 
         self.tac = None
 
         # NAT service
         self.nat = NATService()
 
-    def startFactory(self):
+    def startFactory(self) -> None:
         # start the pool thread with default configs
         self.pool_service = PoolService(self.nat)
-        self.pool_service.start_pool()
+        if self.pool_service:
+            self.pool_service.start_pool()
 
-    def stopFactory(self):
+    def stopFactory(self) -> None:
         log.msg(eventid="cowrie.backend_pool.server", format="Stopping backend pool...")
-
-        self.pool_service.shutdown_pool()
+        if self.pool_service:
+            self.pool_service.shutdown_pool()
 
     def buildProtocol(self, addr):
         log.msg(

src/backend_pool/pool_server.py

@@ -1,11 +1,13 @@
 # Copyright (c) 2019 Guilherme Borges <[email protected]>
 # See the COPYRIGHT file for more information
+from __future__ import annotations
 
 import os
 import time
 from threading import Lock
 
-from twisted.internet import reactor, threads
+from twisted.internet import reactor  # type: ignore
+from twisted.internet import threads
 from twisted.python import log
 
 import backend_pool.libvirt.backend_service
@@ -58,7 +60,7 @@ def __init__(self, nat_service):
             "backend_pool", "guest_telnet_port", fallback=-1
         )
 
-        self.local_pool: str = (
+        self.local_pool: bool = (
             CowrieConfig.get("proxy", "pool", fallback="local") == "local"
         )
         self.pool_only: bool = CowrieConfig.getboolean(

src/backend_pool/pool_service.py

@@ -1,5 +1,8 @@
+from __future__ import annotations
+
 from twisted.conch.ssh import channel, common, connection, transport, userauth
-from twisted.internet import defer, protocol, reactor
+from twisted.internet import defer, protocol
+from twisted.internet import reactor  # type: ignore
 
 
 # object is added for Python 2.7 compatibility (#1198) - as is super with args