master -> develop

.github/actions/python-setup/action.yml

@@ -12,11 +12,13 @@ runs:
       if: ${{ runner.os == 'Linux' }}
       shell: bash
       run: |
-        sudo apt update && sudo apt-get install -y --no-install-recommends libxml2-dev libxslt-dev python3-dev libgeoip-dev ssdeep libfuzzy-dev p7zip-full innoextract unrar upx
+        sudo apt update && sudo apt-get install -y --no-install-recommends libxml2-dev libxslt-dev python3-dev libgeoip-dev ssdeep libfuzzy-dev 7zip innoextract unrar upx
 
     - name: Install poetry
       shell: bash
       run: PIP_BREAK_SYSTEM_PACKAGES=1 pip install poetry poetry-plugin-export
+    #- name: Python Poetry Action
+    #  uses: abatilo/[email protected]
 
     - name: Set up Python ${{ inputs.python-version }}
       uses: actions/setup-python@v5
@@ -27,4 +29,4 @@ runs:
     - name: Install requirements
       shell: bash
       run: |
-        PIP_BREAK_SYSTEM_PACKAGES=1 poetry install --no-interaction --no-root
+        PIP_BREAK_SYSTEM_PACKAGES=1 poetry install --no-interaction

.github/copilot-instructions.md

@@ -0,0 +1,58 @@
+# Copilot Instructions for CAPEv2
+
+## General Architecture
+- CAPEv2 is an automated malware analysis platform, based on Cuckoo Sandbox, with extensions for dynamic, static, and network analysis.
+- The backend is mainly Python, using SQLAlchemy for the database and Django/DRF for the web API.
+- Main components include:
+  - `lib/cuckoo/core/database.py`: database logic and ORM.
+  - `web/apiv2/views.py`: REST API endpoints (Django REST Framework).
+  - `lib/cuckoo/common/`: shared utilities, configuration, helpers.
+  - `storage/`: analysis results and temporary files.
+- Typical flow: sample upload → DB registration → VM assignment → analysis → result storage → API query.
+
+## Conventions and Patterns
+- Heavy use of SQLAlchemy 2.0 ORM, with explicit sessions and nested transactions (`begin_nested`).
+- Database models (Sample, Task, Machine, etc.) are always managed via `Database` object methods.
+- API endpoints always return a dict with `error`, `data`, and, if applicable, `error_value` keys.
+- Validation and request argument parsing is centralized in helpers (`parse_request_arguments`, etc.).
+- Integrity errors (e.g., duplicates) are handled with `try/except IntegrityError` and recovery of the existing object.
+- Tags are managed as comma-separated strings and normalized before associating to models.
+- Code avoids mutable global variables; configuration is accessed via `Config` objects.
+
+## Developer Workflows
+- No Makefile or standard build scripts; dependency management is usually via `poetry` or `pip`.
+- For testing, use virtual environments and run scripts manually.
+- Typical backend startup is via Django (`manage.py runserver`), and analysis workers are launched separately.
+- Database changes require manual migrations (see Alembic comments in `database.py`).
+
+## Integrations and Dependencies
+- Optional integration with MongoDB and Elasticsearch, controlled by configuration (`reporting.conf`).
+- The system can use different compression tools (zlib, 7zip) depending on config.
+- Sample analysis may invoke external utilities (e.g., Sflock, PE parsers).
+
+## Key Pattern Examples
+- IntegrityError handling example:
+  ```python
+  try:
+      with self.session.begin_nested():
+          self.session.add(sample)
+  except IntegrityError:
+      sample = self.session.scalar(select(Sample).where(Sample.md5 == file_md5))
+  ```
+- API response example:
+  ```python
+  return Response({"error": False, "data": result})
+  ```
+- Tag assignment example:
+  ```python
+  tags = ",".join(set(_tags))
+  ```
+
+## Key Files
+- `lib/cuckoo/core/database.py`: database logic, sample/task registration, machine management.
+- `web/apiv2/views.py`: REST endpoints, validation, high-level business logic.
+- `lib/cuckoo/common/`: utilities, helpers, configuration.
+
+---
+
+If you introduce new endpoints, helpers, or models, follow the validation, error handling, and standard response patterns. See the files above for implementation examples.

.github/workflows/export-requirements.yml

@@ -21,12 +21,12 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Install poetry
-        run: pip install poetry
+        run: pip install poetry poetry-plugin-export --user
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
-          # check-latest: true
+          check-latest: true
           python-version: ${{ matrix.python-version }}
           cache: 'poetry'
 

.github/workflows/python-package-windows.yml

@@ -15,7 +15,7 @@ jobs:
     timeout-minutes: 20
     strategy:
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10"]
 
     steps:
       - name: Check out repository code

.github/workflows/python-package.yml

@@ -11,11 +11,11 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04 # ubuntu-latest
     timeout-minutes: 20
     strategy:
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10"]
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4
@@ -34,32 +34,24 @@ jobs:
 
       - name: Install pyattck
         run: |
-          poetry run pip install pyattck==7.1.2 maco
+          poetry run pip install git+https://github.com/CAPESandbox/pyattck maco
 
       - name: Run Ruff
-        run: poetry run ruff . --line-length 132 --ignore E501,E402
+        run: poetry run ruff check . --output-format=github .
 
       - name: Run unit tests
         run: poetry run python -m pytest --import-mode=append
 
-      - name: See if any parser changed
-        uses: dorny/paths-filter@v3
-        id: changes
-        with:
-          filters: |
-            src:
-              - 'modules/processing/parsers/CAPE/*.py'
-
-      - name: Test parsers only if any parser changed
-        if: steps.changes.outputs.src == 'true'
-        run: poetry run python -m pytest tests_parsers -s --import-mode=append
+      # see the mypy configuration in pyproject.toml
+      - name: Run mypy
+        run: poetry run mypy
 
   format:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     strategy:
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10"]
     if: ${{ github.ref == 'refs/heads/master' }}
 
     steps:
@@ -71,20 +63,15 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Format with black
-        run: poetry run black .
-
-      # to be replaced with ruff
-      - name: Format imports with isort
-        run: poetry run isort .
-
       - name: Commit changes if any
         # Skip this step if being run by nektos/act
         if: ${{ !env.ACT }}
         run: |
           git config user.name "GitHub Actions"
           git config user.email "[email protected]"
           if output=$(git status --porcelain) && [ ! -z "$output" ]; then
+            git pull
+            git add .
             git commit -m "style: Automatic code formatting" -a
             git push
           fi

README.md

@@ -152,11 +152,11 @@ A huge thank you to @D00m3dR4v3n for single-handedly porting CAPE to Python 3.
 ## Installation recommendations and scripts for optimal performance
 * Python3
     * agent.py is tested with python (3.7.2|3.8) x86. __You should use x86 python version inside of the VM!__
-    * host tested with python3 version 3.7, 3.8, 3.10, but newer versions should work too
+    * host tested with python3 version 3.10, 3.12, but newer versions should work too
 
 * __Only rooter should be executed as root__, the rest as __cape__ user. Running as root will mess with permissions.
 1. Become familiar with the [documentation](https://capev2.readthedocs.io/en/latest/) and __do read ALL__ config files inside of `conf` folder!
-2. For best compabitility we strongly suggest installing on [Ubuntu 22.04 LTS](https://ubuntu.com/#download) and using Windows 10 21H2 as target.
+2. For best compabitility we strongly suggest installing on [Ubuntu 24.04 LTS](https://ubuntu.com/#download) and using Windows 10 21H2 as target.
 3. `kvm-qemu.sh` and `cape2.sh` __SHOULD BE__ executed from `tmux` session to prevent any OS problems if ``ssh`` connections breaks.
 4. [KVM](https://github.com/kevoreilly/CAPEv2/blob/master/installer/kvm-qemu.sh) is recommended as the hypervisor.
  * Replace `<username>` with a real pattern.
@@ -228,3 +228,4 @@ If you use CAPEv2 in your work, please cite it as specified in the "Cite this re
 
 ### Docs
 * [ReadTheDocs](https://capev2.readthedocs.io/en/latest/#)
+* [DeepWiki](https://deepwiki.com/kevoreilly/CAPEv2/1-overview) - AI generated, some might be wrong but generally pretty accurate.

agent/agent.py

@@ -9,7 +9,6 @@
 import http.server
 import ipaddress
 import json
-import multiprocessing
 import os
 import platform
 import random
@@ -30,7 +29,7 @@
 from zipfile import ZipFile
 
 try:
-    import re2 as re
+    import re2 as re  # type: ignore
 except ImportError:
     import re
 
@@ -44,7 +43,7 @@
 if sys.maxsize > 2**32 and sys.platform == "win32":
     sys.exit("You should install python3 x86! not x64")
 
-AGENT_VERSION = "0.18"
+AGENT_VERSION = "0.20"
 AGENT_FEATURES = [
     "execpy",
     "execute",
@@ -96,7 +95,7 @@ def _missing_(cls, value):
 AGENT_BROWSER_EXT_PATH = ""
 AGENT_BROWSER_LOCK = Lock()
 ANALYZER_FOLDER = ""
-agent_mutexes = {}
+agent_mutexes: dict = {}
 """Holds handles of mutexes held by the agent."""
 state = {
     "status": Status.INIT,
@@ -177,12 +176,12 @@ def __init__(self):
 
     def run(
         self,
-        host: ipaddress.IPv4Address = "0.0.0.0",
+        host: ipaddress.IPv4Address = ipaddress.IPv4Address("0.0.0.0"),
         port: int = 8000,
-        event: multiprocessing.Event = None,
+        event = None,
     ):
         socketserver.ThreadingTCPServer.allow_reuse_address = True
-        self.s = socketserver.ThreadingTCPServer((host, port), self.handler)
+        self.s = socketserver.ThreadingTCPServer((str(host), port), self.handler)
 
         # tell anyone waiting that they're good to go
         if event:
@@ -226,7 +225,6 @@ def handle(self, obj):
             self.close_connection = True
 
     def shutdown(self):
-
         # BaseServer also features a .shutdown() method, but you can't use
         # that from the same thread as that will deadlock the whole thing.
         if hasattr(self, "s"):
@@ -248,7 +246,7 @@ def __init__(self, status_code=200, **kwargs):
     def init(self):
         pass
 
-    def json(self):
+    def json(self) -> str:
         for valkey in self.values:
             if isinstance(self.values[valkey], bytes):
                 self.values[valkey] = self.values[valkey].decode("utf8", "replace")
@@ -324,8 +322,8 @@ def headers(self, obj):
 
 
 class request:
-    form = {}
-    files = {}
+    form: dict = {}
+    files: dict = {}
     client_ip = None
     client_port = None
     method = None
@@ -334,7 +332,7 @@ class request:
     }
 
 
-app = MiniHTTPServer()
+app: MiniHTTPServer = MiniHTTPServer()
 
 
 def isAdmin():
@@ -378,7 +376,7 @@ def get_subprocess_status():
     """Return the subprocess status."""
     async_subprocess = state.get("async_subprocess")
     message = "Analysis status"
-    exitcode = async_subprocess.exitcode
+    exitcode = async_subprocess.poll()
     if exitcode is None or (sys.platform == "win32" and exitcode == 259):
         # Process is still running.
         state["status"] = Status.RUNNING
@@ -546,7 +544,7 @@ def do_mkdir():
 @app.route("/mktemp", methods=("GET", "POST"))
 def do_mktemp():
     suffix = request.form.get("suffix", "")
-    prefix = request.form.get("prefix", "tmp")
+    prefix = request.form.get("prefix", "")
     dirpath = request.form.get("dirpath")
 
     try:
@@ -562,11 +560,13 @@ def do_mktemp():
 @app.route("/mkdtemp", methods=("GET", "POST"))
 def do_mkdtemp():
     suffix = request.form.get("suffix", "")
-    prefix = request.form.get("prefix", "tmp")
+    prefix = request.form.get("prefix", "")
     dirpath = request.form.get("dirpath")
 
     try:
         dirpath = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=dirpath)
+        if sys.platform == "win32":
+            subprocess.call(["icacls", dirpath, "/inheritance:e", "/grant", "BUILTIN\\Users:(OI)(CI)(RX)"])
     except Exception:
         return json_exception("Error creating temporary directory")
 
@@ -713,9 +713,7 @@ def background_subprocess(command_args, cwd, base64_encode, shell=False):
 
 def spawn(args, cwd, base64_encode, shell=False):
     """Kick off a subprocess in the background."""
-    run_subprocess_args = [args, cwd, base64_encode, shell]
-    proc = multiprocessing.Process(target=background_subprocess, name=f"child process {args[1]}", args=run_subprocess_args)
-    proc.start()
+    proc = subprocess.Popen(args, cwd=cwd, shell=shell)
     state["status"] = Status.RUNNING
     state["description"] = ""
     state["async_subprocess"] = proc
@@ -765,7 +763,7 @@ def do_browser_ext():
     AGENT_BROWSER_LOCK.acquire()
     if not AGENT_BROWSER_EXT_PATH:
         try:
-            ext_tmpdir = tempfile.mkdtemp(prefix="tmp")
+            ext_tmpdir = tempfile.mkdtemp(prefix="")
         except Exception:
             AGENT_BROWSER_LOCK.release()
             return json_exception("Error creating temporary directory")
@@ -799,7 +797,6 @@ def do_kill():
 
 
 if __name__ == "__main__":
-    multiprocessing.set_start_method("spawn")
     parser = argparse.ArgumentParser()
     parser.add_argument("host", nargs="?", default="0.0.0.0")
     parser.add_argument("port", type=int, nargs="?", default=8000)