From 94319d0b1459a630bdeeeaeb43ca0fb44eac1831 Mon Sep 17 00:00:00 2001 From: Dobin Rutishauser Date: Tue, 29 Aug 2023 08:15:25 +0200 Subject: [PATCH] fix: iteration/matchidx for incremental scan (refactoring reducer) --- avred.py | 14 +++++++---- filehelper.py | 2 +- minimizer.py | 2 +- plugins/model.py => model/file_model.py | 28 ---------------------- model/plugin_model.py | 31 +++++++++++++++++++++++++ plugins/dotnet/plugin_dotnet.py | 8 ++++--- plugins/office/analyzer_office.py | 3 +-- plugins/office/file_office.py | 2 +- plugins/office/plugin_office.py | 9 +++---- plugins/pe/analyzer_pe.py | 3 +-- plugins/pe/file_pe.py | 2 +- plugins/pe/plugin_pe.py | 8 ++++--- plugins/plain/analyzer_plain.py | 3 +-- plugins/plain/file_plain.py | 2 +- plugins/plain/plugin_plain.py | 9 +++---- reducer.py | 6 ++--- scanning.py | 3 +-- tests/test_model.py | 2 +- tests/test_verifier.py | 25 +++++++++++--------- verifier.py | 2 +- 20 files changed, 88 insertions(+), 76 deletions(-) rename plugins/model.py => model/file_model.py (62%) create mode 100644 model/plugin_model.py diff --git a/avred.py b/avred.py index 08b113c..f582416 100755 --- a/avred.py +++ b/avred.py @@ -24,9 +24,10 @@ from plugins.dotnet.plugin_dotnet import PluginDotNet from plugins.pe.plugin_pe import PluginPe from plugins.office.plugin_office import PluginOffice -from plugins.model import Plugin +from model.plugin_model import Plugin from verifier import verify from minimizer import minimizeMatches +from reducer import Reducer def handler(signum, frame): @@ -132,6 +133,7 @@ def handleFile(filename, args, serverName): logging.error("Unknown filetype, aborting") exit(1) + # scanner is the connection to the AV-oracle scanner = None # load existing outcome if os.path.exists(filenameOutcome): @@ -157,7 +159,6 @@ def handleFile(filename, args, serverName): fileInfo = getFileInfo(file) outcome = Outcome.nullOutcome(fileInfo) - analyzerOptions['scanSpeed'] = ScanSpeed(args.scanspeed) hashCache.load() # scan @@ -191,6 +192,7 @@ def handleFile(filename, args, serverName): isDetected = True # we now know that the file is being detected filePlay = deepcopy(file) # leave original unmodified, apply matches for iterative scanning here iteration = 0 + reducer = Reducer(filePlay, scanner, ScanSpeed(args.scanspeed), iteration=iteration) MAX_ITERATIONS = 6 while isDetected: if iteration > MAX_ITERATIONS: @@ -199,7 +201,7 @@ def handleFile(filename, args, serverName): # get matches logging.info("Scanning for matches...") - matches, scanInfo = plugin.analyzeFile(filePlay, scanner, iteration, analyzerOptions) + matches, scanInfo = plugin.analyzeFile(filePlay, scanner, reducer, analyzerOptions) outcome.matches += matches logging.info("Result: {} matches".format(len(matches))) outcome.scanInfo = scanInfo @@ -213,11 +215,13 @@ def handleFile(filename, args, serverName): logging.info("Still detected on iteration {}, apply {} matches and do again".format( iteration, len(matches) )) + iteration += 1 + # not really necessary to create a new reducer + # but it is tho (reset scan chunk size and similar) + reducer = Reducer(filePlay, scanner, ScanSpeed(args.scanspeed), iteration=iteration) else: break - iteration += 1 - outcome.sections = filePlay.sectionsBag.sections hashCache.save() diff --git a/filehelper.py b/filehelper.py index 6d7cc9d..a9ed54f 100644 --- a/filehelper.py +++ b/filehelper.py @@ -4,7 +4,7 @@ from typing import List, Set, Dict, Tuple, Optional from enum import Enum -from plugins.model import BaseFile +from model.file_model import BaseFile from model.model_base import FileInfo diff --git a/minimizer.py b/minimizer.py index 5ed3672..a262326 100644 --- a/minimizer.py +++ b/minimizer.py @@ -1,7 +1,7 @@ from typing import List from copy import deepcopy -from plugins.model import BaseFile +from model.file_model import BaseFile from model.model_base import Match, ScanSpeed from scanner import Scanner from reducer import Reducer diff --git a/plugins/model.py b/model/file_model.py similarity index 62% rename from plugins/model.py rename to model/file_model.py index 522b3c1..5acaa02 100644 --- a/plugins/model.py +++ b/model/file_model.py @@ -1,12 +1,7 @@ -from dataclasses import dataclass import os import copy -from abc import abstractmethod -from typing import List, Tuple, Set from model.model_data import Match, Data -from model.model_verification import MatchConclusion -from model.model_base import Scanner, OutflankPatch, ScanInfo class BaseFile(): @@ -57,26 +52,3 @@ def loadFromMem(self, data: bytes, filename: str) -> bool: self.filename = filename self.fileData = Data(data) return self.parseFile() - - -class Plugin(): - def __init__(self): - pass - - @abstractmethod - def makeFile(self, filepath: str): - pass - - @abstractmethod - def analyzeFile(self, file: BaseFile, scanner: Scanner, iteration: int = 0, analyzerOptions={}) -> Tuple[Match, ScanInfo]: - pass - - @abstractmethod - def augmentMatches(self, file: BaseFile, matches: List[Match]) -> str: - pass - - @abstractmethod - def outflankFile( - self, filePe: BaseFile, matches: List[Match], matchConclusion: MatchConclusion, scanner: Scanner = None - ) -> List[OutflankPatch]: - pass diff --git a/model/plugin_model.py b/model/plugin_model.py new file mode 100644 index 0000000..722502a --- /dev/null +++ b/model/plugin_model.py @@ -0,0 +1,31 @@ +from abc import abstractmethod +from typing import List, Tuple, Set + +from model.model_data import Match, Data +from model.model_verification import MatchConclusion +from model.model_base import Scanner, OutflankPatch, ScanInfo +from reducer import Reducer +from model.file_model import BaseFile + + +class Plugin(): + def __init__(self): + pass + + @abstractmethod + def makeFile(self, filepath: str): + pass + + @abstractmethod + def analyzeFile(self, file: BaseFile, scanner: Scanner, reducer: Reducer, analyzerOptions={}) -> Tuple[Match, ScanInfo]: + pass + + @abstractmethod + def augmentMatches(self, file: BaseFile, matches: List[Match]) -> str: + pass + + @abstractmethod + def outflankFile( + self, filePe: BaseFile, matches: List[Match], matchConclusion: MatchConclusion, scanner: Scanner = None + ) -> List[OutflankPatch]: + pass diff --git a/plugins/dotnet/plugin_dotnet.py b/plugins/dotnet/plugin_dotnet.py index f38d732..38c18af 100644 --- a/plugins/dotnet/plugin_dotnet.py +++ b/plugins/dotnet/plugin_dotnet.py @@ -3,11 +3,13 @@ from model.model_base import Scanner, OutflankPatch, ScanInfo from model.model_data import Match from model.model_verification import MatchConclusion -from plugins.model import Plugin, BaseFile +from model.plugin_model import Plugin +from model.file_model import BaseFile from plugins.pe.analyzer_pe import analyzeFilePe from plugins.dotnet.augment_dotnet import augmentFileDotnet from plugins.dotnet.outflank_dotnet import outflankDotnet from plugins.pe.file_pe import FilePe +from reducer import Reducer class PluginDotNet(Plugin): @@ -18,9 +20,9 @@ def makeFile(self, filepath: str): return file - def analyzeFile(self, file: BaseFile, scanner: Scanner, iteration: int = 0, analyzerOptions={}) -> Tuple[Match, ScanInfo]: + def analyzeFile(self, file: BaseFile, scanner: Scanner, reducer: Reducer, analyzerOptions={}) -> Tuple[Match, ScanInfo]: # We use the simple PE analyzer - return analyzeFilePe(file, scanner, iteration, analyzerOptions) + return analyzeFilePe(file, scanner, reducer, analyzerOptions) def augmentFile(self, file: BaseFile, matches: List[Match]) -> str: diff --git a/plugins/office/analyzer_office.py b/plugins/office/analyzer_office.py index 10b1b9d..9c7dd72 100644 --- a/plugins/office/analyzer_office.py +++ b/plugins/office/analyzer_office.py @@ -10,10 +10,9 @@ from plugins.office.file_office import FileOffice -def analyzeFileWord(fileOffice: FileOffice, scanner: Scanner, analyzerOptions={}) -> Tuple[Match, ScanInfo]: +def analyzeFileWord(fileOffice: FileOffice, scanner: Scanner, reducer: Reducer, analyzerOptions={}) -> Tuple[Match, ScanInfo]: # Scans a office file given with fileOffice with Scanner scanner. # Returns all matches. - reducer = Reducer(fileOffice, scanner) scanInfo = ScanInfo(scanner.scanner_name, ScanSpeed.Normal) timeStart = time.time() diff --git a/plugins/office/file_office.py b/plugins/office/file_office.py index 7c6c767..6cd2ca8 100644 --- a/plugins/office/file_office.py +++ b/plugins/office/file_office.py @@ -5,7 +5,7 @@ from typing import List from model.model_data import Data -from plugins.model import BaseFile +from model.file_model import BaseFile MAKRO_PATH = 'word/vbaProject.bin' diff --git a/plugins/office/plugin_office.py b/plugins/office/plugin_office.py index 974ec62..1e0d070 100644 --- a/plugins/office/plugin_office.py +++ b/plugins/office/plugin_office.py @@ -1,12 +1,13 @@ -from plugins.model import Plugin, BaseFile +from model.plugin_model import Plugin from model.model_base import Scanner, Match, OutflankPatch, ScanInfo from model.model_data import Match from model.model_verification import MatchConclusion from typing import List, Tuple, Set - +from reducer import Reducer from plugins.office.analyzer_office import analyzeFileWord from plugins.office.augment_office import augmentFileWord from plugins.office.file_office import FileOffice +from model.file_model import BaseFile class PluginOffice(Plugin): @@ -17,9 +18,9 @@ def makeFile(self, filepath: str): return file - def analyzeFile(self, file: BaseFile, scanner: Scanner, iteration: int = 0, analyzerOptions={}) -> Tuple[Match, ScanInfo]: + def analyzeFile(self, file: BaseFile, scanner: Scanner, reducer: Reducer, analyzerOptions={}) -> Tuple[Match, ScanInfo]: # We use the simple PE analyzer - return analyzeFileWord(file, scanner, iteration, analyzerOptions) + return analyzeFileWord(file, scanner, reducer, analyzerOptions) def augmentFile(self, file: BaseFile, matches: List[Match]) -> str: diff --git a/plugins/pe/analyzer_pe.py b/plugins/pe/analyzer_pe.py index 533e78d..ec55aaf 100644 --- a/plugins/pe/analyzer_pe.py +++ b/plugins/pe/analyzer_pe.py @@ -13,14 +13,13 @@ from scanning import scanIsHash -def analyzeFilePe(filePe: FilePe, scanner: Scanner, iteration: int = 0, analyzerOptions={}) -> Tuple[Match, ScanInfo]: +def analyzeFilePe(filePe: FilePe, scanner: Scanner, reducer: Reducer, analyzerOptions={}) -> Tuple[Match, ScanInfo]: """Scans a PE file given with filePe with Scanner scanner. Returns all matches and ScanInfo""" isolate = analyzerOptions.get("isolate", False) scanSpeed = analyzerOptions.get("scanSpeed", ScanSpeed.Normal) scanInfo = ScanInfo(scanner.scanner_name, scanSpeed) # prepare the reducer with the file - reducer = Reducer(filePe, scanner, scanSpeed) timeStart = time.time() matches, scanPipe = scanForMatchesInPe(filePe, scanner, reducer, isolate) scanInfo.scanDuration = round(time.time() - timeStart) diff --git a/plugins/pe/file_pe.py b/plugins/pe/file_pe.py index 67ee282..b542100 100644 --- a/plugins/pe/file_pe.py +++ b/plugins/pe/file_pe.py @@ -4,7 +4,7 @@ import inspect from model.model_code import Section, SectionsBag -from plugins.model import BaseFile +from model.file_model import BaseFile from dotnetfile import DotNetPE from dotnetfile.util import FileLocation diff --git a/plugins/pe/plugin_pe.py b/plugins/pe/plugin_pe.py index a4eadbc..aa5c7b7 100644 --- a/plugins/pe/plugin_pe.py +++ b/plugins/pe/plugin_pe.py @@ -1,4 +1,4 @@ -from plugins.model import Plugin, BaseFile +from model.plugin_model import Plugin from model.model_base import Scanner, OutflankPatch, ScanInfo from model.model_data import Match from model.model_verification import MatchConclusion @@ -9,6 +9,8 @@ from plugins.pe.augment_pe import augmentFilePe from plugins.pe.outflank_pe import outflankPe from plugins.pe.file_pe import FilePe +from reducer import Reducer +from model.file_model import BaseFile class PluginPe(Plugin): @@ -19,9 +21,9 @@ def makeFile(self, filepath: str): return file - def analyzeFile(self, file: BaseFile, scanner: Scanner, iteration: int = 0, analyzerOptions={}) -> Tuple[Match, ScanInfo]: + def analyzeFile(self, file: BaseFile, scanner: Scanner, reducer: Reducer, analyzerOptions={}) -> Tuple[Match, ScanInfo]: # We use the simple PE analyzer - return analyzeFilePe(file, scanner, iteration, analyzerOptions) + return analyzeFilePe(file, scanner, reducer, analyzerOptions) def augmentFile(self, file: BaseFile, matches: List[Match]) -> str: diff --git a/plugins/plain/analyzer_plain.py b/plugins/plain/analyzer_plain.py index 2ecfb36..c94b747 100644 --- a/plugins/plain/analyzer_plain.py +++ b/plugins/plain/analyzer_plain.py @@ -9,8 +9,7 @@ # no PE file, just check its content -def analyzeFilePlain(filePlain: FilePlain, scanner, iteration, analyzerOptions) -> Tuple[Match, ScanInfo]: - reducer = Reducer(filePlain, scanner) +def analyzeFilePlain(filePlain: FilePlain, scanner, reducer, analyzerOptions) -> Tuple[Match, ScanInfo]: scanInfo = ScanInfo(scanner.scanner_name, ScanSpeed.Normal) timeStart = time.time() diff --git a/plugins/plain/file_plain.py b/plugins/plain/file_plain.py index 325fbb9..6272e7b 100644 --- a/plugins/plain/file_plain.py +++ b/plugins/plain/file_plain.py @@ -1,6 +1,6 @@ import logging import os -from plugins.model import BaseFile +from model.file_model import BaseFile class FilePlain(BaseFile): diff --git a/plugins/plain/plugin_plain.py b/plugins/plain/plugin_plain.py index 5bb7ef6..ee914de 100644 --- a/plugins/plain/plugin_plain.py +++ b/plugins/plain/plugin_plain.py @@ -1,9 +1,10 @@ -from plugins.model import Plugin, BaseFile +from model.plugin_model import Plugin from model.model_base import Scanner, OutflankPatch, ScanInfo from model.model_data import Match from model.model_verification import MatchConclusion +from model.file_model import BaseFile from typing import List, Tuple, Set - +from reducer import Reducer from plugins.plain.analyzer_plain import analyzeFilePlain from plugins.plain.file_plain import FilePlain @@ -16,9 +17,9 @@ def makeFile(self, filepath: str): return file - def analyzeFile(self, file: BaseFile, scanner: Scanner, iteration: int = 0, analyzerOptions={}) -> Tuple[Match, ScanInfo]: + def analyzeFile(self, file: BaseFile, scanner: Scanner, reducer: Reducer, analyzerOptions={}) -> Tuple[Match, ScanInfo]: # We use the simple PE analyzer - return analyzeFilePlain(file, scanner, iteration, analyzerOptions) + return analyzeFilePlain(file, scanner, reducer, analyzerOptions) def augmentFile(self, file: BaseFile, matches: List[Match]) -> str: diff --git a/reducer.py b/reducer.py index 44e75b8..ce4816a 100644 --- a/reducer.py +++ b/reducer.py @@ -6,7 +6,7 @@ from model.model_base import Scanner, ScanSpeed from model.model_data import Data, Match -from plugins.model import BaseFile +from model.file_model import BaseFile from myutils import * @@ -16,14 +16,14 @@ class Reducer(): """Reducer will scan data in file with scanner, and return List of matches""" - def __init__(self, file: BaseFile, scanner: Scanner, scanSpeed=ScanSpeed.Normal, iteration: int = 0): + def __init__(self, file: BaseFile, scanner: Scanner, iteration: int = 0, scanSpeed=ScanSpeed.Normal): self.file: BaseFile = file self.scanner: Scanner = scanner self.scanSpeed: ScanSpeed = scanSpeed + self.iteration: int = iteration self.matchesAdded: int = 0 self.chunks_tested: int = 0 - self.iteration: int = iteration self.matchIdx: int = 0 self.minMatchSize: int = 4 diff --git a/scanning.py b/scanning.py index af010ad..35a7e0e 100644 --- a/scanning.py +++ b/scanning.py @@ -1,5 +1,5 @@ from model.model_data import Data -from plugins.model import BaseFile +from model.file_model import BaseFile import logging @@ -23,7 +23,6 @@ def scanIsHash(file: BaseFile, scanner, start=0, size=0) -> bool: d: Data = file.DataCopy() d.patchDataFill(offset, 1) detected = scanner.scannerDetectsBytes(d.getBytes(), file.filename) - logging.info("CheckHash: Offset:{} -> Detected:{}".format(offset, detected)) scanResults.append(detected) # if all modifications result in not-detected, its hash based diff --git a/tests/test_model.py b/tests/test_model.py index 6a93585..114fdd5 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -1,5 +1,5 @@ import unittest -from plugins.model import BaseFile +from model.file_model import BaseFile class ModelTest(unittest.TestCase): diff --git a/tests/test_verifier.py b/tests/test_verifier.py index 0b1467d..1d923f6 100644 --- a/tests/test_verifier.py +++ b/tests/test_verifier.py @@ -5,6 +5,7 @@ from tests.helpers import TestDetection from plugins.pe.analyzer_pe import analyzeFilePe from verifier import verify, getMatchTestsFor +from reducer import Reducer class VerifierTest(unittest.TestCase): @@ -18,21 +19,22 @@ def test_verifyresults(self): detections.append( TestDetection(30823, b"\xff\x98\xb0\xff\xff\xdb\xb1\xff\xff\x68\xb1\xff\xff\x10\xb1\xff") ) # 16 bytes detections.append( TestDetection(29824, b"Unknown error\x00\x00\x00") ) # 16 bytes scanner = ScannerTest(detections) + reducer = Reducer(filePe, scanner) - matches, _ = analyzeFilePe(filePe, scanner) + matches, _ = analyzeFilePe(filePe, scanner, reducer) self.assertEqual(len(matches), 2) verification = verify(filePe, matches, scanner) matchTests = getMatchTestsFor(verification.verifications, TestMatchOrder.ISOLATED, TestMatchModify.MIDDLE8) self.assertTrue(matchTests[0].scanResult == ScanResult.NOT_DETECTED) - #self.assertTrue(matchTests[1].scanResult == ScanResult.NOT_DETECTED) # 15 bytes - self.assertTrue(matchTests[1].scanResult == ScanResult.NOT_SCANNED) + self.assertTrue(matchTests[1].scanResult == ScanResult.NOT_DETECTED) # 15 bytes + #self.assertTrue(matchTests[1].scanResult == ScanResult.NOT_SCANNED) matchTests = getMatchTestsFor(verification.verifications, TestMatchOrder.INCREMENTAL, TestMatchModify.MIDDLE8) self.assertTrue(matchTests[0].scanResult == ScanResult.NOT_DETECTED) - #self.assertTrue(matchTests[1].scanResult == ScanResult.NOT_DETECTED) # 15 bytes - self.assertTrue(matchTests[1].scanResult == ScanResult.NOT_SCANNED) + self.assertTrue(matchTests[1].scanResult == ScanResult.NOT_DETECTED) # 15 bytes + #self.assertTrue(matchTests[1].scanResult == ScanResult.NOT_SCANNED) matchTests = getMatchTestsFor(verification.verifications, TestMatchOrder.INCREMENTAL, TestMatchModify.FULL) self.assertTrue(matchTests[0].scanResult == ScanResult.NOT_DETECTED) @@ -52,8 +54,9 @@ def test_verifyresults_or(self): detections.append( TestDetection(30823, b"\xff\x98\xb0\xff\xff\xdb\xb1\xff\xff\x68\xb1\xff\xff\x10\xb1\xff") ) detections.append( TestDetection(29824, b"Unknown error\x00\x00\x00") ) scanner = ScannerTestOr(detections) - - matches, _ = analyzeFilePe(filePe, scanner) + reducer = Reducer(filePe, scanner) + + matches, _ = analyzeFilePe(filePe, scanner, reducer) for match in matches: print(match) @@ -67,13 +70,13 @@ def test_verifyresults_or(self): matchTests = getMatchTestsFor(verification.verifications, TestMatchOrder.ISOLATED, TestMatchModify.MIDDLE8) self.assertTrue(matchTests[0].scanResult == ScanResult.DETECTED) - #self.assertTrue(matchTests[1].scanResult == ScanResult.DETECTED) # 15 bytes - self.assertTrue(matchTests[1].scanResult == ScanResult.NOT_SCANNED) + self.assertTrue(matchTests[1].scanResult == ScanResult.DETECTED) # 15 bytes + #self.assertTrue(matchTests[1].scanResult == ScanResult.NOT_SCANNED) matchTests = getMatchTestsFor(verification.verifications, TestMatchOrder.INCREMENTAL, TestMatchModify.MIDDLE8) self.assertTrue(matchTests[0].scanResult == ScanResult.DETECTED) - #self.assertTrue(matchTests[1].scanResult == ScanResult.NOT_DETECTED) # 15 bytes - self.assertTrue(matchTests[1].scanResult == ScanResult.NOT_SCANNED) + self.assertTrue(matchTests[1].scanResult == ScanResult.NOT_DETECTED) # 15 bytes + #self.assertTrue(matchTests[1].scanResult == ScanResult.NOT_SCANNED) matchTests = getMatchTestsFor(verification.verifications, TestMatchOrder.DECREMENTAL, TestMatchModify.FULL) self.assertTrue(matchTests[0].scanResult == ScanResult.NOT_DETECTED) diff --git a/verifier.py b/verifier.py index eba7a4b..864905d 100644 --- a/verifier.py +++ b/verifier.py @@ -5,7 +5,7 @@ from model.model_data import Match from model.model_verification import * -from plugins.model import BaseFile +from model.file_model import BaseFile # Middle: 8