diff --git a/src/gen_fraud_graph/config.py b/src/gen_fraud_graph/config.py index 5c98fd6..235f668 100644 --- a/src/gen_fraud_graph/config.py +++ b/src/gen_fraud_graph/config.py @@ -31,6 +31,9 @@ class Config: scale_factor: float = 1.0 num_fraud_rings: int | None = None fraud_ring_depth_range: tuple[int, int] = (4, 7) + num_structuring_patterns: int | None = None + structuring_smurfs_range: tuple[int, int] = (3, 10) + structuring_amount_range: tuple[float, float] = (8_000.00, 9_900.00) embedding_provider: Literal["fake", "local", "openai"] = "fake" embedding_dim: int = 768 workers: int = 1 @@ -48,3 +51,5 @@ def __post_init__(self) -> None: self.num_transactions = int(90_000_000 * self.scale_factor) if self.num_fraud_rings is None: self.num_fraud_rings = max(10, int(1000 * self.scale_factor)) + if self.num_structuring_patterns is None: + self.num_structuring_patterns = max(10, int(500 * self.scale_factor)) diff --git a/src/gen_fraud_graph/generator.py b/src/gen_fraud_graph/generator.py index b49096d..f11936c 100644 --- a/src/gen_fraud_graph/generator.py +++ b/src/gen_fraud_graph/generator.py @@ -16,7 +16,7 @@ from gen_fraud_graph.config import Config from gen_fraud_graph.embeddings import EmbeddingGenerator from gen_fraud_graph.exporters import get_headers -from gen_fraud_graph.typologies import FraudRingGenerator +from gen_fraud_graph.typologies import FraudRingGenerator, StructuringGenerator # --------------------------------------------------------------------------- # Normal transaction descriptions @@ -218,8 +218,7 @@ def _generate_transactions_chunk( writer.writerows(final_rows) if (i + chunk_count) % 50_000 == 0: print( - f" Worker {worker_id} Batch {batch_id}: " - f"{i + chunk_count} transactions written" + f" Worker {worker_id} Batch {batch_id}: {i + chunk_count} transactions written" ) return f"Worker {worker_id} Batch {batch_id}: Generated {count} transactions" @@ -349,16 +348,17 @@ def _generate_transactions(self) -> None: def _generate_fraud(self) -> None: cfg = self.cfg - print("\n[Phase 3] Generating fraud rings...") + print("\n[Phase 3] Generating fraud patterns...") embedder = EmbeddingGenerator(cfg.embedding_provider, dim=cfg.embedding_dim) - # cfg.num_fraud_rings is resolved to int in Config.__post_init__ + + # --- cyclic money-laundering rings --- assert cfg.num_fraud_rings is not None - fraud_gen = FraudRingGenerator( + ring_gen = FraudRingGenerator( num_rings=cfg.num_fraud_rings, depth_range=cfg.fraud_ring_depth_range, ) - n_tx, _ = fraud_gen.generate( + n_ring_tx, next_tx_id = ring_gen.generate( max_account_id=cfg.num_accounts, start_tx_id=cfg.num_transactions, embedder=embedder, @@ -366,4 +366,24 @@ def _generate_fraud(self) -> None: fmt=cfg.output_format, compress=cfg.compress, ) - print(f" Injected {n_tx:,} fraud transactions across {cfg.num_fraud_rings:,} rings") + print(f" Injected {n_ring_tx:,} ring transactions across {cfg.num_fraud_rings:,} rings") + + # --- structuring / smurfing patterns --- + assert cfg.num_structuring_patterns is not None + struct_gen = StructuringGenerator( + num_patterns=cfg.num_structuring_patterns, + smurfs_range=cfg.structuring_smurfs_range, + amount_range=cfg.structuring_amount_range, + ) + n_struct_tx, _ = struct_gen.generate( + max_account_id=cfg.num_accounts, + start_tx_id=next_tx_id, + embedder=embedder, + output_dir=cfg.output_dir, + fmt=cfg.output_format, + compress=cfg.compress, + ) + print( + f" Injected {n_struct_tx:,} structuring transactions " + f"across {cfg.num_structuring_patterns:,} patterns" + ) diff --git a/src/gen_fraud_graph/typologies.py b/src/gen_fraud_graph/typologies.py index c009296..363a27e 100644 --- a/src/gen_fraud_graph/typologies.py +++ b/src/gen_fraud_graph/typologies.py @@ -11,7 +11,7 @@ import numpy as np from gen_fraud_graph.embeddings import EmbeddingGenerator -from gen_fraud_graph.exporters import get_headers, write_output +from gen_fraud_graph.exporters import append_csv, get_headers, write_output # --------------------------------------------------------------------------- # Suspicious transaction descriptions used across typologies @@ -28,6 +28,16 @@ "high-value cross-border wire", ] +# Description specific to structuring/smurfing patterns. +STRUCTURING_DESCRIPTIONS: list[str] = [ + "cash deposit below reporting threshold", + "multiple small deposits same day", + "structured payment just under limit", + "smurfing deposit via branch teller", + "incremental cash deposit sub-threshold", + "repeated near limit ATM deposit", + "fragmented transfer to evade detection", +] # --------------------------------------------------------------------------- # Fraud ring generator (cyclic money-laundering patterns) @@ -152,3 +162,148 @@ def generate( write_output(file_cases, headers_cases, case_rows, compress=compress) return len(tx_rows), current_tx_id + + +@dataclass +class StructuringGenerator: + """Generate structuring (smurfing) fraud patterns. + In a structuring scheme a single coordinator account receives funds from + several "smurf" accounts, each sending amounts just below the BSA/FinCEN + Cash Transaction Report (CTR) threshold of $10,000.00. The coordinator + aggregates these deposits to move a larger sum without triggering a single + reportable event. + + Graph shape:: + + smurf_0 -> coordinator + smurf_1 -> coordinator + ... + smurf_N -> coordinator + Multiple sources converge on one node. + This is a structurally distinct from the cyclic ring produced by + :class:'FraudRingGenerator' and exercises different subgraph-detection + algorithms. + + Args: + num_patterns: How many structuring patterns to create. + smurfs_range: ''(min_smurfs, mac_smurfs)'' - number of feeder + accounts per pattern. Mirrors the real world practice of using + 3-10 smurfs to stay inconspicuous. + amount_range: ''(min_amount, max_amount)'' - each smurf transfer is + drawn uniformly form this range. Defaults to $8_000-$9_900, + deliberately sub-threshold. + """ + + num_patterns: int = 100 + smurfs_range: tuple[int, int] = (3, 10) + amount_range: tuple[float, float] = (8_000.00, 9_900.00) + _descriptions: list[str] = field(default_factory=lambda: STRUCTURING_DESCRIPTIONS) + + def generate( + self, + max_account_id: int, + start_tx_id: int, + embedder: EmbeddingGenerator, + output_dir: str, + fmt: str = "csv", + compress: bool = False, + ) -> tuple[int, int]: + """Generate structuring patterns and append to fraud output files. + + Output files are appended to the same ``fraud/`` directory used by + :class:`FraudRingGenerator` so a single pipeline run can inject both + typologies into one dataset. + + Args: + max_account_id: Upper bound of account IDs already generated. + start_tx_id: First transaction ID to use (must not collide with + IDs already written by the ring generator or normal txs). + embedder: Embedding generator instance — same one used by the + ring generator so embedding provenance is consistent. + output_dir: Root output directory. + fmt: ``"csv"`` or ``"neptune"``. + compress: ZIP the output CSV files. + + Returns: + ``(num_fraud_transactions, next_tx_id)`` + """ + import os + + from tqdm import tqdm + + fraud_dir = os.path.join(output_dir, "fraud") + os.makedirs(fraud_dir, exist_ok=True) + + headers_tx = get_headers("transaction", fmt) # type: ignore[arg-type] + headers_cases = [ + "pattern_id", + "start_acc_id", + "pattern_type", + "depth", + "involved_accounts", + ] + + tx_rows: list[list] = [] + case_rows: list[list] = [] + current_tx_id = start_tx_id + + for pattern_id in tqdm(range(self.num_patterns), desc="Generating structuring patterns"): + min_s, max_s = self.smurfs_range + num_smurfs = random.randint(min_s, max_s) + + # The coordinator sits at a random offset; smurfs occupy the + # num_smurfs slots immediately after it. We need num_smurfs + 1 + # consecutive IDs so we guard against tiny account pools. + needed = num_smurfs + 1 + if max_account_id < needed: + coordinator_idx = 0 + else: + coordinator_idx = random.randint(0, max_account_id - needed) + + coordinator = f"acc_{coordinator_idx}" + smurfs = [f"acc_{coordinator_idx + 1 + i}" for i in range(num_smurfs)] + involved = "|".join([coordinator] + smurfs) + + batch_texts: list[str] = [] + batch_rows: list[list] = [] + + for smurf in smurfs: + amount = round(random.uniform(*self.amount_range), 2) + desc = random.choice(self._descriptions) + batch_texts.append(desc) + + row: list = [f"tx_{current_tx_id}", smurf, coordinator] + if fmt == "neptune": + row.append("TRANSFER") + row.extend([amount, "2024-01-01T12:00:00", desc]) + batch_rows.append(row) + current_tx_id += 1 + + embeddings = embedder.generate(batch_texts) + + for idx, r in enumerate(batch_rows): + if fmt == "neptune": + tx_rows.append(r) + else: + vec = embeddings[idx] + if isinstance(vec, np.ndarray): + vec = vec.tolist() + tx_rows.append(r + ["|".join(map(str, vec))]) + + case_rows.append( + [ + f"struct_{pattern_id}", + coordinator, + "structuring", + num_smurfs, # depth = number of feeder hops + involved, + ] + ) + + # Append to the same fraud files so both typologies land in one CSV. + file_tx = os.path.join(fraud_dir, "transactions_fraud") + file_cases = os.path.join(fraud_dir, "fraud_cases") + append_csv(file_tx + ".csv", headers_tx, tx_rows) + append_csv(file_cases + ".csv", headers_cases, case_rows) + + return len(tx_rows), current_tx_id diff --git a/src/gen_fraud_graph/verify.py b/src/gen_fraud_graph/verify.py index c40c817..2c08861 100644 --- a/src/gen_fraud_graph/verify.py +++ b/src/gen_fraud_graph/verify.py @@ -47,19 +47,28 @@ def verify_fraud_patterns( reader = csv.DictReader(fh) for row in reader: pattern_id = row["pattern_id"] + pattern_type = row.get("pattern_type", "cycle") accounts = row["involved_accounts"].split("|") depth = int(row["depth"]) - # Check that the cycle edges exist - for k in range(depth): - src = accounts[k] - dst = accounts[(k + 1) % depth] - if dst not in edges.get(src, set()): - print(f" FAIL: {pattern_id} — missing edge {src} -> {dst}") - all_valid = False - break + if pattern_type == "cycle": + for k in range(depth): + src = accounts[k] + dst = accounts[(k + 1) % depth] + if dst not in edges.get(src, set()): + print(f" FAIL: {pattern_id} — missing edge {src} -> {dst}") + all_valid = False + break + elif pattern_type == "structuring": + coordinator = accounts[0] + smurfs = accounts[1:] + for smurf in smurfs: + if coordinator not in edges.get(smurf, set()): + print(f" FAIL: {pattern_id} — missing edge {smurf} -> {coordinator}") + all_valid = False + break else: - continue + print(f" WARN: {pattern_id} — unknown pattern_type '{pattern_type}', skipping") if all_valid: print("All fraud patterns verified successfully.") diff --git a/tests/test_generator.py b/tests/test_generator.py index 90bf74b..bb39bec 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -7,14 +7,184 @@ import csv import os +import shutil +import tempfile +import pytest + +from gen_fraud_graph.config import Config +from gen_fraud_graph.embeddings import EmbeddingGenerator +from gen_fraud_graph.exporters import get_headers, write_output from gen_fraud_graph.generator import ( FraudGraphGenerator, _generate_accounts_chunk, _generate_transactions_chunk, _split_workload, ) +from gen_fraud_graph.typologies import FraudRingGenerator, StructuringGenerator +from gen_fraud_graph.verify import verify_fraud_patterns + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def tmp_dir(): + """Create a temporary directory that is cleaned up after the test.""" + d = tempfile.mkdtemp(prefix="gen_fraud_graph_test_") + yield d + shutil.rmtree(d, ignore_errors=True) + + +@pytest.fixture() +def small_config(tmp_dir): + """A tiny config suitable for fast unit tests.""" + return Config( + scale_factor=0.0001, + embedding_provider="fake", + workers=1, + batches_per_worker=1, + output_dir=tmp_dir, + ) + + +# --------------------------------------------------------------------------- +# Config tests +# --------------------------------------------------------------------------- + + +class TestConfig: + def test_defaults(self): + cfg = Config() + assert cfg.num_accounts == 10_000_000 + assert cfg.num_transactions == 90_000_000 + assert cfg.num_fraud_rings == 1000 + + def test_scale_factor(self): + cfg = Config(scale_factor=0.01) + assert cfg.num_accounts == 100_000 + assert cfg.num_transactions == 900_000 + assert cfg.num_fraud_rings == max(10, int(1000 * 0.01)) + + def test_explicit_fraud_rings(self): + cfg = Config(num_fraud_rings=42) + assert cfg.num_fraud_rings == 42 + + def test_tiny_scale(self): + cfg = Config(scale_factor=0.0001) + assert cfg.num_accounts == 1_000 + assert cfg.num_transactions == 9_000 + assert cfg.num_fraud_rings >= 10 + + +# --------------------------------------------------------------------------- +# Embedding tests +# --------------------------------------------------------------------------- + + +class TestEmbeddings: + def test_fake_provider_shape(self): + emb = EmbeddingGenerator("fake", dim=128) + result = emb.generate(["hello", "world"]) + assert result.shape == (2, 128) + + def test_fake_provider_empty(self): + emb = EmbeddingGenerator("fake") + result = emb.generate([]) + assert result == [] + + def test_fake_provider_deterministic_shape(self): + emb = EmbeddingGenerator("fake", dim=768) + texts = [f"text_{i}" for i in range(100)] + result = emb.generate(texts) + assert result.shape == (100, 768) + + +# --------------------------------------------------------------------------- +# Exporter tests +# --------------------------------------------------------------------------- + + +class TestExporters: + def test_csv_headers_account(self): + h = get_headers("account", "csv") + assert "account_id" in h + assert "balance" in h + + def test_csv_headers_transaction(self): + h = get_headers("transaction", "csv") + assert "tx_id" in h + assert "src_id" in h + assert "dst_id" in h + + def test_neptune_headers_account(self): + h = get_headers("account", "neptune") + assert "~id" in h + assert "~label" in h + + def test_neptune_headers_transaction(self): + h = get_headers("transaction", "neptune") + assert "~from" in h + assert "~to" in h + + def test_write_output_csv(self, tmp_dir): + path = os.path.join(tmp_dir, "test") + write_output(path, ["a", "b"], [[1, 2], [3, 4]]) + assert os.path.exists(f"{path}.csv") + + with open(f"{path}.csv") as fh: + reader = csv.reader(fh) + rows = list(reader) + assert rows[0] == ["a", "b"] + assert len(rows) == 3 + def test_write_output_compressed(self, tmp_dir): + path = os.path.join(tmp_dir, "test_zip") + write_output(path, ["x"], [[1], [2]], compress=True) + assert os.path.exists(f"{path}.csv.zip") + + +# --------------------------------------------------------------------------- +# Fraud typology tests +# --------------------------------------------------------------------------- + + +class TestFraudRings: + def test_generate_creates_files(self, tmp_dir): + emb = EmbeddingGenerator("fake", dim=32) + gen = FraudRingGenerator(num_rings=5, depth_range=(3, 5)) + n_tx, next_id = gen.generate( + max_account_id=1000, + start_tx_id=0, + embedder=emb, + output_dir=tmp_dir, + fmt="csv", + ) + assert n_tx > 0 + assert os.path.exists(os.path.join(tmp_dir, "fraud", "transactions_fraud.csv")) + assert os.path.exists(os.path.join(tmp_dir, "fraud", "fraud_cases.csv")) + + def test_fraud_cases_have_correct_columns(self, tmp_dir): + emb = EmbeddingGenerator("fake", dim=32) + gen = FraudRingGenerator(num_rings=3, depth_range=(4, 4)) + gen.generate( + max_account_id=100, + start_tx_id=0, + embedder=emb, + output_dir=tmp_dir, + ) + with open(os.path.join(tmp_dir, "fraud", "fraud_cases.csv")) as fh: + reader = csv.DictReader(fh) + rows = list(reader) + assert len(rows) == 3 + assert "pattern_id" in rows[0] + assert "involved_accounts" in rows[0] + + +# --------------------------------------------------------------------------- +# End-to-end generator tests +# --------------------------------------------------------------------------- class TestWorkloadPlanning: def test_split_workload_distributes_remainder(self): @@ -103,3 +273,163 @@ def test_skip_accounts(self, small_config): assert not os.path.isdir(os.path.join(out, "accounts")) assert os.path.isdir(os.path.join(out, "transactions")) assert os.path.isdir(os.path.join(out, "fraud")) + + +# --------------------------------------------------------------------------- +# Verify tests +# --------------------------------------------------------------------------- + + +class TestVerify: + def test_verify_valid_patterns(self, small_config): + gen = FraudGraphGenerator(small_config) + gen.run() + + cases_path = os.path.join(small_config.output_dir, "fraud", "fraud_cases.csv") + assert verify_fraud_patterns(cases_path, small_config.output_dir) + + +# --------------------------------------------------------------------------- +# Structuring typology test +# --------------------------------------------------------------------------- + + +class TestStructuringGenerator: + def test_generate_creates_files(self, tmp_dir): + """StructuringGenerator must write transactions_fraud.csv and fraud_cases.csv.""" + emb = EmbeddingGenerator("fake", dim=32) + gen = StructuringGenerator(num_patterns=5, smurfs_range=(3, 3)) + + n_tx, next_id = gen.generate( + max_account_id=1000, + start_tx_id=0, + embedder=emb, + output_dir=tmp_dir, + fmt="csv", + ) + assert n_tx > 0 + assert os.path.exists(os.path.join(tmp_dir, "fraud", "transactions_fraud.csv")) + assert os.path.exists(os.path.join(tmp_dir, "fraud", "fraud_cases.csv")) + + def test_pattern_type_is_structuring(self, tmp_dir): + """fraud_cases.csv rows from StructuringGenerator must have pattern_type='structuring'.""" + emb = EmbeddingGenerator("fake", dim=32) + gen = StructuringGenerator(num_patterns=4, smurfs_range=(3, 3)) + gen.generate( + max_account_id=200, + start_tx_id=0, + embedder=emb, + output_dir=tmp_dir, + ) + with open(os.path.join(tmp_dir, "fraud", "fraud_cases.csv")) as fh: + reader = csv.DictReader(fh) + rows = list(reader) + assert len(rows) == 4 + assert all(r["pattern_type"] == "structuring" for r in rows) + + def test_transaction_count_matches_smurfs(self, tmp_dir): + """With fixed smurf count each pattern must emit exactly that many transactions.""" + emb = EmbeddingGenerator("fake", dim=32) + fixed_smurfs = 5 + num_patterns = 3 + gen = StructuringGenerator( + num_patterns=num_patterns, + smurfs_range=(fixed_smurfs, fixed_smurfs), + ) + n_tx, _ = gen.generate( + max_account_id=500, + start_tx_id=0, + embedder=emb, + output_dir=tmp_dir, + ) + assert n_tx == num_patterns * fixed_smurfs + + def test_amounts_are_sub_threshold(self, tmp_dir): + """All structuring amounts must stay below the $10 000 CTR threshold.""" + emb = EmbeddingGenerator("fake", dim=32) + gen = StructuringGenerator(num_patterns=10, smurfs_range=(3, 7)) + gen.generate( + max_account_id=1000, + start_tx_id=0, + embedder=emb, + output_dir=tmp_dir, + ) + with open(os.path.join(tmp_dir, "fraud", "transactions_fraud.csv")) as fh: + reader = csv.DictReader(fh) + amounts = [float(r["amount"]) for r in reader] + assert all(a < 10_000.00 for a in amounts), "Found amount >= CTR threshold" + + def test_all_transactions_fan_into_coordinator(self, tmp_dir): + """Every transaction in a structuring pattern must target the coordinator (fan-in star).""" + emb = EmbeddingGenerator("fake", dim=32) + fixed_smurfs = 4 + gen = StructuringGenerator(num_patterns=2, smurfs_range=(fixed_smurfs, fixed_smurfs)) + gen.generate( + max_account_id=200, + start_tx_id=0, + embedder=emb, + output_dir=tmp_dir, + ) + # Build a map of coordinator → smurf accounts from fraud_cases.csv + coordinators: dict[str, set[str]] = {} + with open(os.path.join(tmp_dir, "fraud", "fraud_cases.csv")) as fh: + for row in csv.DictReader(fh): + if row["pattern_type"] != "structuring": + continue + coord = row["start_acc_id"] + involved = set(row["involved_accounts"].split("|")) + coordinators[coord] = involved + + with open(os.path.join(tmp_dir, "fraud", "transactions_fraud.csv")) as fh: + for row in csv.DictReader(fh): + dst = row["dst_id"] + if dst in coordinators: + src = row["src_id"] + # source must be a known smurf for this coordinator + assert src in coordinators[dst], ( + f"src {src} not a registered smurf of coordinator {dst}" + ) + + def test_tx_ids_do_not_collide_with_start(self, tmp_dir): + """Transaction IDs must begin at start_tx_id and never reuse prior IDs.""" + emb = EmbeddingGenerator("fake", dim=32) + start = 9_999 + gen = StructuringGenerator(num_patterns=3, smurfs_range=(3, 3)) + _, next_id = gen.generate( + max_account_id=500, + start_tx_id=start, + embedder=emb, + output_dir=tmp_dir, + ) + with open(os.path.join(tmp_dir, "fraud", "transactions_fraud.csv")) as fh: + ids = [int(r["tx_id"].lstrip("tx_")) for r in csv.DictReader(fh)] + assert min(ids) == start + assert next_id == start + len(ids) + + def test_neptune_format(self, tmp_dir): + """Neptune format must not include an embedding column.""" + emb = EmbeddingGenerator("fake", dim=32) + gen = StructuringGenerator(num_patterns=2, smurfs_range=(3, 3)) + n_tx, _ = gen.generate( + max_account_id=200, + start_tx_id=0, + embedder=emb, + output_dir=tmp_dir, + fmt="neptune", + ) + assert n_tx > 0 + with open(os.path.join(tmp_dir, "fraud", "transactions_fraud.csv")) as fh: + headers = next(csv.reader(fh)) + assert "embedding" not in headers + + def test_tiny_account_pool(self, tmp_dir): + """Generator must not crash when max_account_id is smaller than smurfs_range[1] + 1.""" + emb = EmbeddingGenerator("fake", dim=32) + gen = StructuringGenerator(num_patterns=5, smurfs_range=(3, 10)) + n_tx, _ = gen.generate( + max_account_id=5, # smaller than max smurfs + 1 + start_tx_id=0, + embedder=emb, + output_dir=tmp_dir, + ) + assert n_tx > 0