diff --git a/PIPELINE.md b/PIPELINE.md new file mode 100644 index 0000000..9b3666a --- /dev/null +++ b/PIPELINE.md @@ -0,0 +1,80 @@ +# PIPELINE.md — Live HUDOC ingestion and eval-set construction + +The "Live" part of LiveHumanRightsBench: a self-refreshing, contamination-controlled +data pipeline that turns the ECtHR's steady output into evaluation material. As new +judgments are published on HUDOC, the pipeline ingests them, removes verdict leakage, +and extends the benchmark, so metrics can be recomputed on guaranteed post-cutoff cases +rather than a frozen snapshot. + +All outputs are public and redistributable (HUDOC sources only, `seed=42`). + +## Auto-refresh chain + +``` +scripts/hudoc_live_refresh.py # orchestrator: detect latest date -> fetch new -> scrub -> append -> (optional) push + ├── scripts/hudoc_scraper.py # HUDOC bulk ingestion (search + full texts) + └── scripts/verdict_leakage_removal.py# 3-stage verdict removal (pattern truncation -> dual-model verify -> repair) + └── scripts/conclusion_scrub.py # mandatory conclusion-scrub stage (regex), marks rows with `+conclusion_scrub` +``` + +`hudoc_live_refresh.py`: +1. determines the latest `decision_date` in the current dataset (local JSON or an HF dataset), +2. downloads judgments published since that date from HUDOC, +3. runs verdict-leakage removal on the new judgments, +4. appends the clean rows to the dataset, +5. optionally pushes the updated dataset to Hugging Face. + +Designed to run as a cron job: + +```bash +# daily at 06:00, refresh the verdict-free corpus in place and push +0 6 * * * cd /path/to/repo && python scripts/hudoc_live_refresh.py \ + --hf-dataset overthelex/echr-verdict-free \ + --output data/processed/echr_live.json \ + --full-pipeline \ + --push-to-hf overthelex/echr-verdict-free \ + --quiet +``` + +Run `python scripts/hudoc_live_refresh.py --help` for the full flag set +(`--country`, `--since-override`, `--workers`, `--keep-temp`, etc.). + +## Eval-set construction + +Builders that turn the verdict-free corpus into the evaluation sets documented in +[`DATA_SPLITS.md`](DATA_SPLITS.md): + +``` +scripts/clean_respondent_names.py # respondent normalization (fills UKRAINE, etc.) +scripts/backfill_decision_dates.py # HUDOC metadata sweep: item_id -> decision_date / appno +scripts/backfill_article_full.py # HUDOC sweep: item_id -> article_full (protocol-aware codes, e.g. P1-1); resumable, --push-to-hf +scripts/build_stratified_sample.py # k-per-state stratified sampling +scripts/build_livehrb_static.py # -> echr-livehrb-static-2k (regular + ukr, 1K + 1K) +scripts/build_temporal_split.py # -> echr-livehrb-temporal-2k (year bins; UA pre/post 2022-02-24) +scripts/build_cutoff_partitions.py # per-model matched pre/post training-cutoff partitions (contamination control) +scripts/enrich_and_push.py # attach HUDOC metadata + push to Hugging Face +``` + +Per-model training cutoffs used by `build_cutoff_partitions.py` live in +[`configs/model_cutoffs.json`](configs/model_cutoffs.json) (extend as new models are added). + +## Outputs + +| Dataset | Built by | +|---------|----------| +| [`overthelex/echr-verdict-free`](https://huggingface.co/datasets/overthelex/echr-verdict-free) (23K pairs, 112 states) | live refresh + leakage removal | +| [`overthelex/echr-ukr-verdict-free`](https://huggingface.co/datasets/overthelex/echr-ukr-verdict-free) (2.6K, cross-family verified) | live refresh + leakage removal | +| [`overthelex/echr-livehrb-static-2k`](https://huggingface.co/datasets/overthelex/echr-livehrb-static-2k) | `build_livehrb_static.py` | +| [`overthelex/echr-livehrb-temporal-2k`](https://huggingface.co/datasets/overthelex/echr-livehrb-temporal-2k) | `build_temporal_split.py` | + +## Dependencies + +Standard library plus `requests` (HUDOC), `datasets` + `huggingface_hub` (HF I/O), +and, for the LLM verification stage, `openai` and/or `boto3` (AWS Bedrock). See +`requirements.txt`. The LLM-backed verification imports are optional at import time +(guarded), so ingestion and the regex conclusion-scrub run without cloud credentials. + +## Not yet wired + +The domestic-to-Strasbourg linkage layer (matching national-court decisions in EDRSR +to their ECtHR follow-on) is planned but not part of this PR. diff --git a/configs/model_cutoffs.json b/configs/model_cutoffs.json new file mode 100644 index 0000000..3ea3844 --- /dev/null +++ b/configs/model_cutoffs.json @@ -0,0 +1,41 @@ +{ + "_comment": "Training-data cutoff registry. Dates are YYYY-MM-DD taken from provider documentation. null = not yet verified -- build_cutoff_partitions.py refuses to partition for models with null cutoff. Always record the source.", + "models": { + "gpt-4o": { + "cutoff": "2023-10-01", + "source": "OpenAI model documentation (knowledge cutoff Oct 2023)" + }, + "gpt-5.2": { + "cutoff": null, + "source": "PLACEHOLDER -- verify OpenAI docs" + }, + "claude-sonnet-4-6": { + "cutoff": "2026-01-01", + "source": "Anthropic models overview (platform.claude.com), verified 2026-06-11: training data cutoff Jan 2026, reliable knowledge cutoff Aug 2025. Conservative choice: training data cutoff." + }, + "claude-opus-4-8": { + "cutoff": null, + "source": "PLACEHOLDER -- verify Anthropic docs" + }, + "deepseek-v4": { + "cutoff": null, + "source": "PLACEHOLDER -- verify DeepSeek docs" + }, + "gemini-3.1-flash": { + "cutoff": null, + "source": "PLACEHOLDER -- verify Google docs" + }, + "llama-3.3-70b": { + "cutoff": "2023-12-01", + "source": "Meta Llama 3.3 model card (knowledge cutoff Dec 2023)" + }, + "qwen3-32b": { + "cutoff": null, + "source": "PLACEHOLDER -- verify Qwen docs" + }, + "nova-pro": { + "cutoff": null, + "source": "PLACEHOLDER -- verify AWS docs" + } + } +} diff --git a/requirements.txt b/requirements.txt index caa21e8..d6537ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,4 +22,6 @@ inspect-ai evaluate datasets -rouge_score \ No newline at end of file +rouge_score +requests +boto3 diff --git a/scripts/backfill_article_full.py b/scripts/backfill_article_full.py new file mode 100644 index 0000000..cd10911 --- /dev/null +++ b/scripts/backfill_article_full.py @@ -0,0 +1,340 @@ +#!/usr/bin/env python3 +""" +Backfill the protocol-aware `article_full` column onto published datasets. + +The published sets (overthelex/echr-verdict-free, echr-livehrb-static-2k, ...) +carry a lossy `article` field with protocol prefixes collapsed ("Article 1 of +Protocol No. 1" -> "1"). They do NOT carry the raw `conclusion` string (it was +removed as a verdict leak), so the full code cannot be recovered from the row +alone. This script re-fetches the HUDOC `conclusion` column per item_id, parses +it with the fixed parser (hudoc_scraper.parse_conclusion_to_pairs -> article_full), +and writes an enriched dataset with a non-breaking `article_full` column added. + +`article` is left untouched for continuity. Rows whose full code cannot be +resolved unambiguously fall back to the legacy `article` value (never fabricated) +and are counted in the coverage report. + +Usage: + # test on a small batch first (recommended before a full run) + python scripts/backfill_article_full.py \ + --dataset data/echr-livehrb-static-2k.parquet \ + --output data/echr-livehrb-static-2k.article_full.parquet \ + --limit 50 + + # full run with resumable HUDOC cache + python scripts/backfill_article_full.py \ + --dataset data/echr-livehrb-static-2k.parquet \ + --output data/echr-livehrb-static-2k.article_full.parquet \ + --map-output data/conclusion_map.json --resume + + # enrich and push back to HuggingFace + python scripts/backfill_article_full.py \ + --hf-dataset overthelex/echr-livehrb-static-2k \ + --output data/static-2k-v12.parquet \ + --map-output data/conclusion_map.json --resume \ + --push-to-hf overthelex/echr-livehrb-static-2k + + # coverage report only (no HUDOC calls), against an existing map + python scripts/backfill_article_full.py \ + --dataset data/echr-livehrb-static-2k.parquet \ + --map-output data/conclusion_map.json --check-only +""" + +import argparse +import json +import os +import sys +import time +from pathlib import Path + +try: + import pandas as pd +except ImportError: + print("ERROR: pandas is required (pip install pandas pyarrow)", file=sys.stderr) + sys.exit(1) + +try: + import requests +except ImportError: + print("ERROR: requests is required (pip install requests)", file=sys.stderr) + sys.exit(1) + +# Reuse the single source of truth for HUDOC access and the fixed parser. +SCRIPTS_DIR = Path(__file__).parent.resolve() +sys.path.insert(0, str(SCRIPTS_DIR)) +from hudoc_scraper import ( # noqa: E402 + build_search_query, + search_hudoc, + parse_search_result, + parse_conclusion_to_pairs, +) + +HUDOC_OFFSET_CAP = 10000 # HUDOC stops serving results past this offset +BULK_PAGE = 500 # max page size for the bulk search endpoint + + +# ── Dataset I/O ────────────────────────────────────────────────────────────── + +def load_dataset(dataset: str, hf_dataset: str) -> pd.DataFrame: + """Load a local parquet/json or an HF dataset into a DataFrame.""" + if hf_dataset: + from datasets import load_dataset as hf_load + ds = hf_load(hf_dataset, split="train") + return ds.to_pandas() + if dataset.endswith(".parquet"): + return pd.read_parquet(dataset) + if dataset.endswith(".json"): + return pd.read_json(dataset) + raise ValueError(f"Unsupported dataset format: {dataset}") + + +def load_map(path: str) -> dict: + if path and os.path.exists(path): + with open(path, encoding="utf-8") as f: + return json.load(f) + return {} + + +def save_map(data: dict, path: str) -> None: + if not path: + return + tmp = f"{path}.tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(data, f, indent=1) + os.replace(tmp, path) + + +# ── HUDOC sweep ────────────────────────────────────────────────────────────── + +def fetch_conclusion_pairs(session, item_id: str) -> list: + """Exact per-item HUDOC query -> parsed (article, article_full, label) pairs. + + HUDOC prefix-matches itemid (itemid:"002-112" ranks 002-11250 first), so we + keep only the exact match -- the same guard that poisoned 50 dates before. + """ + query = f'contentsitename:ECHR AND (itemid:"{item_id}")' + page = search_hudoc(session, query, start=0, length=100) + exact = [r for r in page.get("results", []) + if r.get("columns", {}).get("itemid") == item_id] + if not exact: + return [] + meta = parse_search_result(exact[0]) + pairs = parse_conclusion_to_pairs( + conclusion=meta.get("conclusion", ""), + item_id=item_id, + case_name=meta.get("case_name", ""), + decision_date=meta.get("decision_date", ""), + respondent=meta.get("respondent", ""), + full_text="", # not needed; keeps parse from emitting an "unknown" row + ) + return [{"article": p["article"], + "article_full": p["article_full"], + "violation_label": p["violation_label"]} + for p in pairs if p.get("article")] + + +def _pairs_from_meta(meta: dict) -> list: + return [{"article": p["article"], + "article_full": p["article_full"], + "violation_label": p["violation_label"]} + for p in parse_conclusion_to_pairs( + conclusion=meta.get("conclusion", ""), + item_id=meta.get("item_id", ""), + case_name=meta.get("case_name", ""), + decision_date=meta.get("decision_date", ""), + respondent=meta.get("respondent", ""), + full_text="") + if p.get("article")] + + +def bulk_sweep(target_ids: set, cache: dict, map_path: str, delay: float, + since_year: int, until_year: int) -> tuple: + """Paged bulk sweep over per-year windows (JUDGMENTS collection). + + Pulls itemid+conclusion ~500 at a time and keeps only target ids, so ~30-80 + requests replace tens of thousands of per-item queries. Per-year windows keep + each result set under the 10K deep-pagination cap. 002-* Information Notes are + not in JUDGMENTS and stay in `remaining` for the per-item fallback. + """ + session = requests.Session() + remaining = set(target_ids) - set(cache) + print(f"bulk sweep: {len(remaining)} target ids to find, years " + f"{since_year}..{until_year}") + for year in range(since_year, until_year + 1): + if not remaining: + break + query = build_search_query(since=f"{year}-01-01", until=f"{year + 1}-01-01", + language="ENG", doc_collection="JUDGMENTS") + start, hits = 0, 0 + while True: + resp = search_hudoc(session, query, start=start, length=BULK_PAGE) + results = resp.get("results", []) + if not results: + break + for r in results: + meta = parse_search_result(r) + iid = meta.get("item_id") + if iid in remaining: + cache[iid] = _pairs_from_meta(meta) + remaining.discard(iid) + hits += 1 + start += len(results) + if start >= HUDOC_OFFSET_CAP or len(results) < BULK_PAGE: + break + time.sleep(delay) + if hits: + print(f" {year}: +{hits} (remaining {len(remaining)})") + save_map(cache, map_path) + time.sleep(delay) + save_map(cache, map_path) + return cache, remaining + + +def sweep(item_ids: list, cache: dict, map_path: str, delay: float, + checkpoint_every: int = 50) -> dict: + session = requests.Session() + todo = [i for i in item_ids if i not in cache] + print(f"{len(item_ids)} unique item_ids; {len(todo)} to fetch " + f"({len(item_ids) - len(todo)} cached)") + for n, item_id in enumerate(todo, 1): + try: + cache[item_id] = fetch_conclusion_pairs(session, item_id) + tag = ",".join(sorted({p["article_full"] for p in cache[item_id]})) or "NONE" + except Exception as e: # network hiccup: leave uncached, resumable + print(f" [{n}/{len(todo)}] {item_id} -> ERROR {e}", file=sys.stderr) + continue + if n % 20 == 0 or n == len(todo): + print(f" [{n}/{len(todo)}] {item_id} -> {tag}") + if n % checkpoint_every == 0: + save_map(cache, map_path) + time.sleep(delay) + save_map(cache, map_path) + return cache + + +# ── Enrichment ─────────────────────────────────────────────────────────────── + +def build_lookup(cache: dict) -> dict: + """(item_id, article, label) -> set(article_full).""" + lut = {} + for item_id, pairs in cache.items(): + for p in pairs: + key = (item_id, str(p["article"]), p["violation_label"]) + lut.setdefault(key, set()).add(p["article_full"]) + return lut + + +def enrich(df: pd.DataFrame, cache: dict) -> tuple: + lut = build_lookup(cache) + filled = protocol = fallback = ambiguous = 0 + out = [] + for _, row in df.iterrows(): + item_id = str(row.get("item_id", "")) + article = str(row.get("article", "")) + label = str(row.get("violation_label", "")) + cands = lut.get((item_id, article, label)) + if cands is None: + # try without the label (some rows differ only by conclusion phrasing) + cands = set().union(*[v for k, v in lut.items() + if k[0] == item_id and k[1] == article]) or None + if cands and len(cands) == 1: + code = next(iter(cands)) + filled += 1 + if code.startswith("P"): + protocol += 1 + elif cands and len(cands) > 1: + code = article # ambiguous within the case -> keep legacy + ambiguous += 1 + else: + code = article # unresolved -> keep legacy (never fabricate) + fallback += 1 + out.append(code) + df = df.copy() + df["article_full"] = out + stats = {"rows": len(df), "resolved": filled, "protocol_coded": protocol, + "ambiguous_kept_legacy": ambiguous, "unresolved_kept_legacy": fallback} + return df, stats + + +def push_to_hf(df: pd.DataFrame, repo: str) -> None: + from datasets import Dataset + print(f"Pushing {len(df)} rows to {repo} (split=train) ...") + Dataset.from_pandas(df, preserve_index=False).push_to_hub(repo, split="train") + print(" pushed.") + + +# ── Main ───────────────────────────────────────────────────────────────────── + +def main() -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + src = ap.add_mutually_exclusive_group(required=True) + src.add_argument("--dataset", help="Local parquet/json to enrich") + src.add_argument("--hf-dataset", help="HF dataset repo to enrich") + ap.add_argument("--output", help="Output path (.parquet or .json)") + ap.add_argument("--map-output", default=None, + help="JSON cache of item_id -> conclusion pairs (resume)") + ap.add_argument("--resume", action="store_true", + help="Reuse an existing --map-output cache") + ap.add_argument("--check-only", action="store_true", + help="Coverage report only; no HUDOC calls") + ap.add_argument("--limit", type=int, default=0, + help="Fetch at most N new item_ids (small test batch)") + ap.add_argument("--bulk", action="store_true", + help="Bulk paged sweep by year window (fast), then per-item " + "fallback for uncovered ids (e.g. 002-* Info Notes)") + ap.add_argument("--since-year", type=int, default=1959, + help="First year for the bulk sweep window (default 1959)") + ap.add_argument("--delay", type=float, default=1.0, + help="Seconds between HUDOC requests (politeness)") + ap.add_argument("--push-to-hf", default=None, help="HF repo to push enriched set") + args = ap.parse_args() + + df = load_dataset(args.dataset or "", args.hf_dataset or "") + if "item_id" not in df.columns or "article" not in df.columns: + print("ERROR: dataset needs item_id and article columns", file=sys.stderr) + return 1 + print(f"Loaded {len(df)} rows.") + + cache = load_map(args.map_output) if (args.resume or args.check_only) else {} + + if not args.check_only: + item_ids = list(dict.fromkeys(str(i) for i in df["item_id"])) + if args.bulk: + until_year = time.gmtime().tm_year + cache, remaining = bulk_sweep(set(item_ids), cache, args.map_output, + args.delay, args.since_year, until_year) + if remaining: + print(f"per-item fallback for {len(remaining)} uncovered ids " + f"(002-* Info Notes etc.)") + cache = sweep(sorted(remaining), cache, args.map_output, args.delay) + else: + if args.limit: + uncached = [i for i in item_ids if i not in cache][:args.limit] + item_ids = [i for i in item_ids if i in cache] + uncached + print(f"--limit {args.limit}: fetching up to {len(uncached)} new item_ids") + cache = sweep(item_ids, cache, args.map_output, args.delay) + + enriched, stats = enrich(df, cache) + print("\nCoverage:") + for k, v in stats.items(): + print(f" {k}: {v}") + dist = enriched["article_full"].astype(str).value_counts() + print("\narticle_full top values:") + print(dist.head(20).to_string()) + + if args.output: + if args.output.endswith(".parquet"): + enriched.to_parquet(args.output, index=False) + else: + enriched.to_json(args.output, orient="records", indent=1) + print(f"\nWrote {args.output}") + + if args.push_to_hf: + push_to_hf(enriched, args.push_to_hf) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/backfill_decision_dates.py b/scripts/backfill_decision_dates.py new file mode 100644 index 0000000..336f516 --- /dev/null +++ b/scripts/backfill_decision_dates.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +""" +Backfill decision dates and application numbers from HUDOC metadata. + +The published datasets (overthelex/echr-verdict-free) carry item_id but not +decision_date / application_number. Both are required for the contamination +audit (pre/post-cutoff partitions) and the facts-ablated variants. + +This script sweeps the HUDOC search API (metadata only, no document bodies), +collects {item_id -> decision_date, application_number, importance, ecli} +for all English judgments, and writes a lookup map. The sweep is paged by +date range, so it is resumable and HUDOC-friendly. + +Usage: + # Full sweep (all English judgments, ~30K metadata rows, no text download) + python scripts/backfill_decision_dates.py --output data/processed/decision_dates.json + + # Restrict to a date range + python scripts/backfill_decision_dates.py --since 2020-01-01 --output data/processed/decision_dates.json + + # Resume an interrupted sweep + python scripts/backfill_decision_dates.py --output data/processed/decision_dates.json --resume + + # Verify coverage against a dataset file + python scripts/backfill_decision_dates.py --check data/processed/stratified_sample.json \ + --output data/processed/decision_dates.json +""" + +import argparse +import json +import os +import sys +import time + +import requests + +# Reuse the HUDOC client from the scraper +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from hudoc_scraper import ( # noqa: E402 + build_search_query, + search_hudoc, + parse_search_result, + PAGE_SIZE, +) + + +def load_existing(path: str) -> dict: + if os.path.exists(path): + with open(path) as f: + return json.load(f) + return {} + + +# HUDOC's search API stops returning results past offset ~10000 (Lucene +# deep-pagination cap), so ranges with more matches are swept in +# per-year windows. +HUDOC_OFFSET_CAP = 9500 +EARLIEST_YEAR = 1959 + + +def sweep_range(session, since: str, until: str, collected: dict, + output: str, checkpoint_every: int = 2000) -> int: + """Page through one date range (must be under the offset cap).""" + query = build_search_query(since=since or "", until=until or "", + language="ENG", doc_collection="JUDGMENTS") + first = search_hudoc(session, query, start=0, length=1) + total = first.get("resultcount", 0) + if total == 0: + return 0 + + start = 0 + since_checkpoint = 0 + while start < total: + page = search_hudoc(session, query, start=start, length=PAGE_SIZE) + results = page.get("results", []) + if not results: + break + for r in results: + meta = parse_search_result(r) + item_id = meta.get("item_id") + if not item_id: + continue + collected[item_id] = { + "decision_date": meta.get("decision_date", ""), + "application_number": meta.get("application_number", ""), + "importance": meta.get("importance", ""), + "ecli": meta.get("ecli", ""), + } + start += len(results) + since_checkpoint += len(results) + print(f" [{since or '*'} .. {until or '*'}] {start}/{total} " + f"(collected {len(collected)} total)") + if since_checkpoint >= checkpoint_every: + save(collected, output) + since_checkpoint = 0 + time.sleep(0.5) + + return total + + +def sweep_metadata(since: str, until: str, existing: dict, output: str) -> dict: + """Sweep HUDOC metadata, splitting into per-year windows when the range + exceeds the deep-pagination cap.""" + session = requests.Session() + query = build_search_query(since=since or "", until=until or "", + language="ENG", doc_collection="JUDGMENTS") + first = search_hudoc(session, query, start=0, length=1) + total = first.get("resultcount", 0) + print(f"HUDOC reports {total} English judgments in range " + f"[{since or '*'} .. {until or '*'}]") + + collected = dict(existing) + + if total <= HUDOC_OFFSET_CAP: + sweep_range(session, since, until, collected, output) + return collected + + start_year = int(since[:4]) if since else EARLIEST_YEAR + end_year = int(until[:4]) if until else time.gmtime().tm_year + print(f"Range exceeds HUDOC offset cap ({HUDOC_OFFSET_CAP}); " + f"sweeping per-year windows {start_year}..{end_year}") + for year in range(start_year, end_year + 1): + y_since = max(since or "", f"{year}-01-01") + y_until = f"{year + 1}-01-01" + if until: + y_until = min(until, y_until) + n = sweep_range(session, y_since, y_until, collected, output) + print(f" year {year}: {n} judgments") + save(collected, output) + + return collected + + +def fill_missing(item_ids: list, collected: dict, output: str) -> None: + """Targeted per-item queries for ids the bulk sweep did not cover + (e.g. 002-* legal summaries, non-JUDGMENTS doc types).""" + session = requests.Session() + for i, item_id in enumerate(item_ids, 1): + query = f'contentsitename:ECHR AND (itemid:"{item_id}")' + # HUDOC does PREFIX matching on itemid (itemid:"002-112" ranks + # 002-11250 first) -- fetch a page and keep the exact id only. + # This poisoned 50 dates across published sets before 2026-07-03. + page = search_hudoc(session, query, start=0, length=100) + results = [r for r in page.get("results", []) + if r.get("columns", {}).get("itemid") == item_id] + if results: + meta = parse_search_result(results[0]) + collected[item_id] = { + "decision_date": meta.get("decision_date", ""), + "application_number": meta.get("application_number", ""), + "importance": meta.get("importance", ""), + "ecli": meta.get("ecli", ""), + } + print(f" [{i}/{len(item_ids)}] {item_id} -> {meta.get('decision_date', '?')}") + else: + print(f" [{i}/{len(item_ids)}] {item_id} -> NOT FOUND") + if i % 25 == 0: + save(collected, output) + time.sleep(0.4) + save(collected, output) + + +def save(data: dict, path: str) -> None: + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + tmp = path + ".tmp" + with open(tmp, "w") as f: + json.dump(data, f, indent=1) + os.replace(tmp, path) + + +def check_coverage(dataset_path: str, dates: dict) -> None: + with open(dataset_path) as f: + records = json.load(f) + item_ids = {r["item_id"] for r in records} + missing = sorted(i for i in item_ids + if i not in dates or not dates[i].get("decision_date")) + print(f"Coverage check against {dataset_path}:") + print(f" dataset item_ids: {len(item_ids)}") + print(f" with decision_date: {len(item_ids) - len(missing)}") + print(f" missing: {len(missing)}") + if missing: + print(" first missing:", missing[:10]) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--output", required=True, help="Output JSON map path") + ap.add_argument("--since", default=None, help="Start date YYYY-MM-DD") + ap.add_argument("--until", default=None, help="End date YYYY-MM-DD") + ap.add_argument("--resume", action="store_true", + help="Merge into existing output instead of starting fresh") + ap.add_argument("--check", default=None, + help="Dataset JSON to verify coverage against (skips sweep if used alone with existing output)") + ap.add_argument("--check-only", action="store_true", + help="Only run the coverage check, no sweep") + ap.add_argument("--fill-missing", default=None, + help="Dataset JSON; targeted per-item queries for item_ids " + "missing from the existing output map") + args = ap.parse_args() + + existing = load_existing(args.output) if (args.resume or args.check_only + or args.fill_missing) else {} + + if args.fill_missing: + with open(args.fill_missing) as f: + records = json.load(f) + missing = sorted({r["item_id"] for r in records} + - {i for i, v in existing.items() if v.get("decision_date")}) + print(f"{len(missing)} item_ids missing dates; querying individually") + fill_missing(missing, existing, args.output) + if args.check: + check_coverage(args.check, existing) + return 0 + + if not args.check_only: + collected = sweep_metadata(args.since, args.until, existing, args.output) + save(collected, args.output) + print(f"Saved {len(collected)} item_id -> metadata entries to {args.output}") + else: + collected = existing + print(f"Loaded {len(collected)} entries from {args.output}") + + if args.check: + check_coverage(args.check, collected) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/build_cutoff_partitions.py b/scripts/build_cutoff_partitions.py new file mode 100644 index 0000000..3bbfa0b --- /dev/null +++ b/scripts/build_cutoff_partitions.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +""" +Build pre/post-cutoff matched partitions per model for the contamination +audit (RQ1). + +For each model m with training cutoff c_m (configs/model_cutoffs.json), +evaluation cases are split into: + + D_pre(m) = cases decided before c_m (potentially in training data) + D_post(m) = cases decided after c_m (guaranteed unseen) + +The contamination gap is Delta_m = Acc_m(D_pre) - Acc_m(D_post) on MATCHED +samples: each post-cutoff case is greedily paired with an unused pre-cutoff +case sharing (respondent, article, violation_label), so the two partitions +have identical distributions over the matching key. Unmatched cases are +reported and excluded from the matched analysis. + +Models with a null cutoff in the registry are refused -- verify the provider +documentation first, never guess. + +Usage: + python scripts/build_cutoff_partitions.py \ + --dataset data/processed/stratified_sample.json \ + --dates data/processed/decision_dates.json \ + --cutoffs configs/model_cutoffs.json \ + --models gpt-4o llama-3.3-70b \ + --output data/processed/partitions/ + + # All models with verified cutoffs: + python scripts/build_cutoff_partitions.py \ + --dataset ... --dates ... --cutoffs configs/model_cutoffs.json --all \ + --output data/processed/partitions/ +""" + +import argparse +import json +import os +import random +import sys +from collections import defaultdict + + +def load_json(path): + with open(path) as f: + return json.load(f) + + +def match_key(rec: dict) -> tuple: + return (rec.get("respondent", ""), rec.get("article", ""), + rec.get("violation_label", "")) + + +def build_partition(records: list, dates: dict, cutoff: str, seed: int) -> dict: + """Split records on cutoff date and greedily 1:1 match post->pre on + (respondent, article, violation_label).""" + pre, post, undated = [], [], [] + for rec in records: + d = dates.get(rec.get("item_id", ""), {}).get("decision_date", "") + if not d: + undated.append(rec["item_id"]) + continue + (pre if d < cutoff else post).append(rec) + + pre_by_key = defaultdict(list) + for rec in pre: + pre_by_key[match_key(rec)].append(rec) + + rng = random.Random(seed) + for bucket in pre_by_key.values(): + rng.shuffle(bucket) + + matched_pairs, unmatched_post = [], [] + post_shuffled = list(post) + rng.shuffle(post_shuffled) + for rec in post_shuffled: + bucket = pre_by_key.get(match_key(rec)) + if bucket: + matched_pairs.append({"post": rec["item_id"] + "|" + rec["article"], + "pre": bucket.pop()["item_id"] + "|" + rec["article"], + "key": list(match_key(rec))}) + else: + unmatched_post.append(rec["item_id"]) + + return { + "cutoff": cutoff, + "n_pre_total": len(pre), + "n_post_total": len(post), + "n_undated": len(undated), + "n_matched_pairs": len(matched_pairs), + "n_unmatched_post": len(unmatched_post), + "matched_pairs": matched_pairs, + "unmatched_post_item_ids": unmatched_post, + "undated_item_ids": undated, + } + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--dataset", required=True, help="Dataset JSON (list of records)") + ap.add_argument("--dates", required=True, + help="decision_dates.json from backfill_decision_dates.py") + ap.add_argument("--cutoffs", default="configs/model_cutoffs.json") + ap.add_argument("--models", nargs="*", default=[], + help="Model keys from the cutoff registry") + ap.add_argument("--all", action="store_true", + help="Partition for every model with a verified (non-null) cutoff") + ap.add_argument("--seed", type=int, default=42) + ap.add_argument("--output", required=True, help="Output directory") + args = ap.parse_args() + + records = load_json(args.dataset) + dates = load_json(args.dates) + registry = load_json(args.cutoffs)["models"] + + if args.all: + models = [m for m, v in registry.items() if v.get("cutoff")] + skipped = [m for m, v in registry.items() if not v.get("cutoff")] + if skipped: + print(f"Skipping models with unverified cutoffs: {', '.join(skipped)}") + else: + models = args.models + if not models: + ap.error("Provide --models or --all") + + os.makedirs(args.output, exist_ok=True) + for m in models: + if m not in registry: + print(f"ERROR: '{m}' not in cutoff registry {args.cutoffs}", file=sys.stderr) + return 1 + cutoff = registry[m].get("cutoff") + if not cutoff: + print(f"ERROR: cutoff for '{m}' is null -- verify provider docs and " + f"fill configs/model_cutoffs.json (source field is mandatory)", + file=sys.stderr) + return 1 + + part = build_partition(records, dates, cutoff, args.seed) + part["model"] = m + part["cutoff_source"] = registry[m].get("source", "") + out_path = os.path.join(args.output, f"partition_{m.replace('/', '_')}.json") + with open(out_path, "w") as f: + json.dump(part, f, indent=1) + print(f"{m}: cutoff {cutoff} -> pre {part['n_pre_total']}, " + f"post {part['n_post_total']}, matched pairs {part['n_matched_pairs']}, " + f"unmatched post {part['n_unmatched_post']}, undated {part['n_undated']} " + f"-> {out_path}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/build_livehrb_static.py b/scripts/build_livehrb_static.py new file mode 100644 index 0000000..5eecf8a --- /dev/null +++ b/scripts/build_livehrb_static.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +""" +LSYC-29: balanced 1K + 1K static eval set for LiveHumanRightsBench. + +- regular 1K: from overthelex/echr-verdict-free, EXCLUDING Ukraine respondents, + pair-level round-robin stratified by respondent (country) -> even country coverage. +- ukr 1K: from overthelex/echr-ukr-verdict-free, pair-level round-robin + stratified by ECHR article. +- Natural violation/no-violation distribution preserved (base-rate balancing is a + separate task, STRAS-4). decision_date/appno/importance attached from + decision_dates.json where available. +- Fully reproducible: fixed seed, deterministic round-robin. +""" +import argparse, json, os, random, sys + +from conclusion_scrub import scrub_records +from collections import defaultdict, Counter + +def load_hf(name): + from datasets import load_dataset + ds = load_dataset(name, split="train") + return [dict(r) for r in ds] + +def round_robin(records, strata_key, target, seed): + rng = random.Random(seed) + strata = defaultdict(list) + for r in records: + strata[str(r.get(strata_key) or "UNK")].append(r) + # deterministic: sort strata by key, shuffle members with seeded rng + order = sorted(strata.keys()) + for k in order: + rng.shuffle(strata[k]) + picked, i = [], 0 + while len(picked) < target and any(strata[k] for k in order): + k = order[i % len(order)] + if strata[k]: + picked.append(strata[k].pop()) + i += 1 + return picked + +def enrich(rec, dates): + d = dates.get(rec.get("item_id", ""), {}) + rec = dict(rec) + rec["decision_date"] = d.get("decision_date", "") + rec["application_number"] = d.get("application_number", "") + rec["importance"] = d.get("importance", "") + return rec + +def stats(name, items): + lab = Counter(x["violation_label"] for x in items) + dated = sum(1 for x in items if x["decision_date"]) + yrs = sorted({x["decision_date"][:4] for x in items if x["decision_date"]}) + ncases = len({x["item_id"] for x in items}) + narts = len({x["article"] for x in items}) + ncoun = len({x.get("respondent") for x in items}) + print(f"[{name}] pairs={len(items)} cases={ncases} articles={narts} countries={ncoun} " + f"labels={dict(lab)} dated={dated}/{len(items)} years={yrs[:1]}..{yrs[-1:]}") + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--dates", default="data/processed/decision_dates.json") + ap.add_argument("--out", default="data/processed/livehrb_static_1k1k.json") + ap.add_argument("--target", type=int, default=1000) + ap.add_argument("--seed", type=int, default=42) + a = ap.parse_args() + + dates = json.load(open(a.dates)) + print("regular pool: overthelex/echr-verdict-free") + reg_all = load_hf("overthelex/echr-verdict-free") + reg_pool = [r for r in reg_all if "ukrain" not in str(r.get("respondent", "")).lower()] + print(f" {len(reg_all)} pairs, {len(reg_pool)} after excluding Ukraine") + print("ukr pool: overthelex/echr-ukr-verdict-free") + ukr_pool = load_hf("overthelex/echr-ukr-verdict-free") + print(f" {len(ukr_pool)} pairs") + + reg = [enrich(r, dates) for r in round_robin(reg_pool, "respondent", a.target, a.seed)] + ukr = [enrich(r, dates) for r in round_robin(ukr_pool, "article", a.target, a.seed)] + for x in reg: x["group"] = "regular" + for x in ukr: x["group"] = "ukr" + + # Defense in depth: scrub conclusion sentences even if the source sets + # regress, and hard-fail on any own-article label leakage. + reg = scrub_records(reg) + ukr = scrub_records(ukr) + + stats("regular", reg); stats("ukr", ukr) + out = { + "meta": { + "task": "LSYC-29 LiveHumanRightsBench static eval set", + "seed": a.seed, "target_per_group": a.target, + "sources": ["overthelex/echr-verdict-free (regular, ex-UA)", + "overthelex/echr-ukr-verdict-free (ukr)"], + "stratify": {"regular": "respondent(country)", "ukr": "article"}, + "note": "natural label distribution; base-rate balancing = STRAS-4", + "counts": {"regular": len(reg), "ukr": len(ukr)}, + }, + "items": reg + ukr, + } + os.makedirs(os.path.dirname(a.out), exist_ok=True) + json.dump(out, open(a.out, "w"), ensure_ascii=False) + print("wrote", a.out, "total", len(reg) + len(ukr)) + +if __name__ == "__main__": + main() diff --git a/scripts/build_stratified_sample.py b/scripts/build_stratified_sample.py new file mode 100755 index 0000000..3ae7b3e --- /dev/null +++ b/scripts/build_stratified_sample.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +""" +Build a balanced stratified sample from the multi-jurisdictional ECtHR +verdict-free dataset. + +Stratification strategy: + 1. Rank respondent states by number of case-article pairs + 2. Select top N countries + 3. For each country, sample K *cases* (not pairs) uniformly at random, + then include all article-pairs belonging to those cases + 4. If a country has fewer than K cases, take all of them + +Usage: + # From HuggingFace dataset + python scripts/build_stratified_sample.py \ + --output data/processed/stratified_sample.json \ + --cases-per-country 20 --top-n-countries 20 + + # From local JSON + python scripts/build_stratified_sample.py \ + --input data/processed/echr_full.json \ + --output data/processed/stratified_sample.json + + # Custom parameters + python scripts/build_stratified_sample.py \ + --output data/processed/stratified_sample_10x30.json \ + --cases-per-country 30 --top-n-countries 10 --seed 123 +""" + +import argparse +import json +import os +import random +import sys +from collections import Counter, defaultdict + + +def load_from_huggingface() -> list: + """Load the dataset from HuggingFace Hub.""" + try: + from datasets import load_dataset + except ImportError: + print("ERROR: 'datasets' package required for HuggingFace loading.", file=sys.stderr) + print("Install with: pip install datasets", file=sys.stderr) + sys.exit(1) + + print("Loading dataset from HuggingFace: overthelex/echr-verdict-free ...") + ds = load_dataset("overthelex/echr-verdict-free", split="train") + records = [dict(row) for row in ds] + print(f" Loaded {len(records)} records from HuggingFace.") + return records + + +def load_from_json(path: str) -> list: + """Load dataset from a local JSON file.""" + print(f"Loading dataset from {path} ...") + with open(path) as f: + records = json.load(f) + print(f" Loaded {len(records)} records from JSON.") + return records + + +def build_stratified_sample( + records: list, + cases_per_country: int, + top_n_countries: int, + seed: int, +) -> list: + """ + Build a stratified sample: + - Rank countries by pair count, take top N + - For each country, sample K unique cases, include all their pairs + """ + rng = random.Random(seed) + + # Group by respondent state + country_pairs = defaultdict(list) + for rec in records: + country = rec.get("respondent", "UNKNOWN") + country_pairs[country].append(rec) + + # Rank by pair count + country_ranking = sorted( + country_pairs.keys(), + key=lambda c: len(country_pairs[c]), + reverse=True, + ) + selected_countries = country_ranking[:top_n_countries] + + # Sample cases per country + sampled = [] + + for country in selected_countries: + pairs = country_pairs[country] + + # Group pairs by case (item_id) + cases = defaultdict(list) + for p in pairs: + cases[p["item_id"]].append(p) + + case_ids = sorted(cases.keys()) # sort for determinism before sampling + n_available = len(case_ids) + n_sample = min(cases_per_country, n_available) + + chosen_ids = rng.sample(case_ids, n_sample) + + for cid in sorted(chosen_ids): # deterministic output order + sampled.extend(cases[cid]) + + return sampled + + +def print_stats(records: list, label: str = "Dataset") -> None: + """Print summary statistics for a set of records.""" + if not records: + print(f"\n{label}: 0 records") + return + + total_pairs = len(records) + unique_cases = len({r["item_id"] for r in records}) + articles = Counter(r["article"] for r in records) + countries = Counter(r.get("respondent", "UNKNOWN") for r in records) + violations = sum(1 for r in records if r.get("violation_label") == "violation") + no_violations = sum(1 for r in records if r.get("violation_label") == "no_violation") + + print(f"\n{'=' * 65}") + print(f" {label}") + print(f"{'=' * 65}") + print(f" Total case-article pairs : {total_pairs:,}") + print(f" Unique cases : {unique_cases:,}") + print(f" Countries : {len(countries):,}") + print(f" Unique articles : {len(articles):,}") + print(f" Violations : {violations:,} ({100 * violations / total_pairs:.1f}%)") + print(f" No violations : {no_violations:,} ({100 * no_violations / total_pairs:.1f}%)") + + # Per-country breakdown + print(f"\n {'Country':<25} {'Pairs':>6} {'Cases':>6} {'Viol%':>6}") + print(f" {'-' * 25} {'-' * 6} {'-' * 6} {'-' * 6}") + + country_cases = defaultdict(set) + country_violations = defaultdict(int) + country_pair_count = Counter() + + for r in records: + c = r.get("respondent", "UNKNOWN") + country_cases[c].add(r["item_id"]) + country_pair_count[c] += 1 + if r.get("violation_label") == "violation": + country_violations[c] += 1 + + for country in sorted( + country_pair_count.keys(), + key=lambda x: country_pair_count[x], + reverse=True, + ): + n_pairs = country_pair_count[country] + n_cases = len(country_cases[country]) + viol_rate = 100 * country_violations[country] / n_pairs if n_pairs else 0 + print(f" {country:<25} {n_pairs:>6} {n_cases:>6} {viol_rate:>5.1f}%") + + # Article distribution + print(f"\n Article distribution:") + for art, count in articles.most_common(15): + print(f" Art. {art:<6} : {count:>5} ({100 * count / total_pairs:.1f}%)") + if len(articles) > 15: + print(f" ... and {len(articles) - 15} more articles") + + +def main(): + parser = argparse.ArgumentParser( + description="Build a balanced stratified sample from the ECtHR verdict-free dataset.", + ) + parser.add_argument( + "--input", + type=str, + default=None, + help="Path to input JSON file. If not provided, loads from HuggingFace.", + ) + parser.add_argument( + "--output", + type=str, + required=True, + help="Output path for the stratified sample JSON.", + ) + parser.add_argument( + "--cases-per-country", + type=int, + default=20, + help="Number of cases to sample per country (default: 20).", + ) + parser.add_argument( + "--top-n-countries", + type=int, + default=20, + help="Number of top respondent states to include (default: 20).", + ) + parser.add_argument( + "--seed", + type=int, + default=42, + help="Random seed for reproducibility (default: 42).", + ) + args = parser.parse_args() + + # Load data + if args.input: + records = load_from_json(args.input) + else: + records = load_from_huggingface() + + if not records: + print("ERROR: No records loaded.", file=sys.stderr) + sys.exit(1) + + # Show source stats + print_stats(records, label="Source dataset") + + # Build sample + print( + f"\nBuilding stratified sample: top {args.top_n_countries} countries, " + f"{args.cases_per_country} cases each, seed={args.seed}" + ) + + sample = build_stratified_sample( + records, + cases_per_country=args.cases_per_country, + top_n_countries=args.top_n_countries, + seed=args.seed, + ) + + # Show sample stats + print_stats(sample, label="Stratified sample") + + # Write output + with open(args.output, "w") as f: + json.dump(sample, f, ensure_ascii=False, indent=2) + + file_size = os.path.getsize(args.output) + print(f"\nWritten {len(sample)} records to {args.output}") + print(f"File size: {file_size / (1024 * 1024):.1f} MB") + + +if __name__ == "__main__": + main() diff --git a/scripts/build_temporal_split.py b/scripts/build_temporal_split.py new file mode 100644 index 0000000..7bccdcd --- /dev/null +++ b/scripts/build_temporal_split.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +""" +LSYC-30: temporally binned live split for LiveHumanRightsBench. + +Two axes of temporal contamination control (ECtHR HUDOC, verdict-free, public +sources only -> redistributable). Reproducible: fixed seed, deterministic +round-robin. + +- regular_temporal (1K): ex-Ukraine cases from overthelex/echr-verdict-free, + binned by decision YEAR over a 10-year window (2017-2026), 100/bin, + round-robin stratified by respondent within each year -> even country + time + coverage. Enables per-model pre/post training-cutoff analysis and temporal + drift plots (temporal-drift framework arXiv:2605.24452). +- ukr_temporal (1K): overthelex/echr-ukr-verdict-free split at Russia's + full-scale invasion (2022-02-24): 500 pre / 500 post, round-robin by article + within each stratum -> geopolitical-shift axis. + +Adds two columns to the base echr schema: group (regular_temporal|ukr_temporal) +and bin (year string, or pre_2022|post_2022). +""" +import argparse + +import json, os, random, datetime as dt +from conclusion_scrub import scrub_records +from collections import defaultdict, Counter + +BOUNDARY = dt.date(2022, 2, 24) # Russia full-scale invasion of Ukraine + + +def load_hf(name): + from datasets import load_dataset + return [dict(r) for r in load_dataset(name, split="train")] + + +def parse_date(s): + if not s: + return None + s = s[:10] + for f in ("%d/%m/%Y", "%Y-%m-%d", "%d-%m-%Y"): + try: + return dt.datetime.strptime(s, f).date() + except ValueError: + pass + try: + return dt.date(int(s[:4]), 1, 1) + except ValueError: + return None + + +def get_date(rec, dates): + d = rec.get("decision_date") or dates.get(rec.get("item_id", ""), {}).get("decision_date", "") + return parse_date(d) + + +def round_robin(records, strata_key, target, seed): + rng = random.Random(seed) + strata = defaultdict(list) + for r in records: + strata[str(r.get(strata_key) or "UNK")].append(r) + order = sorted(strata.keys()) + for k in order: + rng.shuffle(strata[k]) + picked, i = [], 0 + while len(picked) < target and any(strata[k] for k in order): + k = order[i % len(order)] + if strata[k]: + picked.append(strata[k].pop()) + i += 1 + return picked + + +def enrich(rec, dates, group, bin_): + d = dates.get(rec.get("item_id", ""), {}) + rec = dict(rec) + rec["decision_date"] = rec.get("decision_date") or d.get("decision_date", "") + rec["application_number"] = rec.get("application_number") or d.get("application_number", "") + rec["importance"] = rec.get("importance") or d.get("importance", "") + rec["ecli"] = rec.get("ecli") or d.get("ecli", "") + rec["group"] = group + rec["bin"] = bin_ + return rec + + +def stats(name, items): + lab = Counter(x["violation_label"] for x in items) + bins = Counter(x["bin"] for x in items) + dated = sum(1 for x in items if x["decision_date"]) + ncoun = len({x.get("respondent") for x in items}) + print(f"[{name}] pairs={len(items)} labels={dict(lab)} countries={ncoun} " + f"dated={dated}/{len(items)} bins={dict(sorted(bins.items()))}") + + +def build_card(repo, meta, reg_n, ukr_n): + return """--- +license: cc-by-4.0 +task_categories: +- text-classification +language: +- en +tags: +- legal +- echr +- human-rights +- verdict-free +- temporal +size_categories: +- 1K geopolitical-shift axis. + +Natural violation/no-violation base rate preserved (base-rate balancing is a +separate task). Public HUDOC sources only -> redistributable. Seed=%d. + +## Columns + +item_id, case_name, respondent, article, violation_label, verdict_free_text, +decision_date, application_number, importance, ecli, **group** +(regular_temporal | ukr_temporal), **bin** (year string | pre_2022 | post_2022), +plus length fields. + +## Usage + + from datasets import load_dataset + ds = load_dataset("%s", split="train") + # per-year drift on regular cases + y2024 = ds.filter(lambda r: r["group"] == "regular_temporal" and r["bin"] == "2024") + # Ukraine pre/post full-scale invasion + ukr_pre = ds.filter(lambda r: r["bin"] == "pre_2022") + ukr_post = ds.filter(lambda r: r["bin"] == "post_2022") +""" % (reg_n, meta["year_start"], meta["year_end"], ukr_n, + meta["ukr_pre"], meta["ukr_post"], meta["seed"], repo) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--dates", default="data/processed/decision_dates.json") + ap.add_argument("--out", default="data/processed/livehrb_temporal_2k.json") + ap.add_argument("--seed", type=int, default=42) + ap.add_argument("--year-start", type=int, default=2017) + ap.add_argument("--year-end", type=int, default=2026) + ap.add_argument("--per-bin", type=int, default=100) + ap.add_argument("--ukr-per-side", type=int, default=500) + ap.add_argument("--repo", default="overthelex/echr-livehrb-temporal-2k") + ap.add_argument("--push", action="store_true") + a = ap.parse_args() + + dates = json.load(open(a.dates)) + + # --- regular_temporal: ex-UA, binned by year --- + reg_all = load_hf("overthelex/echr-verdict-free") + by_year = defaultdict(list) + for r in reg_all: + if "ukrain" in str(r.get("respondent", "")).lower(): + continue + d = get_date(r, dates) + if d and a.year_start <= d.year <= a.year_end: + by_year[d.year].append(r) + reg = [] + for y in range(a.year_start, a.year_end + 1): + pool = by_year.get(y, []) + picked = round_robin(pool, "respondent", a.per_bin, a.seed + y) + reg += [enrich(r, dates, "regular_temporal", str(y)) for r in picked] + print(f" regular {y}: pool={len(pool)} picked={len(picked)}") + + # --- ukr_temporal: pre/post invasion --- + ukr_all = load_hf("overthelex/echr-ukr-verdict-free") + pre, post = [], [] + for r in ukr_all: + d = get_date(r, dates) + if not d: + continue + (pre if d < BOUNDARY else post).append(r) + ukr = [enrich(r, dates, "ukr_temporal", "pre_2022") + for r in round_robin(pre, "article", a.ukr_per_side, a.seed)] + ukr += [enrich(r, dates, "ukr_temporal", "post_2022") + for r in round_robin(post, "article", a.ukr_per_side, a.seed)] + print(f" ukr pool pre={len(pre)} post={len(post)}") + + # Defense in depth: scrub conclusion sentences even if the source sets + # regress, and hard-fail on any own-article label leakage. + reg = scrub_records(reg) + ukr = scrub_records(ukr) + + stats("regular_temporal", reg) + stats("ukr_temporal", ukr) + items = reg + ukr + + ukr_pre = sum(1 for x in ukr if x["bin"] == "pre_2022") + ukr_post = sum(1 for x in ukr if x["bin"] == "post_2022") + meta = { + "task": "LSYC-30 LiveHumanRightsBench temporal split", + "seed": a.seed, + "year_start": a.year_start, "year_end": a.year_end, + "per_bin": a.per_bin, "ukr_per_side": a.ukr_per_side, + "ukr_pre": ukr_pre, "ukr_post": ukr_post, + "sources": ["overthelex/echr-verdict-free (regular, ex-UA)", + "overthelex/echr-ukr-verdict-free (ukr)"], + "axes": {"regular_temporal": "year bins (per-model cutoff / drift)", + "ukr_temporal": "pre/post 2022-02-24 full-scale invasion"}, + "framework": "temporal-drift arXiv:2605.24452", + "counts": {"regular_temporal": len(reg), "ukr_temporal": len(ukr)}, + } + out = {"meta": meta, "items": items} + os.makedirs(os.path.dirname(a.out), exist_ok=True) + json.dump(out, open(a.out, "w"), ensure_ascii=False) + print("wrote", a.out, "total", len(items)) + + if a.push: + from datasets import Dataset + from huggingface_hub import HfApi + ds = Dataset.from_list(items) + ds.push_to_hub(a.repo, commit_message=( + "LiveHumanRightsBench temporal split: 1K regular year-binned + " + "1K Ukrainian pre/post 2022-02-24 (verdict-free, dated)")) + card = build_card(a.repo, meta, len(reg), len(ukr)) + HfApi().upload_file(path_or_fileobj=card.encode(), path_in_repo="README.md", + repo_id=a.repo, repo_type="dataset", + commit_message="Add dataset card") + print("PUSHED", a.repo) + + +if __name__ == "__main__": + main() diff --git a/scripts/clean_respondent_names.py b/scripts/clean_respondent_names.py new file mode 100644 index 0000000..7077116 --- /dev/null +++ b/scripts/clean_respondent_names.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +""" +Clean and normalize respondent names in the ECtHR verdict-free dataset. + +Fixes: + 1. UNKNOWN respondents - re-extract from case_name ("v. COUNTRY" / "c. COUNTRY") + 2. Broken parsing ("V. FRANCE", "AND OTHERS V. RUSSIA") - re-extract from case_name + 3. Duplicate country names - normalize to canonical form + +Usage: + # Preview changes (dry run) + python scripts/clean_respondent_names.py --preview + + # Apply changes and save + python scripts/clean_respondent_names.py --apply --output data/processed/echr_verdict_free_clean.json + + # Push cleaned dataset to HuggingFace + python scripts/clean_respondent_names.py --apply --push overthelex/echr-verdict-free +""" + +import argparse +import json +import re +from collections import Counter + +COUNTRY_NORMALIZATION = { + "TÜRKİYE": "TURKEY", + "TÜRKIYE": "TURKEY", + "TURQUIE": "TURKEY", + "REPUBLIC OF MOLDOVA": "MOLDOVA", + "THE REPUBLIC OF MOLDOVA": "MOLDOVA", + '"THE FORMER YUGOSLAV REPUBLIC OF MACEDONIA"': "NORTH MACEDONIA", + "THE FORMER YUGOSLAV REPUBLIC OF MACEDONIA": "NORTH MACEDONIA", + '"THE FORMER YOUGOSLAV REPUBLIC OF MACEDONIA"': "NORTH MACEDONIA", + "THE FORMER YOUGOSLAV REPUBLIC OF MACEDONIA": "NORTH MACEDONIA", + "FORMER YUGOSLAV REPUBLIC OF MACEDONIA": "NORTH MACEDONIA", + "THE NETHERLANDS": "NETHERLANDS", + "THE UNITED KINGDOM": "UNITED KINGDOM", + "THE CZECH REPUBLIC": "CZECH REPUBLIC", + "ITALIE": "ITALY", + "FINLANDE": "FINLAND", +} + + +def strip_gc_suffix(name: str) -> str: + return re.sub(r"\s*\[GC\]\s*$", "", name, flags=re.IGNORECASE).strip() + + +def extract_respondent(case_name: str) -> str | None: + m = re.search(r"\bv\.\s+(.+?)(?:\s*$|\s*\()", case_name, re.IGNORECASE) + if not m: + m = re.search(r"\bc\.\s+(.+?)(?:\s*$|\s*\()", case_name, re.IGNORECASE) + if m: + country = m.group(1).strip().rstrip(".") + country = country.strip('"').strip("'").strip() + country = strip_gc_suffix(country) + return country.upper() + return None + + +KNOWN_COUNTRIES = { + "RUSSIA", "UKRAINE", "TURKEY", "ROMANIA", "HUNGARY", + "POLAND", "AZERBAIJAN", "BULGARIA", "CROATIA", "ITALY", + "MOLDOVA", "SERBIA", "ARMENIA", "LITHUANIA", "SLOVAKIA", + "SLOVENIA", "GREECE", "GEORGIA", "FRANCE", "UNITED KINGDOM", + "GERMANY", "ALBANIA", "MALTA", "BOSNIA AND HERZEGOVINA", + "AUSTRIA", "LATVIA", "SWITZERLAND", "SPAIN", "PORTUGAL", + "BELGIUM", "SWEDEN", "CZECH REPUBLIC", "MONTENEGRO", + "FINLAND", "NETHERLANDS", "CYPRUS", "ESTONIA", + "NORTH MACEDONIA", "NORWAY", "DENMARK", "ICELAND", + "IRELAND", "SAN MARINO", "LUXEMBOURG", "LIECHTENSTEIN", + "ANDORRA", +} + + +def normalize_country(name: str) -> str: + name = name.strip().strip('"').strip("'").strip() + name = strip_gc_suffix(name) + if name in COUNTRY_NORMALIZATION: + return COUNTRY_NORMALIZATION[name] + if name.startswith("THE "): + stripped = name[4:] + if stripped in COUNTRY_NORMALIZATION: + return COUNTRY_NORMALIZATION[stripped] + if stripped in KNOWN_COUNTRIES: + return stripped + if name.startswith("V. ") or name.startswith("AND "): + for country in sorted(KNOWN_COUNTRIES, key=len, reverse=True): + if name.endswith(country): + return country + for orig, norm in COUNTRY_NORMALIZATION.items(): + if name.endswith(orig): + return norm + return name + + +def needs_reextraction(respondent: str) -> bool: + if respondent == "UNKNOWN": + return True + if respondent.startswith("V. "): + return True + if respondent.startswith("AND "): + return True + if "V." in respondent and respondent not in ("AZERBAIJAN", "SLOVENIA"): + parts = respondent.split() + if any(p == "V." for p in parts) and not all( + p in ("V.", "AND") or len(p) > 3 for p in parts + ): + return True + return False + + +def is_multi_respondent(respondent: str) -> bool: + if " AND " in respondent: + parts = [p.strip() for p in respondent.split(" AND ")] + return all(len(p) > 3 and not p.startswith("V.") and p.isalpha() or " " in p for p in parts) + return False + + +def clean_dataset(records: list[dict], preview: bool = True) -> list[dict]: + changes = [] + reextracted = 0 + normalized = 0 + multi_respondent_split = 0 + unchanged = 0 + + for rec in records: + old = rec["respondent"] + new = old + + if needs_reextraction(old): + extracted = extract_respondent(rec["case_name"]) + if extracted: + new = extracted + reextracted += 1 + else: + new = old + + new = normalize_country(new) + + if " AND " in new and not is_multi_respondent(new): + parts = new.split(" AND ") + last = parts[-1].strip() + for country in COUNTRY_NORMALIZATION: + if last == country or last == normalize_country(country): + new = normalize_country(last) + break + else: + known_countries = { + "RUSSIA", "UKRAINE", "TURKEY", "ROMANIA", "HUNGARY", + "POLAND", "AZERBAIJAN", "BULGARIA", "CROATIA", "ITALY", + "MOLDOVA", "SERBIA", "ARMENIA", "LITHUANIA", "SLOVAKIA", + "SLOVENIA", "GREECE", "GEORGIA", "FRANCE", "UNITED KINGDOM", + "GERMANY", "ALBANIA", "MALTA", "BOSNIA AND HERZEGOVINA", + "AUSTRIA", "LATVIA", "SWITZERLAND", "SPAIN", "PORTUGAL", + "BELGIUM", "SWEDEN", "CZECH REPUBLIC", "MONTENEGRO", + "FINLAND", "NETHERLANDS", "CYPRUS", "ESTONIA", + "NORTH MACEDONIA", "NORWAY", "DENMARK", "ICELAND", + "IRELAND", "SAN MARINO", "LUXEMBOURG", "LIECHTENSTEIN", + "ANDORRA", + } + if last in known_countries: + new = last + + new = normalize_country(new) + + if old != new: + if old not in (rec.get("_reported", set())): + changes.append((rec["item_id"], rec["case_name"][:80], old, new)) + if not preview: + rec["respondent"] = new + if old == "UNKNOWN": + pass + else: + normalized += 1 + else: + unchanged += 1 + + if preview: + print(f"\n{'='*100}") + print(f"PREVIEW: {len(changes)} changes out of {len(records)} records") + print(f" Re-extracted from case_name: {reextracted}") + print(f" Normalized country name: {normalized}") + print(f" Unchanged: {unchanged}") + print(f"{'='*100}\n") + + by_type = {"UNKNOWN->": [], "V./AND->": [], "normalize": []} + for item_id, case_name, old, new in changes: + if old == "UNKNOWN": + by_type["UNKNOWN->"].append((item_id, case_name, old, new)) + elif old.startswith("V. ") or old.startswith("AND "): + by_type["V./AND->"].append((item_id, case_name, old, new)) + else: + by_type["normalize"].append((item_id, case_name, old, new)) + + for category, items in by_type.items(): + if not items: + continue + unique_mappings = Counter((old, new) for _, _, old, new in items) + print(f"\n--- {category} ({len(items)} records, {len(unique_mappings)} unique mappings) ---") + for (old, new), count in unique_mappings.most_common(): + print(f" {old:50s} -> {new:30s} ({count} records)") + + print(f"\n--- Resulting respondent distribution (top 30) ---") + result_counts = Counter() + for rec in records: + old = rec["respondent"] + for item_id, _, o, n in changes: + if item_id == rec["item_id"] and o == old: + result_counts[n] += 1 + break + else: + result_counts[normalize_country(old)] += 1 + + for country, count in result_counts.most_common(30): + print(f" {count:>6} {country}") + print(f" ... {len(result_counts)} total unique respondent values") + + return records + + +def main(): + parser = argparse.ArgumentParser(description="Clean respondent names in ECtHR dataset") + parser.add_argument("--input", help="Input JSON file") + parser.add_argument("--output", help="Output JSON file") + parser.add_argument("--preview", action="store_true", help="Preview changes without applying") + parser.add_argument("--apply", action="store_true", help="Apply changes") + parser.add_argument("--push", help="Push to HuggingFace dataset (e.g. overthelex/echr-verdict-free)") + args = parser.parse_args() + + if args.input: + print(f"Loading from {args.input}...") + with open(args.input) as f: + records = json.load(f) + else: + print("Loading from HuggingFace overthelex/echr-verdict-free...") + from datasets import load_dataset + ds = load_dataset("overthelex/echr-verdict-free", split="train") + records = [dict(r) for r in ds] + + print(f"Loaded {len(records)} records") + + if args.preview or not args.apply: + clean_dataset(records, preview=True) + return + + if args.apply: + clean_dataset(records, preview=False) + + result_counts = Counter(r["respondent"] for r in records) + print(f"\nAfter cleaning: {len(result_counts)} unique respondent values") + for country, count in result_counts.most_common(30): + print(f" {count:>6} {country}") + + if args.output: + with open(args.output, "w") as f: + json.dump(records, f, ensure_ascii=False, indent=2) + print(f"\nSaved to {args.output}") + + if args.push: + from datasets import Dataset + ds = Dataset.from_list(records) + ds.push_to_hub(args.push) + print(f"\nPushed to {args.push}") + + +if __name__ == "__main__": + main() diff --git a/scripts/conclusion_scrub.py b/scripts/conclusion_scrub.py new file mode 100644 index 0000000..f7713f2 --- /dev/null +++ b/scripts/conclusion_scrub.py @@ -0,0 +1,128 @@ +"""Sentence-level scrubber for verdict-conclusion leakage in ECHR texts. + +Stage-1 truncation (court_assessment_start / conclusory_pattern) misses +per-article conclusion sentences embedded in the Court's assessment, e.g. +"there has been no violation of Article 8 of the Convention." — a direct +gold-label leak. The 2026-07-03 audit measured own-article label leakage of +6.2% (livehrb-static-2k), 5.5% (livehrb-temporal-2k), 7.3% (echr-verdict-free) +and 2.8% (echr-ukr-verdict-free) before this scrub was applied to the +published data. This module makes the scrub a mandatory pipeline stage so +future live refreshes do not regress. + +Usage: + from conclusion_scrub import scrub_text, scrub_record, has_own_article_leak + + clean, n_dropped = scrub_text(text) + record = scrub_record(record) # dict with verdict_free_text etc. + assert not has_own_article_leak(clean, article) +""" + +from __future__ import annotations + +import re + +# Sentences matching any of these reveal the Court's own conclusion and must +# be dropped. Applied to ALL articles, not just the target one: multi-article +# cases share the same text across (case, article) pairs. +LEAK_SENT = re.compile( + r"(there has (accordingly |therefore )?been (a|no) violation of)" + r"|(there has (accordingly |therefore )?been (a|no) breach of)" + r"|(court (therefore |accordingly )?(concludes|finds|holds|considers)" + r" that there (has|have|had) been (a|no))" + r"|(finds? (that there is|there has been) (a|no) violation)" + r"|(does not find (a|any) (violation|breach))" + r"|(no violation of article \d+[\w\s./()-]* has (therefore )?occurred)" + r"|(unanimously,? that)" + r"|(discloses? (a|no) (violation|breach) of article)", + re.I, +) + +# ECHR paragraphs are numbered ("60. Accordingly, ..."), so split on sentence +# enders followed by whitespace + optional paragraph number + capital/quote. +SENT_SPLIT = re.compile(r"(?<=[.!?])\s+(?=(?:\d{1,3}\.\s+)?[A-Z“\"(])") + +SCRUB_MARKER = "+conclusion_scrub" + + +def scrub_text(text: str) -> tuple[str, int]: + """Drop conclusion sentences. Returns (clean_text, n_sentences_dropped).""" + kept, dropped = [], 0 + for sent in SENT_SPLIT.split(text): + if LEAK_SENT.search(sent): + dropped += 1 + else: + kept.append(sent) + return " ".join(kept), dropped + + +def has_own_article_leak(text: str, article) -> bool: + """True if the text states the verdict for the given target article.""" + pattern = re.compile( + r"there has (accordingly )?been (a|no) violation of Article\s+" + + re.escape(str(article)) + r"\b", + re.I, + ) + return bool(pattern.search(text)) + + +def scrub_record(rec: dict, text_key: str = "verdict_free_text") -> dict: + """Scrub one dataset record in place-compatible copy; updates + verdict_removal_method / verdict_free_length / retention_percentage.""" + rec = dict(rec) + clean, dropped = scrub_text(rec[text_key]) + if dropped: + rec[text_key] = clean + method = rec.get("verdict_removal_method", "") + if SCRUB_MARKER not in method: + rec["verdict_removal_method"] = method + SCRUB_MARKER + rec["verdict_free_length"] = len(clean) + if rec.get("original_length"): + rec["retention_percentage"] = round( + len(clean) / rec["original_length"] * 100, 1 + ) + return rec + + +def scrub_records(records: list[dict], text_key: str = "verdict_free_text", + verbose: bool = True) -> list[dict]: + """Scrub a list of records; asserts zero own-article leakage afterwards.""" + out = [scrub_record(r, text_key) for r in records] + touched = sum(1 for orig, r in zip(records, out) if r[text_key] != orig[text_key]) + leaks = sum( + 1 for r in out + if "article" in r and has_own_article_leak(r[text_key], r["article"]) + ) + if verbose: + print(f"[conclusion_scrub] scrubbed {touched}/{len(out)} records, " + f"own-article leaks after scrub: {leaks}") + assert leaks == 0, f"conclusion scrub left {leaks} own-article leaks" + return out + + +if __name__ == "__main__": + # self-test + leaky = ( + "57. The applicant complained under Article 8. " + "58. The Court considers the complaint admissible. " + "59. Accordingly, there has been no violation of Article 8 of the Convention. " + "60. The applicant also relied on Article 13." + ) + clean, dropped = scrub_text(leaky) + assert dropped == 1, dropped + assert "no violation of Article 8" not in clean + assert "Article 13" in clean and "admissible" in clean + assert has_own_article_leak(leaky, 8) and not has_own_article_leak(clean, 8) + + rec = scrub_record({ + "verdict_free_text": leaky, "article": "8", + "verdict_removal_method": "court_assessment_start", + "original_length": 1000, + }) + assert rec["verdict_removal_method"].endswith(SCRUB_MARKER) + assert rec["verdict_free_length"] == len(clean) + + kept_clean = "The applicant alleged a violation of Article 3. The facts are as follows." + same, zero = scrub_text(kept_clean) + assert zero == 0 and same == kept_clean + + print("conclusion_scrub self-test OK") diff --git a/scripts/enrich_and_push.py b/scripts/enrich_and_push.py new file mode 100644 index 0000000..eb434c8 --- /dev/null +++ b/scripts/enrich_and_push.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +""" +Backfill missing HUDOC dates for a verdict-free dataset, bake date columns in, +and (optionally) re-publish to the Hub. Reusable for echr-verdict-free and +echr-ukr-verdict-free. +""" +import argparse, json, os, sys, time +import requests +sys.path.insert(0, os.path.expanduser("~/strasbourgbench/scripts")) +from hudoc_scraper import search_hudoc, parse_search_result # noqa +from datasets import load_dataset + +def backfill(missing, dates, batch, sleep): + sess = requests.Session(); found = 0 + for b in range(0, len(missing), batch): + chunk = missing[b:b+batch] + q = 'contentsitename:ECHR AND (' + " OR ".join(f'itemid:"{i}"' for i in chunk) + ')' + try: + resp = search_hudoc(sess, q, 0, max(100, batch*2)) + except Exception as e: + print(" batch err", e); time.sleep(2); continue + for r in resp.get("results", []): + rec = parse_search_result(r); iid = rec.get("item_id") + if iid and rec.get("decision_date"): + dates[iid] = {"decision_date": rec["decision_date"], + "application_number": rec.get("application_number", ""), + "importance": rec.get("importance", ""), + "ecli": rec.get("ecli", "")} + found += 1 + time.sleep(sleep) + return found + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--dataset", required=True) + ap.add_argument("--dates", default=os.path.expanduser("~/strasbourgbench/data/processed/decision_dates.json")) + ap.add_argument("--batch", type=int, default=50) + ap.add_argument("--sleep", type=float, default=0.25) + ap.add_argument("--push", action="store_true") + a = ap.parse_args() + + dates = json.load(open(a.dates)) + ds = load_dataset(a.dataset, split="train") + uniq = sorted({r["item_id"] for r in ds}) + missing = [i for i in uniq if not (i in dates and dates[i].get("decision_date"))] + print(f"{a.dataset}: {len(ds)} pairs, {len(uniq)} cases, missing dates {len(missing)}") + if missing: + n = backfill(missing, dates, a.batch, a.sleep) + json.dump(dates, open(a.dates, "w"), ensure_ascii=False) + print(f" backfilled {n}") + + def add(r): + d = dates.get(r["item_id"], {}) + r["decision_date"] = d.get("decision_date", "") + r["application_number"] = d.get("application_number", "") + r["importance"] = d.get("importance", "") + r["ecli"] = d.get("ecli", "") + return r + ds2 = ds.map(add) + dated = sum(1 for x in ds2["decision_date"] if x) + print(f" dated pairs: {dated}/{len(ds2)} = {dated/len(ds2)*100:.1f}%") + if a.push: + ds2.push_to_hub(a.dataset, + commit_message="Add decision_date/application_number/importance/ecli (HUDOC backfill)") + print(" PUSHED", a.dataset) + +if __name__ == "__main__": + main() diff --git a/scripts/hudoc_live_refresh.py b/scripts/hudoc_live_refresh.py new file mode 100755 index 0000000..a5abe65 --- /dev/null +++ b/scripts/hudoc_live_refresh.py @@ -0,0 +1,570 @@ +#!/usr/bin/env python3 +""" +HUDOC Live Refresh Orchestrator for LiveHumanRightsBench + +Automated pipeline that: + 1. Determines the latest decision_date in the current dataset + 2. Downloads new judgments from HUDOC since that date + 3. Runs the verdict-leakage removal pipeline on new judgments + 4. Appends clean results to the existing dataset + 5. Optionally pushes the updated dataset to HuggingFace + +Designed to run as a cron job for continuous "live" benchmark updates. + +Usage: + # Basic refresh from existing dataset + python scripts/hudoc_live_refresh.py \ + --dataset data/processed/echr_cases_ukr_eng_final.json \ + --output data/processed/echr_live_updated.json + + # Refresh from HuggingFace dataset + python scripts/hudoc_live_refresh.py \ + --hf-dataset overthelex/echr-verdict-free \ + --output data/processed/echr_live_updated.json + + # Refresh with full pipeline (LLM verification) + python scripts/hudoc_live_refresh.py \ + --dataset data/processed/echr_cases_ukr_eng_final.json \ + --output data/processed/echr_live_updated.json \ + --full-pipeline + + # Refresh and push to HuggingFace + python scripts/hudoc_live_refresh.py \ + --dataset data/processed/echr_cases_ukr_eng_final.json \ + --output data/processed/echr_live_updated.json \ + --push-to-hf overthelex/echr-verdict-free + + # Filter to a specific country + python scripts/hudoc_live_refresh.py \ + --dataset data/processed/echr_cases_ukr_eng_final.json \ + --country UKR \ + --output data/processed/echr_ukr_live.json + + # Cron-friendly: quiet output, auto-detect dates + python scripts/hudoc_live_refresh.py \ + --dataset data/processed/echr_live.json \ + --output data/processed/echr_live.json \ + --quiet +""" + +import argparse +import json +import os +import subprocess +import sys +import tempfile +import time +from datetime import datetime, timedelta +from pathlib import Path + + +# ── Constants ────────────────────────────────────────────────────────────────── + +SCRIPTS_DIR = Path(__file__).parent.resolve() +SCRAPER_SCRIPT = SCRIPTS_DIR / "hudoc_scraper.py" +REMOVAL_SCRIPT = SCRIPTS_DIR / "verdict_leakage_removal.py" + + +# ── Dataset helpers ──────────────────────────────────────────────────────────── + +def load_dataset(path: str) -> list[dict]: + """Load existing dataset from a JSON file.""" + if not os.path.exists(path): + print(f" Dataset file not found: {path}") + return [] + with open(path) as f: + data = json.load(f) + print(f" Loaded {len(data)} existing records from {path}") + return data + + +def load_from_huggingface(dataset_name: str) -> list[dict]: + """Load dataset from HuggingFace Hub.""" + try: + from datasets import load_dataset + except ImportError: + print("ERROR: 'datasets' package required for HuggingFace loading.", file=sys.stderr) + print("Install with: pip install datasets", file=sys.stderr) + sys.exit(1) + + print(f" Loading dataset from HuggingFace: {dataset_name} ...") + ds = load_dataset(dataset_name, split="train") + records = [dict(row) for row in ds] + print(f" Loaded {len(records)} records from HuggingFace.") + return records + + +def get_latest_date(records: list[dict]) -> str: + """ + Find the latest decision_date in the dataset. + Returns date string in YYYY-MM-DD format, or empty string if none found. + """ + dates = [] + for r in records: + date_val = (r.get("decision_date") or r.get("date") + or r.get("kp_date") or r.get("judgment_date") or "") + date_str = str(date_val)[:10] if date_val else "" + if date_str and len(date_str) == 10: + try: + datetime.strptime(date_str, "%Y-%m-%d") + dates.append(date_str) + except ValueError: + continue + return max(dates) if dates else "" + + +def get_existing_item_ids(records: list[dict]) -> set[str]: + """Get set of item_ids already in the dataset.""" + return {r.get("item_id", "") for r in records if r.get("item_id")} + + +def deduplicate_records(existing: list[dict], new_records: list[dict]) -> list[dict]: + """ + Merge new records into existing, deduplicating by (item_id, article) pair. + New records take precedence over existing ones (fresher data). + """ + seen = set() + merged = [] + + # Index existing records + for r in existing: + key = (r.get("item_id", ""), r.get("article", "")) + if key not in seen: + seen.add(key) + merged.append(r) + + # Add new records, replacing existing duplicates + added = 0 + replaced = 0 + for r in new_records: + key = (r.get("item_id", ""), r.get("article", "")) + if key in seen: + # Replace: find and update the existing record + for i, existing_r in enumerate(merged): + if (existing_r.get("item_id"), existing_r.get("article")) == key: + merged[i] = r + replaced += 1 + break + else: + seen.add(key) + merged.append(r) + added += 1 + + print(f" Deduplication: {added} added, {replaced} replaced, {len(merged)} total") + return merged + + +# ── Pipeline orchestration ───────────────────────────────────────────────────── + +def run_scraper( + since: str, + until: str = "", + country: str = "", + output_path: str = "", + resume: bool = True, + delay: float = 1.0, +) -> list[dict]: + """ + Run the HUDOC scraper and return the downloaded records. + """ + cmd = [ + sys.executable, str(SCRAPER_SCRIPT), + "--since", since, + "--output", output_path, + "--delay", str(delay), + ] + if until: + cmd.extend(["--until", until]) + if country: + cmd.extend(["--country", country]) + if resume: + cmd.append("--resume") + + print(f"\n{'='*60}") + print(f"Step 1: Downloading new judgments from HUDOC") + print(f"{'='*60}") + print(f" Since: {since}") + print(f" Until: {until or 'now'}") + print(f" Country: {country or 'all'}") + print(f" Command: {' '.join(cmd)}") + + result = subprocess.run(cmd, capture_output=False, text=True) + + if result.returncode != 0: + print(f"\n ERROR: Scraper exited with code {result.returncode}", file=sys.stderr) + return [] + + # Load the scraper output + if os.path.exists(output_path): + with open(output_path) as f: + records = json.load(f) + return records + return [] + + +def run_verdict_removal( + input_path: str, + output_path: str, + stage1_only: bool = True, + workers: int = 5, + cutoff_date: str = "", + end_date: str = "", +) -> list[dict]: + """ + Run the verdict-leakage removal pipeline on the given input file. + """ + cmd = [ + sys.executable, str(REMOVAL_SCRIPT), + "--source", "json", + "--input", input_path, + "--output", output_path, + "--workers", str(workers), + ] + if stage1_only: + cmd.append("--stage1-only") + if cutoff_date: + cmd.extend(["--cutoff-date", cutoff_date]) + if end_date: + cmd.extend(["--end-date", end_date]) + + print(f"\n{'='*60}") + print(f"Step 2: Running verdict-leakage removal") + print(f"{'='*60}") + print(f" Mode: {'stage1 only (pattern-based)' if stage1_only else 'full pipeline (pattern + LLM)'}") + print(f" Input: {input_path}") + print(f" Output: {output_path}") + print(f" Command: {' '.join(cmd)}") + + result = subprocess.run(cmd, capture_output=False, text=True) + + if result.returncode != 0: + print(f"\n ERROR: Verdict removal exited with code {result.returncode}", file=sys.stderr) + return [] + + if os.path.exists(output_path): + with open(output_path) as f: + records = json.load(f) + return records + return [] + + +def push_to_huggingface(records: list[dict], repo_id: str, commit_message: str = ""): + """ + Push the updated dataset to HuggingFace Hub. + """ + try: + from datasets import Dataset + from huggingface_hub import HfApi + except ImportError: + print("ERROR: 'datasets' and 'huggingface_hub' packages required for HuggingFace push.", file=sys.stderr) + print("Install with: pip install datasets huggingface_hub", file=sys.stderr) + return False + + print(f"\n{'='*60}") + print(f"Step 4: Pushing to HuggingFace") + print(f"{'='*60}") + print(f" Repository: {repo_id}") + print(f" Records: {len(records)}") + + if not commit_message: + today = datetime.now().strftime("%Y-%m-%d") + dates = sorted(set(r.get("decision_date", "") for r in records if r.get("decision_date"))) + date_range = f"{dates[0]} to {dates[-1]}" if dates else "unknown" + commit_message = f"Live refresh {today}: {len(records)} records, dates {date_range}" + + # Prepare dataset - remove full_case_text to keep dataset size manageable + # (the HF dataset should have the verdict-free text, not the original) + columns_to_keep = [ + "item_id", "case_name", "article", "violation_label", + "full_case_text_no_verdict", "decision_date", "respondent", + "verdict_removal_method", "original_length", "verdict_free_length", + "retention_percentage", + ] + + clean_records = [] + for r in records: + clean = {k: r.get(k, "") for k in columns_to_keep} + # Rename for consistency + if "full_case_text_no_verdict" in clean and clean["full_case_text_no_verdict"]: + clean["case_text"] = clean.pop("full_case_text_no_verdict") + else: + clean["case_text"] = r.get("case_text", r.get("full_case_text", "")) + clean.pop("full_case_text_no_verdict", None) + clean_records.append(clean) + + try: + ds = Dataset.from_list(clean_records) + ds.push_to_hub(repo_id, commit_message=commit_message) + print(f" Pushed successfully: {commit_message}") + return True + except Exception as e: + print(f" ERROR pushing to HuggingFace: {e}", file=sys.stderr) + return False + + +# ── Main orchestrator ────────────────────────────────────────────────────────── + +def main(): + parser = argparse.ArgumentParser( + description="Live refresh orchestrator for LiveHumanRightsBench", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +This script automates the full "live" loop: + 1. Check the latest decision date in the current dataset + 2. Download new HUDOC judgments since that date + 3. Run verdict-leakage removal on new judgments + 4. Merge clean results into the existing dataset + 5. Optionally push to HuggingFace + +For cron usage: + # Run daily at 06:00 UTC + 0 6 * * * cd /path/to/Legal-Sycophancy && python scripts/hudoc_live_refresh.py \\ + --dataset data/processed/echr_live.json \\ + --output data/processed/echr_live.json \\ + --quiet >> logs/live_refresh.log 2>&1 +""", + ) + + # Input sources + input_group = parser.add_mutually_exclusive_group(required=True) + input_group.add_argument( + "--dataset", type=str, + help="Path to existing dataset JSON file", + ) + input_group.add_argument( + "--hf-dataset", type=str, + help="HuggingFace dataset name (e.g., overthelex/echr-verdict-free)", + ) + + # Output + parser.add_argument( + "--output", required=True, + help="Output path for the updated dataset", + ) + + # Filtering + parser.add_argument( + "--country", default="", + help="Filter by respondent state (e.g., UKR)", + ) + parser.add_argument( + "--since-override", default="", + help="Override auto-detected start date (YYYY-MM-DD). Useful for initial seeding.", + ) + + # Pipeline options + parser.add_argument( + "--full-pipeline", action="store_true", + help="Run full verdict removal (pattern + LLM). Default is stage1 only.", + ) + parser.add_argument( + "--workers", type=int, default=5, + help="Number of parallel workers for LLM verification (default: 5)", + ) + parser.add_argument( + "--delay", type=float, default=1.0, + help="Delay between HUDOC requests in seconds (default: 1.0)", + ) + + # HuggingFace push + parser.add_argument( + "--push-to-hf", default="", + help="Push updated dataset to this HuggingFace repo (e.g., overthelex/echr-verdict-free)", + ) + + # Misc + parser.add_argument( + "--quiet", action="store_true", + help="Minimal output (for cron jobs)", + ) + parser.add_argument( + "--keep-temp", action="store_true", + help="Keep temporary files (raw downloads, intermediate results)", + ) + + args = parser.parse_args() + + start_time = time.time() + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + + if not args.quiet: + print(f"{'='*60}") + print(f" LiveHumanRightsBench - HUDOC Live Refresh") + print(f" Started: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + print(f"{'='*60}") + + # ── Step 0: Load existing dataset ────────────────────────────────────── + + if args.dataset: + existing_records = load_dataset(args.dataset) + else: + existing_records = load_from_huggingface(args.hf_dataset) + + # ── Step 0.5: Determine date range ───────────────────────────────────── + + if args.since_override: + since_date = args.since_override + print(f" Using override start date: {since_date}") + elif existing_records: + latest = get_latest_date(existing_records) + if latest: + # Start from the day after the latest known date to avoid re-downloading + # the same cases. But subtract 1 day as a safety margin for HUDOC + # indexing lag. + try: + latest_dt = datetime.strptime(latest, "%Y-%m-%d") + since_dt = latest_dt - timedelta(days=1) + since_date = since_dt.strftime("%Y-%m-%d") + except ValueError: + since_date = latest + print(f" Latest date in dataset: {latest}") + print(f" Fetching since: {since_date} (1-day overlap for safety)") + else: + print(" WARNING: No dates found in existing dataset. Using 2020-01-01 as default.", file=sys.stderr) + since_date = "2020-01-01" + else: + print(" No existing dataset. Using 2020-01-01 as default start date.") + since_date = "2020-01-01" + + # Check if there's anything new to fetch + today = datetime.now().strftime("%Y-%m-%d") + if since_date >= today: + print(f"\n Dataset is already up to date (latest: {since_date}, today: {today})") + print(f" Nothing to do.") + return + + # ── Step 1: Download new judgments ────────────────────────────────────── + + # Use temp directory for intermediate files + temp_dir = tempfile.mkdtemp(prefix="hudoc_refresh_") + raw_path = os.path.join(temp_dir, f"hudoc_raw_{timestamp}.json") + clean_path = os.path.join(temp_dir, f"hudoc_clean_{timestamp}.json") + + scraped_records = run_scraper( + since=since_date, + country=args.country, + output_path=raw_path, + resume=True, + delay=args.delay, + ) + + if not scraped_records: + print("\n No new judgments found. Dataset is up to date.") + if not args.keep_temp: + _cleanup_temp(temp_dir) + return + + # Filter out records we already have + existing_ids = get_existing_item_ids(existing_records) + truly_new = [r for r in scraped_records if r.get("item_id") not in existing_ids] + + if not truly_new: + print(f"\n All {len(scraped_records)} scraped records already exist in dataset.") + print(f" Dataset is up to date.") + if not args.keep_temp: + _cleanup_temp(temp_dir) + return + + print(f"\n New records to process: {truly_new_count(truly_new, existing_ids)}") + + # Save the truly new records for verdict removal + new_raw_path = os.path.join(temp_dir, f"hudoc_new_{timestamp}.json") + with open(new_raw_path, "w") as f: + json.dump(truly_new, f, ensure_ascii=False) + + # ── Step 2: Run verdict-leakage removal ──────────────────────────────── + + clean_records = run_verdict_removal( + input_path=new_raw_path, + output_path=clean_path, + stage1_only=not args.full_pipeline, + workers=args.workers, + ) + + if not clean_records: + print("\n WARNING: Verdict removal produced no clean records.", file=sys.stderr) + if not args.keep_temp: + _cleanup_temp(temp_dir) + return + + # ── Step 3: Merge with existing dataset ──────────────────────────────── + + print(f"\n{'='*60}") + print(f"Step 3: Merging datasets") + print(f"{'='*60}") + print(f" Existing records: {len(existing_records)}") + print(f" New clean records: {len(clean_records)}") + + merged = deduplicate_records(existing_records, clean_records) + + # Sort by decision_date descending + merged.sort( + key=lambda r: r.get("decision_date", "") or "0000-00-00", + reverse=True, + ) + + # Save merged dataset + os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True) + with open(args.output, "w") as f: + json.dump(merged, f, indent=2, ensure_ascii=False) + + print(f" Saved {len(merged)} records to {args.output}") + + # ── Step 4 (optional): Push to HuggingFace ───────────────────────────── + + if args.push_to_hf: + push_to_huggingface(merged, args.push_to_hf) + + # ── Cleanup ──────────────────────────────────────────────────────────── + + if not args.keep_temp: + _cleanup_temp(temp_dir) + + # ── Summary ──────────────────────────────────────────────────────────── + + elapsed = time.time() - start_time + + print(f"\n{'='*60}") + print(f" Live Refresh Complete") + print(f"{'='*60}") + print(f" Elapsed: {elapsed:.0f}s ({elapsed/60:.1f}m)") + print(f" Existing records: {len(existing_records)}") + print(f" New records added: {len(merged) - len(existing_records)}") + print(f" Total records: {len(merged)}") + + # Date range + dates = sorted(set(r.get("decision_date", "") for r in merged if r.get("decision_date"))) + if dates: + print(f" Date range: {dates[0]} to {dates[-1]}") + + # Per-country breakdown (top 10) + from collections import Counter + countries = Counter(r.get("respondent", "?") for r in merged) + labels = Counter(r.get("violation_label", "?") for r in merged) + print(f" Violation labels: {dict(labels)}") + if len(countries) <= 10: + print(f" Countries: {dict(countries.most_common())}") + else: + top10 = countries.most_common(10) + print(f" Top 10 countries: {dict(top10)}") + print(f" Total countries: {len(countries)}") + + print(f"\n Output: {args.output}") + + +def truly_new_count(records: list[dict], existing_ids: set[str]) -> int: + """Count records whose item_id is not in existing_ids.""" + return sum(1 for r in records if r.get("item_id") not in existing_ids) + + +def _cleanup_temp(temp_dir: str): + """Remove temporary directory and its contents.""" + import shutil + try: + shutil.rmtree(temp_dir) + except OSError: + pass + + +if __name__ == "__main__": + main() diff --git a/scripts/hudoc_scraper.py b/scripts/hudoc_scraper.py new file mode 100755 index 0000000..a1e4d9f --- /dev/null +++ b/scripts/hudoc_scraper.py @@ -0,0 +1,664 @@ +#!/usr/bin/env python3 +""" +HUDOC Bulk Downloader for ECtHR Judgments + +Downloads judgment texts from the HUDOC public API (hudoc.echr.coe.int), +extracts metadata, and outputs JSON compatible with the verdict_leakage_removal +pipeline. + +Usage: + # Fetch all English judgments since 2025-01-01 + python scripts/hudoc_scraper.py --since 2025-01-01 --output data/raw/hudoc_2025.json + + # Fetch Ukrainian cases only + python scripts/hudoc_scraper.py --since 2024-01-01 --country UKR --output data/raw/hudoc_ukr_2024.json + + # Fetch a specific date range + python scripts/hudoc_scraper.py --since 2024-01-01 --until 2025-01-01 --output data/raw/hudoc_2024.json + + # Resume a previous download (skip already-downloaded item_ids) + python scripts/hudoc_scraper.py --since 2024-01-01 --output data/raw/hudoc_2024.json --resume + + # Dry run (metadata only, no full text download) + python scripts/hudoc_scraper.py --since 2025-01-01 --dry-run --output data/raw/hudoc_meta.json + +API reference: + Search endpoint: https://hudoc.echr.coe.int/app/query/results + Document body: https://hudoc.echr.coe.int/app/conversion/docx/html/body?library=ECHR&id={item_id} +""" + +import argparse +import json +import os +import re +import sys +import time +from dataclasses import dataclass, asdict, field +from datetime import datetime, date +from html.parser import HTMLParser +from typing import Optional +from urllib.parse import quote + +try: + import requests +except ImportError: + print("ERROR: 'requests' package is required. Install with: pip install requests", file=sys.stderr) + sys.exit(1) + + +# ── Configuration ────────────────────────────────────────────────────────────── + +HUDOC_SEARCH_URL = "https://hudoc.echr.coe.int/app/query/results" +HUDOC_DOC_BODY_URL = "https://hudoc.echr.coe.int/app/conversion/docx/html/body" + +# Fields to retrieve from the search API +SEARCH_FIELDS = [ + "itemid", "docname", "appno", "article", "conclusion", + "kpdate", "respondent", "respondentOrderEng", + "languageisocode", "importance", "ecli", + "doctypebranch", "separateopinion", +] + +PAGE_SIZE = 50 # HUDOC returns up to 500 per page; use 50 for politeness +REQUEST_DELAY = 1.0 # seconds between requests (be polite) +MAX_RETRIES = 3 +RETRY_BACKOFF = 5.0 # seconds, multiplied by attempt number + +# User-Agent header for polite scraping +HEADERS = { + "User-Agent": "LiveHumanRightsBench/1.0 (academic research; https://github.com/overthelex/Legal-Sycophancy)", + "Accept": "application/json", +} + + +# ── HTML to text conversion ─────────────────────────────────────────────────── + +class HTMLTextExtractor(HTMLParser): + """Simple HTML-to-text converter that preserves paragraph structure.""" + + def __init__(self): + super().__init__() + self._text_parts = [] + self._skip = False + + def handle_starttag(self, tag, attrs): + if tag in ("script", "style"): + self._skip = True + elif tag in ("p", "div", "br", "h1", "h2", "h3", "h4", "h5", "h6", "li", "tr"): + self._text_parts.append("\n") + + def handle_endtag(self, tag): + if tag in ("script", "style"): + self._skip = False + elif tag in ("p", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "tr", "table"): + self._text_parts.append("\n") + elif tag == "td": + self._text_parts.append(" ") + + def handle_data(self, data): + if not self._skip: + self._text_parts.append(data) + + def get_text(self) -> str: + raw = "".join(self._text_parts) + # Collapse multiple blank lines into two newlines + raw = re.sub(r"\n{3,}", "\n\n", raw) + # Collapse multiple spaces within lines + lines = [re.sub(r"[ \t]+", " ", line.strip()) for line in raw.split("\n")] + return "\n".join(lines).strip() + + +def html_to_text(html: str) -> str: + """Convert HTML to clean plain text.""" + parser = HTMLTextExtractor() + parser.feed(html) + return parser.get_text() + + +# ── HUDOC API client ────────────────────────────────────────────────────────── + +def build_search_query( + since: str = "", + until: str = "", + country: str = "", + language: str = "ENG", + doc_collection: str = "JUDGMENTS", +) -> str: + """ + Build a HUDOC search query string. + + The query uses HUDOC's custom query language: + - contentsitename:ECHR -- search the ECHR library + - documentcollectionid:"JUDGMENTS" -- only judgments + - languageisocode:"ENG" -- English language + - respondent:"UKR" -- respondent state + - kpdate range filters + """ + parts = [ + 'contentsitename:ECHR', + '(NOT (doctype:PR OR doctype:HFCOMOLD OR doctype:HECOMOLD))', + ] + + if language: + parts.append(f'((languageisocode:"{language}"))') + + if doc_collection: + parts.append(f'((documentcollectionid:"{doc_collection}"))') + + if country: + parts.append(f'((respondent:"{country.upper()}"))') + + # Date range filter using kpdate (Lucene range syntax) + if since or until: + range_start = f"{since}T00:00:00" if since else "*" + range_end = f"{until}T00:00:00" if until else "*" + parts.append(f'((kpdate:[{range_start} TO {range_end}]))') + + return " AND ".join(parts) + + +def search_hudoc( + session: requests.Session, + query: str, + start: int = 0, + length: int = PAGE_SIZE, +) -> dict: + """ + Execute a search query against the HUDOC API. + + Returns the raw JSON response dict with keys: + - resultcount: total number of matching results + - results: list of result items + """ + params = { + "query": query, + "select": ",".join(SEARCH_FIELDS), + "sort": "kpdate Descending", + "start": str(start), + "length": str(length), + } + + for attempt in range(1, MAX_RETRIES + 1): + try: + resp = session.get(HUDOC_SEARCH_URL, params=params, headers=HEADERS, timeout=60) + resp.raise_for_status() + return resp.json() + except requests.exceptions.RequestException as e: + if attempt == MAX_RETRIES: + raise + wait = RETRY_BACKOFF * attempt + print(f" [retry {attempt}/{MAX_RETRIES}] Search request failed: {e}. Waiting {wait}s...") + time.sleep(wait) + + return {"resultcount": 0, "results": []} + + +def fetch_document_text(session: requests.Session, item_id: str) -> Optional[str]: + """ + Fetch the full text of a HUDOC document by its item_id. + + Downloads the HTML body and converts to plain text. + Returns None on failure. + """ + url = f"{HUDOC_DOC_BODY_URL}?library=ECHR&id={item_id}" + + for attempt in range(1, MAX_RETRIES + 1): + try: + resp = session.get(url, headers=HEADERS, timeout=120) + resp.raise_for_status() + text = html_to_text(resp.text) + if text and len(text) > 100: + return text + return None + except requests.exceptions.RequestException as e: + if attempt == MAX_RETRIES: + print(f" [FAIL] Could not fetch document {item_id}: {e}") + return None + wait = RETRY_BACKOFF * attempt + print(f" [retry {attempt}/{MAX_RETRIES}] Doc fetch failed for {item_id}: {e}. Waiting {wait}s...") + time.sleep(wait) + + return None + + +# ── Result parsing ───────────────────────────────────────────────────────────── + +def parse_search_result(result: dict) -> dict: + """ + Parse a single HUDOC search result into a flat metadata dict. + + Input: {"columns": {"itemid": "001-...", "docname": "CASE OF ...", ...}} + Output: flat dict with normalized field names + """ + cols = result.get("columns", {}) + + # Parse kpdate + kpdate_raw = cols.get("kpdate", "") + decision_date = "" + if kpdate_raw: + try: + decision_date = kpdate_raw[:10] # "2026-06-09T00:00:00" -> "2026-06-09" + except (ValueError, IndexError): + decision_date = "" + + # Parse articles (semicolon-separated) + articles_raw = cols.get("article", "") + articles = [a.strip() for a in articles_raw.split(";") if a.strip()] if articles_raw else [] + + # Parse application numbers + appno = cols.get("appno", "") + + return { + "item_id": cols.get("itemid", ""), + "case_name": cols.get("docname", ""), + "application_number": appno, + "decision_date": decision_date, + "respondent": cols.get("respondent", ""), + "articles": articles, + "conclusion": cols.get("conclusion", ""), + "language": cols.get("languageisocode", ""), + "importance": cols.get("importance", ""), + "ecli": cols.get("ecli", ""), + "doc_type_branch": cols.get("doctypebranch", ""), + "separate_opinion": cols.get("separateopinion", ""), + } + + +def parse_article_code(part: str) -> tuple[Optional[str], Optional[str]]: + """ + Extract the article from a conclusion fragment, returning + (article, article_full). + + - article : legacy collapsed code (bare number, kept for continuity) + - article_full : protocol-aware code (e.g. "P1-1", "P4-2"), never lossy + + "Article 1 of Protocol No. 1" -> ("1", "P1-1") (property, NOT Convention Art 1) + "Article 2 of Protocol No. 4" -> ("2", "P4-2") + "P1-1" -> ("1", "P1-1") + "Article 6" -> ("6", "6") + """ + # "Article of Protocol No. " + m = re.search(r'Article\s+(\d+)\s+of\s+Protocol\s+(?:No\.?\s*)?(\d+)', + part, re.IGNORECASE) + if m: + art, proto = m.group(1), m.group(2) + return art, f"P{proto}-{art}" + # Already-coded thesaurus form "P-" + m = re.search(r'\bP(\d+)-(\d+)\b', part) + if m: + proto, art = m.group(1), m.group(2) + return art, f"P{proto}-{art}" + # Plain Convention article "Article " + m = re.search(r'Article\s+(\d+)', part, re.IGNORECASE) + if m: + return m.group(1), m.group(1) + return None, None + + +def parse_conclusion_to_pairs(conclusion: str, item_id: str, case_name: str, + decision_date: str, respondent: str, + full_text: str) -> list[dict]: + """ + Parse the HUDOC conclusion string into case-article pairs, + matching the format expected by verdict_leakage_removal.py. + + Each pair has the full CaseRecord-compatible fields: + item_id, case_name, article, article_full, violation_label, + full_case_text, decision_date + """ + results = [] + if not conclusion: + # No conclusion available - still include the case as "unknown" + results.append({ + "item_id": item_id, + "case_name": case_name, + "article": "", + "article_full": "", + "violation_label": "unknown", + "full_case_text": full_text, + "decision_date": decision_date, + "respondent": respondent, + }) + return results + + parts = re.split(r'[;]', conclusion) + for part in parts: + part = part.strip() + article, article_full = parse_article_code(part) + if not article: + continue + + if re.search(r'no\s+violation', part, re.IGNORECASE): + label = "no_violation" + elif re.search(r'violation', part, re.IGNORECASE): + label = "violation" + else: + continue + + results.append({ + "item_id": item_id, + "case_name": case_name, + "article": article, + "article_full": article_full, + "violation_label": label, + "full_case_text": full_text, + "decision_date": decision_date, + "respondent": respondent, + }) + + # If no article-violation pairs were parsed, include as unknown + if not results and full_text: + results.append({ + "item_id": item_id, + "case_name": case_name, + "article": "", + "article_full": "", + "violation_label": "unknown", + "full_case_text": full_text, + "decision_date": decision_date, + "respondent": respondent, + }) + + return results + + +# ── Main scraper ─────────────────────────────────────────────────────────────── + +def load_existing_item_ids(output_path: str) -> set[str]: + """Load item_ids from an existing output file for resume capability.""" + if not os.path.exists(output_path): + return set() + try: + with open(output_path) as f: + data = json.load(f) + ids = {item.get("item_id", "") for item in data} + return ids + except (json.JSONDecodeError, IOError): + return set() + + +def scrape_hudoc( + since: str = "", + until: str = "", + country: str = "", + language: str = "ENG", + output_path: str = "data/raw/hudoc_cases.json", + resume: bool = False, + dry_run: bool = False, + page_size: int = PAGE_SIZE, + delay: float = REQUEST_DELAY, +) -> list[dict]: + """ + Main scraping function. + + 1. Query HUDOC search API to get metadata for matching judgments + 2. Download full text for each judgment + 3. Parse conclusions into case-article pairs + 4. Output JSON compatible with verdict_leakage_removal.py + + Returns list of case-article pair dicts. + """ + session = requests.Session() + + # Build search query + query = build_search_query( + since=since, until=until, country=country, language=language, + ) + + # Load existing IDs for resume + existing_ids = set() + existing_records = [] + if resume: + existing_ids = load_existing_item_ids(output_path) + if existing_ids: + print(f"Resume mode: found {len(existing_ids)} already-downloaded item_ids") + try: + with open(output_path) as f: + existing_records = json.load(f) + except (json.JSONDecodeError, IOError): + existing_records = [] + + # Phase 1: Search for matching judgments + print(f"\nSearching HUDOC...") + print(f" Query: {query}") + print(f" Date range: {since or 'any'} to {until or 'any'}") + print(f" Country: {country or 'all'}") + print(f" Language: {language}") + + first_page = search_hudoc(session, query, start=0, length=page_size) + total_count = first_page.get("resultcount", 0) + print(f" Total matching judgments: {total_count:,}") + + if total_count == 0: + print(" No judgments found. Exiting.") + return [] + + # Collect all metadata + all_metadata = [] + results = first_page.get("results", []) + for r in results: + meta = parse_search_result(r) + if meta["item_id"]: + all_metadata.append(meta) + + # Paginate through remaining results + fetched = len(results) + while fetched < total_count: + time.sleep(delay) + page = search_hudoc(session, query, start=fetched, length=page_size) + results = page.get("results", []) + if not results: + break + for r in results: + meta = parse_search_result(r) + if meta["item_id"]: + all_metadata.append(meta) + fetched += len(results) + print(f"\r Fetched metadata: {fetched:,}/{total_count:,}", end="", flush=True) + + print(f"\n Collected metadata for {len(all_metadata)} judgments") + + # Filter out already-downloaded + if existing_ids: + before = len(all_metadata) + all_metadata = [m for m in all_metadata if m["item_id"] not in existing_ids] + skipped = before - len(all_metadata) + print(f" Skipping {skipped} already-downloaded judgments") + print(f" New judgments to download: {len(all_metadata)}") + + if dry_run: + print("\n [DRY RUN] Skipping full text download. Outputting metadata only.") + # Output metadata without full text + output_records = [] + for meta in all_metadata: + output_records.append({ + "item_id": meta["item_id"], + "case_name": meta["case_name"], + "article": ";".join(meta["articles"]), + "violation_label": "", + "full_case_text": "", + "decision_date": meta["decision_date"], + "respondent": meta["respondent"], + "application_number": meta["application_number"], + "conclusion": meta["conclusion"], + "ecli": meta["ecli"], + }) + return existing_records + output_records + + # Phase 2: Download full text for each judgment + print(f"\nDownloading full texts ({len(all_metadata)} documents)...") + all_case_pairs = list(existing_records) + downloaded = 0 + failed = 0 + skipped_short = 0 + + for i, meta in enumerate(all_metadata): + item_id = meta["item_id"] + + # Rate limiting + if i > 0: + time.sleep(delay) + + # Download full text + full_text = fetch_document_text(session, item_id) + downloaded += 1 + + if not full_text: + failed += 1 + print(f"\r Progress: {i+1}/{len(all_metadata)} | Downloaded: {downloaded} | Failed: {failed}", end="", flush=True) + continue + + if len(full_text) < 500: + skipped_short += 1 + print(f"\r Progress: {i+1}/{len(all_metadata)} | Downloaded: {downloaded} | Short: {skipped_short}", end="", flush=True) + continue + + # Parse into case-article pairs + pairs = parse_conclusion_to_pairs( + conclusion=meta["conclusion"], + item_id=item_id, + case_name=meta["case_name"], + decision_date=meta["decision_date"], + respondent=meta["respondent"], + full_text=full_text, + ) + all_case_pairs.extend(pairs) + + if (i + 1) % 10 == 0 or (i + 1) == len(all_metadata): + print(f"\r Progress: {i+1}/{len(all_metadata)} | Pairs: {len(all_case_pairs)} | Failed: {failed} | Short: {skipped_short} ", end="", flush=True) + + # Checkpoint: save every 100 documents + if (i + 1) % 100 == 0: + _save_checkpoint(all_case_pairs, output_path) + + print() + + # Final stats + new_pairs = len(all_case_pairs) - len(existing_records) + print(f"\n Download complete:") + print(f" Total documents attempted: {len(all_metadata)}") + print(f" Failed downloads: {failed}") + print(f" Skipped (too short): {skipped_short}") + print(f" New case-article pairs: {new_pairs}") + print(f" Total case-article pairs: {len(all_case_pairs)}") + + return all_case_pairs + + +def _save_checkpoint(records: list[dict], output_path: str): + """Save intermediate checkpoint to disk.""" + os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True) + checkpoint_path = output_path + ".checkpoint" + with open(checkpoint_path, "w") as f: + json.dump(records, f, ensure_ascii=False) + os.replace(checkpoint_path, output_path) + + +# ── CLI ──────────────────────────────────────────────────────────────────────── + +def main(): + parser = argparse.ArgumentParser( + description="Download ECtHR judgments from HUDOC for the LiveHumanRightsBench pipeline", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + %(prog)s --since 2025-01-01 --output data/raw/hudoc_2025.json + %(prog)s --since 2024-01-01 --until 2025-01-01 --country UKR --output data/raw/hudoc_ukr_2024.json + %(prog)s --since 2025-06-01 --dry-run --output data/raw/hudoc_meta.json + %(prog)s --since 2025-01-01 --output data/raw/hudoc_2025.json --resume +""", + ) + parser.add_argument( + "--since", required=True, + help="Fetch judgments from this date onwards (YYYY-MM-DD)", + ) + parser.add_argument( + "--until", default="", + help="Fetch judgments before this date (YYYY-MM-DD, exclusive)", + ) + parser.add_argument( + "--country", default="", + help="Respondent state ISO code (e.g., UKR, GBR, FRA). Empty = all countries", + ) + parser.add_argument( + "--language", default="ENG", + help="Language ISO code (default: ENG)", + ) + parser.add_argument( + "--output", default="data/raw/hudoc_cases.json", + help="Output JSON file path", + ) + parser.add_argument( + "--resume", action="store_true", + help="Resume: skip item_ids already present in the output file", + ) + parser.add_argument( + "--dry-run", action="store_true", + help="Metadata only - do not download full text", + ) + parser.add_argument( + "--page-size", type=int, default=PAGE_SIZE, + help=f"Number of results per search page (default: {PAGE_SIZE})", + ) + parser.add_argument( + "--delay", type=float, default=REQUEST_DELAY, + help=f"Delay between requests in seconds (default: {REQUEST_DELAY})", + ) + + args = parser.parse_args() + + # Validate dates + for date_arg, name in [(args.since, "--since"), (args.until, "--until")]: + if date_arg: + try: + datetime.strptime(date_arg, "%Y-%m-%d") + except ValueError: + print(f"ERROR: {name} must be in YYYY-MM-DD format, got: {date_arg}", file=sys.stderr) + sys.exit(1) + + start_time = time.time() + + records = scrape_hudoc( + since=args.since, + until=args.until, + country=args.country, + language=args.language, + output_path=args.output, + resume=args.resume, + dry_run=args.dry_run, + page_size=args.page_size, + delay=args.delay, + ) + + if records: + # Save final output + os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True) + with open(args.output, "w") as f: + json.dump(records, f, indent=2, ensure_ascii=False) + + elapsed = time.time() - start_time + print(f"\nSaved {len(records)} case-article pairs to {args.output}") + print(f"Elapsed time: {elapsed:.0f}s ({elapsed/60:.1f}m)") + + # Summary stats + from collections import Counter + countries = Counter(r.get("respondent", "?") for r in records) + labels = Counter(r.get("violation_label", "?") for r in records) + dates = sorted(set(r.get("decision_date", "") for r in records if r.get("decision_date"))) + + print(f"\nSummary:") + print(f" Unique cases: {len(set(r['item_id'] for r in records))}") + print(f" Case-article pairs: {len(records)}") + if dates: + print(f" Date range: {dates[0]} to {dates[-1]}") + print(f" Violation labels: {dict(labels)}") + if len(countries) <= 20: + print(f" Countries: {dict(countries.most_common())}") + else: + print(f" Countries: {len(countries)} ({', '.join(c for c, _ in countries.most_common(10))}, ...)") + else: + print("\nNo records to save.") + + +if __name__ == "__main__": + main() diff --git a/scripts/verdict_leakage_removal.py b/scripts/verdict_leakage_removal.py new file mode 100644 index 0000000..f1d09ca --- /dev/null +++ b/scripts/verdict_leakage_removal.py @@ -0,0 +1,630 @@ +#!/usr/bin/env python3 +""" +Verdict-Leakage Removal Pipeline for ECtHR Cases + +Three-stage pipeline as described in the paper: + 1. Pattern-based truncation -- remove text after markers introducing the Court's assessment + 2. Dual LLM verification -- two models independently check for remaining verdict leakage + 3. Repair or exclude -- fix leaking cases or drop those that are too short + +Usage: + # From HUDOC database (prod) + python scripts/verdict_leakage_removal.py \ + --source db \ + --db "dbname=secondlayer_prod user=secondlayer host=172.18.0.13" \ + --country UKR \ + --output data/processed/echr_cases_ukr_clean.json \ + --workers 10 + + # From existing JSON (HUDOC export) + python scripts/verdict_leakage_removal.py \ + --source json \ + --input data/raw/echr_cases_ukr.json \ + --output data/processed/echr_cases_ukr_clean.json + + # Dry run (stage 1 only, no LLM calls) + python scripts/verdict_leakage_removal.py \ + --source db --country UKR --stage1-only --output /dev/stdout + + # Time-windowed extraction ("Live" benchmark support) + python scripts/verdict_leakage_removal.py \ + --source db --country UKR \ + --cutoff-date 2024-01-01 --end-date 2025-01-01 \ + --output data/processed/echr_cases_ukr_2024.json +""" + +import argparse +import json +import os +import re +import sys +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, asdict +from typing import Optional + +try: + import openai +except ImportError: + openai = None + +try: + import boto3 +except ImportError: + boto3 = None + +from conclusion_scrub import scrub_text, SCRUB_MARKER + +# ── Stage 1: Pattern-based truncation ──────────────────────────────────────── + +# Markers that introduce the Court's own assessment/conclusions. +# Text at or after these markers is removed. +COURT_ASSESSMENT_MARKERS = [ + # Section headers (uppercase, often standalone) + r"^[IVX]+\.\s*THE COURT['’]?S ASSESSMENT", + r"^[IVX]+\.\s*MERITS", + r"^[IVX]+\.\s*THE MERITS", + r"^THE COURT['’]?S ASSESSMENT", + r"^MERITS$", + # Operative provisions and final sections + r"^FOR THESE REASONS", + r"^OPERATIVE PROVISIONS", + # Article 41 (just satisfaction) -- always post-verdict + r"^[IVX]+\.\s*APPLICATION OF ARTICLE 41", + r"^APPLICATION OF ARTICLE 41", + r"^ARTICLE 41", + # Common assessment openers within paragraphs + r"The Court will now assess", + r"The Court will first examine", + r"The Court must therefore determine", + r"The Court will examine the merits", + r"The Court notes at the outset that", + r"The Court observes that the parties", + r"Turning to the merits", + r"As to the merits", +] + +# Conclusory patterns that directly reveal the verdict +CONCLUSORY_PATTERNS = [ + r"(?:finds|concludes|holds)\s+that\s+there\s+has\s+(?:been|not been)\s+(?:a\s+)?violation", + r"(?:finds|concludes|holds)\s+that\s+there\s+has\s+(?:been|not been)\s+(?:a\s+)?breach", + r"there\s+has\s+(?:been|not been)\s+(?:a\s+)?violation\s+of\s+Article", + r"there\s+has\s+(?:been|not been)\s+(?:a\s+)?breach\s+of\s+Article", + r"no\s+violation\s+of\s+Article", + r"violated?\s+Article", + r"Accordingly,?\s+the\s+Court\s+(?:finds|concludes|holds)", + r"It\s+follows\s+that\s+there\s+has\s+(?:been|not been)", + r"In\s+the\s+light\s+of\s+the\s+above.*(?:finds|concludes)", + r"The\s+Court\s+therefore\s+concludes", + r"dismisses?\s+the\s+(?:application|complaint)", + r"Holds\s+that", + r"Decides\s+that", +] + +COMPILED_MARKERS = [re.compile(m, re.IGNORECASE | re.MULTILINE) for m in COURT_ASSESSMENT_MARKERS] +COMPILED_CONCLUSORY = [re.compile(p, re.IGNORECASE) for p in CONCLUSORY_PATTERNS] + +MIN_VERDICT_FREE_LENGTH = 500 + + +@dataclass +class CaseRecord: + item_id: str + case_name: str + article: str + violation_label: str + full_case_text: str + decision_date: str = "" + respondent: str = "" + full_case_text_no_verdict: str = "" + verdict_removal_method: str = "" + original_length: int = 0 + verdict_free_length: int = 0 + retention_percentage: float = 0.0 + leakage_check_primary: str = "" + leakage_check_secondary: str = "" + excluded: bool = False + exclusion_reason: str = "" + + +def stage1_truncate(text: str) -> tuple[str, str]: + """ + Pattern-based truncation: find the earliest Court assessment marker + and truncate everything from that point onward. + + Returns (truncated_text, method_used). + """ + earliest_pos = len(text) + method = "no_truncation" + + for pattern in COMPILED_MARKERS: + match = pattern.search(text) + if match and match.start() < earliest_pos: + earliest_pos = match.start() + method = f"court_assessment_start" + + if earliest_pos < len(text): + truncated = text[:earliest_pos].rstrip() + return truncated, method + + # Fallback: look for conclusory patterns and truncate before the sentence + for pattern in COMPILED_CONCLUSORY: + match = pattern.search(text) + if match: + sentence_start = text.rfind('.', 0, match.start()) + if sentence_start > 0: + truncated = text[:sentence_start + 1].rstrip() + return truncated, "aggressive_fix: conclusory_pattern" + + return text, "no_truncation" + + +# ── Stage 2: LLM-based verification ───────────────────────────────────────── + +LEAKAGE_CHECK_PROMPT = """You are a legal expert reviewing a case document from the European Court of Human Rights (ECtHR). + +The following text is supposed to contain ONLY the factual background, procedural history, parties' arguments, cited legal standards, and relevant domestic law. It should NOT contain: +- The Court's own legal reasoning or assessment +- The Court's findings or conclusions +- Whether the Court found a violation or not +- Operative provisions or dispositif +- Article 41 (just satisfaction) analysis + +Review the text carefully and determine: +1. Does this text reveal the Court's verdict (violation or no violation)? +2. If yes, quote the specific sentence(s) that leak the verdict. + +Text to review: +--- +{text} +--- + +Respond in this exact JSON format: +{{"leaks_verdict": true/false, "leaking_sentences": ["sentence1", "sentence2"] or [], "confidence": "high"/"medium"/"low"}} + +Respond with ONLY the JSON, no other text.""" + + +def check_leakage_bedrock(bedrock_client, model_id: str, text: str, max_chars: int = 50000) -> dict: + """Check a single case for verdict leakage using AWS Bedrock.""" + truncated_text = text[:max_chars] if len(text) > max_chars else text + prompt = LEAKAGE_CHECK_PROMPT.format(text=truncated_text) + try: + response = bedrock_client.converse( + modelId=model_id, + system=[{"text": "You are a legal document reviewer. Respond only in valid JSON."}], + messages=[{"role": "user", "content": [{"text": prompt}]}], + inferenceConfig={"temperature": 0.0, "maxTokens": 500}, + ) + content = response["output"]["message"]["content"][0]["text"].strip() + if content.startswith("```"): + content = content.split("```")[1] + if content.startswith("json"): + content = content[4:] + return json.loads(content) + except Exception as e: + return {"leaks_verdict": None, "error": str(e), "leaking_sentences": []} + + +def check_leakage_sync(client, model: str, text: str, max_chars: int = 50000) -> dict: + """Check a single case for verdict leakage using OpenAI-compatible API.""" + truncated_text = text[:max_chars] if len(text) > max_chars else text + try: + response = client.chat.completions.create( + model=model, + messages=[ + {"role": "system", "content": "You are a legal document reviewer. Respond only in valid JSON."}, + {"role": "user", "content": LEAKAGE_CHECK_PROMPT.format(text=truncated_text)}, + ], + temperature=0.0, + max_tokens=500, + ) + content = response.choices[0].message.content.strip() + if content.startswith("```"): + content = content.split("```")[1] + if content.startswith("json"): + content = content[4:] + return json.loads(content) + except Exception as e: + return {"leaks_verdict": None, "error": str(e), "leaking_sentences": []} + + +def stage2_verify(case: CaseRecord, verifiers: list[tuple], use_bedrock: bool = False) -> CaseRecord: + """ + Multi-model LLM verification: N models independently check for verdict leakage. + If ANY model flags leakage, attempt repair. + + verifiers: list of (client, model_id, label) tuples + """ + text = case.full_case_text_no_verdict + if not text or case.excluded: + return case + + check_fn = check_leakage_bedrock if use_bedrock else check_leakage_sync + + results = {} + all_leaking_sentences = [] + any_leaks = False + + for client, model_id, label in verifiers: + result = check_fn(client, model_id, text) + results[label] = result + if result.get("leaks_verdict", False): + any_leaks = True + all_leaking_sentences.extend(result.get("leaking_sentences", [])) + + case.leakage_check_primary = json.dumps(results.get("primary", results.get(list(results.keys())[0], {}))) + case.leakage_check_secondary = json.dumps({k: v for k, v in results.items() if k != "primary"}) + + if any_leaks: + case = stage3_repair(case, all_leaking_sentences) + + return case + + +# ── Stage 3: Repair or exclude ─────────────────────────────────────────────── + +def stage3_repair(case: CaseRecord, leaking_sentences: list[str]) -> CaseRecord: + """ + Attempt to repair a leaking case by removing identified leaking sentences. + If the result is too short, exclude the case. + """ + text = case.full_case_text_no_verdict + + for sentence in leaking_sentences: + if not sentence: + continue + # Try exact removal + escaped = re.escape(sentence.strip()) + new_text = re.sub(escaped, "", text, count=1) + if new_text != text: + text = new_text + case.verdict_removal_method += f"; surgical_fix: removed leaking sentence" + continue + + # Fuzzy: find and truncate before the sentence + idx = text.lower().find(sentence.lower()[:50]) + if idx > 0: + sentence_start = text.rfind('.', 0, idx) + if sentence_start > 0: + text = text[:sentence_start + 1].rstrip() + case.verdict_removal_method += f"; surgical_fix: truncated before leaking sentence" + + # Also scan for any remaining conclusory patterns + for pattern in COMPILED_CONCLUSORY: + match = pattern.search(text) + if match: + sentence_start = text.rfind('.', 0, match.start()) + if sentence_start > 0: + text = text[:sentence_start + 1].rstrip() + case.verdict_removal_method += "; conclusory_pattern_cleanup" + + text = text.strip() + + if len(text) < MIN_VERDICT_FREE_LENGTH: + case.excluded = True + case.exclusion_reason = "too_short_after_repair" + else: + case.full_case_text_no_verdict = text + case.verdict_free_length = len(text) + case.retention_percentage = len(text) / case.original_length * 100 if case.original_length else 0 + + return case + + +# ── Data loading ───────────────────────────────────────────────────────────── + +def load_from_db(db_url: str, country: str = "", cutoff_date: str = "", end_date: str = "", lang: str = "ENG") -> list[CaseRecord]: + """Load ECtHR cases from PostgreSQL echr_cases table.""" + import psycopg2 + + conn = psycopg2.connect(db_url) + with conn.cursor() as cur: + conditions = ["full_text IS NOT NULL", "length(full_text) > 500", "conclusion IS NOT NULL"] + params = [] + + if country: + conditions.append("respondent = %s") + params.append(country) + if cutoff_date: + conditions.append("kp_date >= %s") + params.append(cutoff_date) + if end_date: + conditions.append("kp_date < %s") + params.append(end_date) + if lang: + conditions.append("language_iso = %s") + params.append(lang) + + # Exclude translations + conditions.append("doc_name NOT ILIKE %s") + params.append("%Translation%") + + query = f"SELECT item_id, doc_name, conclusion, full_text, respondent, kp_date FROM echr_cases WHERE {' AND '.join(conditions)} ORDER BY item_id" + cur.execute(query, params if params else None) + rows = cur.fetchall() + conn.close() + + cases = [] + for row in rows: + item_id, doc_name, conclusion, full_text = row[0], row[1], row[2], row[3] + respondent = row[4] if len(row) > 4 else "" + kp_date = row[5] if len(row) > 5 else None + decision_date_str = str(kp_date) if kp_date else "" + articles_violations = parse_conclusion(conclusion or "") + for article, label in articles_violations: + cases.append(CaseRecord( + item_id=item_id, + case_name=doc_name or "", + article=article, + violation_label=label, + full_case_text=full_text, + decision_date=decision_date_str, + respondent=respondent or "", + original_length=len(full_text), + )) + + return cases + + +def parse_conclusion(conclusion: str) -> list[tuple[str, str]]: + """ + Parse ECtHR conclusion field into (article, violation_label) pairs. + + Examples: + "Violation of Article 3" -> [("3", "violation")] + "No violation of Article 8; Violation of Article 3" -> [("8", "no_violation"), ("3", "violation")] + """ + results = [] + if not conclusion: + return results + + parts = re.split(r'[;]', conclusion) + for part in parts: + part = part.strip() + article_match = re.search(r'Article\s+([\dP][\d-]*)', part, re.IGNORECASE) + if not article_match: + continue + article = article_match.group(1) + + if re.search(r'no\s+violation', part, re.IGNORECASE): + label = "no_violation" + elif re.search(r'violation', part, re.IGNORECASE): + label = "violation" + else: + continue + + results.append((article, label)) + + return results + + +def load_from_json(path: str, cutoff_date: str = "", end_date: str = "") -> list[CaseRecord]: + """Load cases from a JSON file, optionally filtering by date range.""" + with open(path) as f: + data = json.load(f) + + cases = [] + skipped_no_date = 0 + skipped_out_of_range = 0 + + for item in data: + # Try multiple possible date field names + date_val = (item.get("decision_date") or item.get("date") + or item.get("kp_date") or item.get("judgment_date") or "") + date_str = str(date_val)[:10] if date_val else "" + + # Apply date filters + if cutoff_date or end_date: + if not date_str: + skipped_no_date += 1 + continue + if cutoff_date and date_str < cutoff_date: + skipped_out_of_range += 1 + continue + if end_date and date_str >= end_date: + skipped_out_of_range += 1 + continue + + cases.append(CaseRecord( + item_id=item.get("item_id", ""), + case_name=item.get("case_name", ""), + article=item.get("article", ""), + violation_label=item.get("violation_label", ""), + full_case_text=item.get("full_case_text", item.get("case_text", "")), + decision_date=date_str, + respondent=item.get("respondent", ""), + original_length=len(item.get("full_case_text", item.get("case_text", ""))), + )) + + if cutoff_date or end_date: + print(f" Date filter: skipped {skipped_out_of_range} out-of-range, {skipped_no_date} missing date") + + return cases + + +# ── Main pipeline ──────────────────────────────────────────────────────────── + +def run_pipeline(cases: list[CaseRecord], args) -> list[CaseRecord]: + """Run the full 3-stage pipeline.""" + + print(f"\n{'='*60}") + print(f"Stage 1: Pattern-based truncation ({len(cases)} cases)") + print(f"{'='*60}") + + for case in cases: + truncated, method = stage1_truncate(case.full_case_text) + # Mandatory conclusion scrub: Stage 1 truncation misses per-article + # conclusion sentences inside the assessment (measured 2.8-7.3% + # own-article label leakage across published sets, 2026-07-03 audit). + clean, dropped = scrub_text(truncated) + if dropped: + truncated = clean + method += SCRUB_MARKER + case.full_case_text_no_verdict = truncated + case.verdict_removal_method = method + case.verdict_free_length = len(truncated) + case.retention_percentage = len(truncated) / case.original_length * 100 if case.original_length else 0 + + # Stats + truncated_count = sum(1 for c in cases if c.verdict_removal_method != "no_truncation") + avg_retention = sum(c.retention_percentage for c in cases) / len(cases) if cases else 0 + print(f" Truncated: {truncated_count}/{len(cases)}") + print(f" Average retention: {avg_retention:.1f}%") + + # Exclude cases that became too short + for case in cases: + if case.verdict_free_length < MIN_VERDICT_FREE_LENGTH: + case.excluded = True + case.exclusion_reason = "too_short_after_truncation" + + excluded = sum(1 for c in cases if c.excluded) + print(f" Excluded (too short): {excluded}") + + if args.stage1_only: + print("\n --stage1-only: skipping LLM verification") + return cases + + # Stage 2+3: LLM verification and repair + # Only verify untruncated cases (truncated ones are already clean) + needs_verification = [c for c in cases if not c.excluded and c.verdict_removal_method == "no_truncation"] + already_clean = [c for c in cases if not c.excluded and c.verdict_removal_method != "no_truncation"] + print(f"\n{'='*60}") + print(f"Stage 2+3: LLM verification & repair ({len(needs_verification)} untruncated cases)") + print(f" Already truncated (skipping): {len(already_clean)}") + print(f"{'='*60}") + + if not needs_verification: + print(" No cases need LLM verification.") + return cases + + use_bedrock = args.backend == "bedrock" + + if use_bedrock: + if not boto3: + print(" ERROR: boto3 not installed. pip install boto3") + return cases + bedrock = boto3.client("bedrock-runtime", region_name=args.aws_region) + verifiers = [(bedrock, m.strip(), f"v{i}") for i, m in enumerate(args.models.split(","))] + else: + api_key = os.environ.get("OPENAI_API_KEY") or os.environ.get("OPENROUTER_API_KEY", "") + if not api_key: + print(" WARNING: No API key found (OPENAI_API_KEY or OPENROUTER_API_KEY)") + print(" Skipping LLM verification. Set API key to enable.") + return cases + use_openrouter = bool(os.environ.get("OPENROUTER_API_KEY")) + base_url = "https://openrouter.ai/api/v1" if use_openrouter else None + client_kwargs = {"api_key": api_key} + if base_url: + client_kwargs["base_url"] = base_url + client = openai.OpenAI(**client_kwargs) + verifiers = [(client, m.strip(), f"v{i}") for i, m in enumerate(args.models.split(","))] + + # Set first as "primary" label + verifiers[0] = (verifiers[0][0], verifiers[0][1], "primary") + + print(f" Backend: {'bedrock' if use_bedrock else 'openai/openrouter'}") + print(f" Verifiers ({len(verifiers)}):") + for client, model_id, label in verifiers: + print(f" [{label}] {model_id}") + print(f" Workers: {args.workers}") + + done = 0 + + def process_case(case): + return stage2_verify(case, verifiers, use_bedrock) + + with ThreadPoolExecutor(max_workers=args.workers) as executor: + futures = {executor.submit(process_case, c): c for c in needs_verification} + for future in as_completed(futures): + result = future.result() + idx = cases.index(futures[future]) + cases[idx] = result + done += 1 + if done % 10 == 0 or done == len(needs_verification): + flagged = sum(1 for c in cases if c.leakage_check_primary and + json.loads(c.leakage_check_primary).get("leaks_verdict")) + print(f"\r Verified: {done}/{len(needs_verification)} | Flagged: {flagged}", end="", flush=True) + + print() + + final_excluded = sum(1 for c in cases if c.excluded) + final_count = len(cases) - final_excluded + print(f"\n Final dataset: {final_count} cases ({final_excluded} excluded)") + + return cases + + +def main(): + parser = argparse.ArgumentParser(description="Verdict-leakage removal pipeline for ECtHR cases") + parser.add_argument("--source", choices=["db", "json"], default="db") + parser.add_argument("--db", default="dbname=secondlayer_prod user=secondlayer password=secondlayer host=172.18.0.13") + parser.add_argument("--country", default="", help="Respondent country code (empty = all countries)") + parser.add_argument("--lang", default="ENG", help="Language ISO code filter (ENG, FRE, etc.)") + parser.add_argument("--cutoff-date", default="", help="Only include cases decided on or after this date (YYYY-MM-DD)") + parser.add_argument("--end-date", default="", help="Only include cases decided before this date (YYYY-MM-DD, exclusive)") + parser.add_argument("--input", help="Input JSON file (when --source json)") + parser.add_argument("--output", default="data/processed/echr_cases_clean.json") + parser.add_argument("--workers", type=int, default=5) + parser.add_argument("--backend", choices=["openai", "bedrock"], default="bedrock", help="LLM backend") + parser.add_argument("--aws-region", default="us-east-1", help="AWS region for Bedrock") + parser.add_argument("--models", default="us.anthropic.claude-sonnet-4-6-20250514-v1:0,us.anthropic.claude-haiku-4-5-20251001-v1:0,amazon.nova-pro-v1:0,qwen.qwen3-32b-v1:0", + help="Comma-separated verifier model IDs") + parser.add_argument("--stage1-only", action="store_true", help="Only run pattern truncation (no LLM calls)") + parser.add_argument("--min-length", type=int, default=500, help="Minimum verdict-free text length") + args = parser.parse_args() + + global MIN_VERDICT_FREE_LENGTH + MIN_VERDICT_FREE_LENGTH = args.min_length + + # Load cases + # Date range summary + date_range_parts = [] + if args.cutoff_date: + date_range_parts.append(f"from {args.cutoff_date}") + if args.end_date: + date_range_parts.append(f"until {args.end_date}") + date_range_str = " ".join(date_range_parts) if date_range_parts else "all dates" + + if args.source == "db": + print(f"Loading cases from database (country={args.country or 'ALL'}, lang={args.lang}, dates={date_range_str})...") + cases = load_from_db(args.db, args.country, args.cutoff_date, args.end_date, args.lang) + else: + print(f"Loading cases from {args.input} (dates={date_range_str})...") + cases = load_from_json(args.input, args.cutoff_date, args.end_date) + + print(f"Loaded {len(cases)} case-article pairs") + + # Print date range summary + dates = sorted([c.decision_date for c in cases if c.decision_date]) + if dates: + print(f" Date range in dataset: {dates[0]} to {dates[-1]} ({len(set(dates))} distinct dates)") + elif args.cutoff_date or args.end_date: + print(f" Warning: no date information available on loaded cases") + + if not cases: + print("No cases to process.") + return + + # Run pipeline + cases = run_pipeline(cases, args) + + # Save results + output_cases = [asdict(c) for c in cases if not c.excluded] + os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True) + with open(args.output, "w") as f: + json.dump(output_cases, f, indent=2, ensure_ascii=False) + + print(f"\nSaved {len(output_cases)} clean cases to {args.output}") + + # Summary + from collections import Counter + labels = Counter(c.violation_label for c in cases if not c.excluded) + articles = Counter(c.article for c in cases if not c.excluded) + methods = Counter(c.verdict_removal_method.split(";")[0].strip() for c in cases if not c.excluded) + print(f"\nViolation labels: {dict(labels)}") + print(f"Articles: {dict(articles)}") + print(f"Removal methods: {dict(methods)}") + + +if __name__ == "__main__": + main()