From d6044d72aab2f69f412c920d0728fb758493184b Mon Sep 17 00:00:00 2001 From: TaigaWalk Date: Thu, 5 Jun 2025 08:54:44 -0400 Subject: [PATCH] Add GitHub Actions-based Chronicle ingestors and README --- .../.github/workflows/1password.yml | 39 +++++++ github_actions/.github/workflows/entra.yml | 40 +++++++ github_actions/.github/workflows/github.yml | 39 +++++++ .../.github/workflows/snowflake.yml | 43 +++++++ .../.github/workflows/thinkst-audit.yml | 40 +++++++ .../1password-chronicle-ingestor/README.md | 66 +++++++++++ .../1password-chronicle-ingestor/main.py | 37 ++++++ .../requirements.txt | 2 + github_actions/README.md | 76 +++++++++++++ .../README.md | 107 ++++++++++++++++++ .../main.py | 61 ++++++++++ .../requirements.txt | 2 + .../github-chronicle-ingestor/README.md | 67 +++++++++++ .../github-chronicle-ingestor/main.py | 60 ++++++++++ .../requirements.txt | 2 + .../snowflake-chronicle-ingestor/README.md | 72 ++++++++++++ .../snowflake-chronicle-ingestor/main.py | 89 +++++++++++++++ .../requirements.txt | 5 + .../README.md | 67 +++++++++++ .../thinkst-audit-chronicle-ingestor/main.py | 73 ++++++++++++ .../requirements.txt | 2 + 21 files changed, 989 insertions(+) create mode 100644 github_actions/.github/workflows/1password.yml create mode 100644 github_actions/.github/workflows/entra.yml create mode 100644 github_actions/.github/workflows/github.yml create mode 100644 github_actions/.github/workflows/snowflake.yml create mode 100644 github_actions/.github/workflows/thinkst-audit.yml create mode 100644 github_actions/1password-chronicle-ingestor/README.md create mode 100644 github_actions/1password-chronicle-ingestor/main.py create mode 100644 github_actions/1password-chronicle-ingestor/requirements.txt create mode 100644 github_actions/README.md create mode 100644 github_actions/entra-noninteractive-chronicle-ingestor/README.md create mode 100644 github_actions/entra-noninteractive-chronicle-ingestor/main.py create mode 100644 github_actions/entra-noninteractive-chronicle-ingestor/requirements.txt create mode 100644 github_actions/github-chronicle-ingestor/README.md create mode 100644 github_actions/github-chronicle-ingestor/main.py create mode 100644 github_actions/github-chronicle-ingestor/requirements.txt create mode 100644 github_actions/snowflake-chronicle-ingestor/README.md create mode 100644 github_actions/snowflake-chronicle-ingestor/main.py create mode 100644 github_actions/snowflake-chronicle-ingestor/requirements.txt create mode 100644 github_actions/thinkst-audit-chronicle-ingestor/README.md create mode 100644 github_actions/thinkst-audit-chronicle-ingestor/main.py create mode 100644 github_actions/thinkst-audit-chronicle-ingestor/requirements.txt diff --git a/github_actions/.github/workflows/1password.yml b/github_actions/.github/workflows/1password.yml new file mode 100644 index 0000000..2570f3c --- /dev/null +++ b/github_actions/.github/workflows/1password.yml @@ -0,0 +1,39 @@ +name: Ingest 1Password Events into Chronicle + +on: + workflow_dispatch: + +jobs: + ingest: + runs-on: ubuntu-latest + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + pip install -r chronicle-scripts/1password-chronicle-ingestor/requirements.txt + + - name: Write Chronicle service account key to file + shell: bash + run: | + echo "${{ secrets.CHRONICLE_CREDENTIALS_JSON }}" | base64 --decode > chronicle_sa.json + + - name: Run 1Password ingestion script + env: + PYTHONPATH: chronicle-scripts + CHRONICLE_CUSTOMER_ID: ${{ secrets.CHRONICLE_CUSTOMER_ID }} + CHRONICLE_REGION: ${{ secrets.CHRONICLE_REGION }} + GOOGLE_APPLICATION_CREDENTIALS: chronicle_sa.json + CHRONICLE_SERVICE_ACCOUNT: chronicle_sa.json + CHRONICLE_NAMESPACE: ${{ secrets.CHRONICLE_NAMESPACE }} + ONEPASSWORD_TOKEN: ${{ secrets.ONEPASSWORD_TOKEN }} + EVENTS_API_URL: https://events.1password.com/api/v2/auditevents + run: | + python chronicle-scripts/1password-chronicle-ingestor/main.py --creds-file chronicle_sa.json diff --git a/github_actions/.github/workflows/entra.yml b/github_actions/.github/workflows/entra.yml new file mode 100644 index 0000000..9f57626 --- /dev/null +++ b/github_actions/.github/workflows/entra.yml @@ -0,0 +1,40 @@ +name: Ingest Microsoft Entra Non-Interactive Sign-ins into Chronicle + +on: + workflow_dispatch: # Manual trigger only + +jobs: + ingest: + runs-on: ubuntu-latest + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + pip install -r chronicle-scripts/entra-noninteractive-chronicle-ingestor/requirements.txt + + - name: Write Chronicle service account key to file + shell: bash + run: | + echo "${{ secrets.CHRONICLE_CREDENTIALS_JSON }}" | base64 --decode > chronicle_sa.json + + - name: Run Entra non-interactive ingestion script + env: + PYTHONPATH: chronicle-scripts + CHRONICLE_CUSTOMER_ID: ${{ secrets.CHRONICLE_CUSTOMER_ID }} + CHRONICLE_REGION: ${{ secrets.CHRONICLE_REGION }} + GOOGLE_APPLICATION_CREDENTIALS: chronicle_sa.json + CHRONICLE_SERVICE_ACCOUNT: chronicle_sa.json + CHRONICLE_NAMESPACE: ${{ secrets.CHRONICLE_NAMESPACE }} + GRAPH_CLIENT_ID: ${{ secrets.GRAPH_CLIENT_ID }} + GRAPH_CLIENT_SECRET: ${{ secrets.GRAPH_CLIENT_SECRET }} + GRAPH_TENANT_ID: ${{ secrets.GRAPH_TENANT_ID }} + run: | + python chronicle-scripts/entra-noninteractive-chronicle-ingestor/main.py --creds-file chronicle_sa.json diff --git a/github_actions/.github/workflows/github.yml b/github_actions/.github/workflows/github.yml new file mode 100644 index 0000000..b13bb90 --- /dev/null +++ b/github_actions/.github/workflows/github.yml @@ -0,0 +1,39 @@ +name: Ingest GitHub Audit Logs into Chronicle + +on: + workflow_dispatch: + +jobs: + ingest: + runs-on: ubuntu-latest + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + pip install -r chronicle-scripts/github-chronicle-ingestor/requirements.txt + + - name: Write Chronicle service account key to file + run: | + echo "${{ secrets.CHRONICLE_CREDENTIALS_JSON }}" | base64 --decode > chronicle_sa.json + + - name: Run GitHub ingestion script + env: + PYTHONPATH: chronicle-scripts + GOOGLE_APPLICATION_CREDENTIALS: chronicle_sa.json + CHRONICLE_CUSTOMER_ID: ${{ secrets.CHRONICLE_CUSTOMER_ID }} + CHRONICLE_REGION: ${{ secrets.CHRONICLE_REGION }} + CHRONICLE_NAMESPACE: ${{ secrets.CHRONICLE_NAMESPACE }} + CHRONICLE_SERVICE_ACCOUNT: chronicle_sa.json + GITHUB_AUDIT_TOKEN: ${{ secrets.GITHUB_AUDIT_TOKEN }} + GITHUB_AUDIT_URL: ${{ secrets.GITHUB_AUDIT_URL }} + run: | + python chronicle-scripts/github-chronicle-ingestor/main.py --creds-file chronicle_sa.json + diff --git a/github_actions/.github/workflows/snowflake.yml b/github_actions/.github/workflows/snowflake.yml new file mode 100644 index 0000000..ea0bf67 --- /dev/null +++ b/github_actions/.github/workflows/snowflake.yml @@ -0,0 +1,43 @@ +name: Ingest Snowflake Audit Logs into Chronicle + +on: + workflow_dispatch: + +jobs: + ingest: + runs-on: ubuntu-latest + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + pip install -r chronicle-scripts/snowflake-chronicle-ingestor/requirements.txt + + - name: Write Chronicle service account key to file + shell: bash + run: | + echo "${{ secrets.CHRONICLE_CREDENTIALS_JSON }}" | base64 --decode > chronicle_sa.json + + - name: Run Snowflake ingestion script + env: + PYTHONPATH: chronicle-scripts + CHRONICLE_CUSTOMER_ID: ${{ secrets.CHRONICLE_CUSTOMER_ID }} + CHRONICLE_REGION: ${{ secrets.CHRONICLE_REGION }} + CHRONICLE_NAMESPACE: ${{ secrets.CHRONICLE_NAMESPACE }} + GOOGLE_APPLICATION_CREDENTIALS: chronicle_sa.json + CHRONICLE_SERVICE_ACCOUNT: chronicle_sa.json + SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} + SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }} + run: | + python chronicle-scripts/snowflake-chronicle-ingestor/main.py --creds-file chronicle_sa.json + diff --git a/github_actions/.github/workflows/thinkst-audit.yml b/github_actions/.github/workflows/thinkst-audit.yml new file mode 100644 index 0000000..6739823 --- /dev/null +++ b/github_actions/.github/workflows/thinkst-audit.yml @@ -0,0 +1,40 @@ +name: Ingest Thinkst Canary Audit Logs into Chronicle + +on: + workflow_dispatch: + +jobs: + ingest: + runs-on: ubuntu-latest + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + pip install -r chronicle-scripts/thinkst-canary-chronicle-ingestor/requirements.txt + + - name: Write Chronicle service account key to file + shell: bash + run: | + echo "${{ secrets.CHRONICLE_CREDENTIALS_JSON }}" | base64 --decode > chronicle_sa.json + + - name: Run Thinkst Canary ingestion script + env: + PYTHONPATH: chronicle-scripts + CHRONICLE_CUSTOMER_ID: ${{ secrets.CHRONICLE_CUSTOMER_ID }} + CHRONICLE_REGION: ${{ secrets.CHRONICLE_REGION }} + GOOGLE_APPLICATION_CREDENTIALS: chronicle_sa.json + CHRONICLE_SERVICE_ACCOUNT: chronicle_sa.json + CHRONICLE_NAMESPACE: ${{ secrets.CHRONICLE_NAMESPACE }} + CANARY_CONSOLE_ID: ${{ secrets.CANARY_CONSOLE_ID }} + CANARY_AUTH_TOKEN: ${{ secrets.CANARY_AUTH_TOKEN }} + run: | + python chronicle-scripts/thinkst-canary-chronicle-ingestor/main.py --creds-file chronicle_sa.json + diff --git a/github_actions/1password-chronicle-ingestor/README.md b/github_actions/1password-chronicle-ingestor/README.md new file mode 100644 index 0000000..2cf6315 --- /dev/null +++ b/github_actions/1password-chronicle-ingestor/README.md @@ -0,0 +1,66 @@ +# 1Password Events Ingestor for Chronicle + +This module ingests security-related events from the **1Password Events API** into **Google Chronicle** using the Unstructured Ingestion API. + +--- + +## πŸ” What It Does + +- Authenticates to the 1Password Events API using a bearer token +- Fetches audit events since the last run +- Normalizes and forwards logs to Chronicle +- Supports execution via GitHub Actions or locally + +--- + +## πŸ›  Requirements + +- Python 3.7+ +- GitHub Actions enabled +- GitHub Secrets: + - `ONEPASSWORD_TOKEN` + - `EVENTS_API_URL` (defaults to `https://events.1password.com/api/v2/auditevents`) + - `CHRONICLE_CUSTOMER_ID` + - `CHRONICLE_REGION` + - `CHRONICLE_NAMESPACE` (optional) + - `CHRONICLE_CREDENTIALS_JSON` (base64-encoded service account JSON) + +--- + +## βš™ GitHub Actions Workflow + +Location: `.github/workflows/1password.yml` + +This workflow: +- Runs **manually only** +- Installs dependencies +- Writes Chronicle service credentials +- Executes the 1Password ingestion script + +--- + +## πŸ§ͺ Running Locally + +```bash +export ONEPASSWORD_TOKEN=... +export EVENTS_API_URL=https://events.1password.com/api/v2/auditevents +export CHRONICLE_CUSTOMER_ID=... +export CHRONICLE_REGION=... +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/chronicle_sa.json + +python main.py --creds-file /path/to/chronicle_sa.json +``` + +--- + +## πŸ“„ Notes + +- The `EVENTS_API_URL` must be accessible and your token scoped appropriately +- Script defaults to collecting events from the last 5 minutes or since the last successful run +- Designed for organizations needing audit coverage from 1Password + +--- + +## 🀝 Contributions + +Contributions are welcome via pull request or issue! diff --git a/github_actions/1password-chronicle-ingestor/main.py b/github_actions/1password-chronicle-ingestor/main.py new file mode 100644 index 0000000..e132c3f --- /dev/null +++ b/github_actions/1password-chronicle-ingestor/main.py @@ -0,0 +1,37 @@ +import os +import requests +from datetime import datetime, timezone +from common import ingest, env_constants, utils + +CHRONICLE_LOG_TYPE = "ONEPASSWORD" +EVENTS_API_URL = os.getenv("EVENTS_API_URL") +ONEPASSWORD_TOKEN = os.getenv("ONEPASSWORD_TOKEN") + +def fetch_events(start_time: datetime): + headers = { + "Authorization": f"Bearer {ONEPASSWORD_TOKEN}", + "Content-Type": "application/json" + } + + iso_since = start_time.replace(tzinfo=timezone.utc).isoformat(timespec="seconds") + body = { + "limit": 1000, + "since": iso_since + } + + print(f"[INFO] Fetching 1Password events since {iso_since} from {EVENTS_API_URL}") + response = requests.post(EVENTS_API_URL, headers=headers, json=body) + response.raise_for_status() + + data = response.json() + print(f"[INFO] Retrieved {len(data.get('items', []))} events from 1Password.") + return data.get("items", []) + +def main(): + last_run_time = utils.get_last_run_at() + logs = fetch_events(last_run_time) + ingest.ingest(logs, CHRONICLE_LOG_TYPE) + print("[INFO] Ingestion completed.") + +if __name__ == "__main__": + main() diff --git a/github_actions/1password-chronicle-ingestor/requirements.txt b/github_actions/1password-chronicle-ingestor/requirements.txt new file mode 100644 index 0000000..524b652 --- /dev/null +++ b/github_actions/1password-chronicle-ingestor/requirements.txt @@ -0,0 +1,2 @@ +requests +google-auth>=2.0.0 diff --git a/github_actions/README.md b/github_actions/README.md new file mode 100644 index 0000000..34d4fcc --- /dev/null +++ b/github_actions/README.md @@ -0,0 +1,76 @@ +# πŸ›‘ Chronicle Ingestion Scripts + +This repository contains custom ingestion connectors designed to forward third-party security logs into [Google Chronicle](https://cloud.google.com/chronicle). + +Each connector is a lightweight, self-contained Python script that authenticates to a third-party API or platform, collects relevant logs, and pushes them into Chronicle using the **Unstructured Ingestion API**. + +--- + +## πŸ“Œ Why This Exists + +While Chronicle provides powerful detection capabilities, it does **not offer native integrations** for many widely used security tools and services. This repository was created to: + +- Enable ingestion of log sources not supported by default in Chronicle +- Provide a **cost-effective alternative** to GCP Cloud Functions or Cloud Run for those seeking to reduce infrastructure spend +- Standardize and automate ingestion workflows using open-source tooling and GitHub Actions + +--- + +## βœ… Key Features + +- Python-based, modular connectors +- GitHub Actions support for **manual** and **scheduled (cron)** runs +- Secure credential injection via GitHub Secrets + +--- + +## πŸ”— Supported Integrations + +| Integration | Log Type | Description | +|--------------------|------------------|-------------------------------------------------------------------------| +| Microsoft Entra ID | `AZURE_AD` | Captures non-interactive sign-in events via Microsoft Graph API (Beta) | +| 1Password | `ONEPASSWORD` | Pulls audit events using 1Password Events API | +| GitHub | `GITHUB` | Collects GitHub org audit logs using the REST API | +| Snowflake | `SNOWFLAKE` | Gathers usage logs from ACCOUNT_USAGE views in Snowflake | +| Thinkst Canary | `THINKST_CANARY` | Ingests audit trail logs from the Canary console | + +--- + +## 🧱 Folder Structure + +``` +chronicle-scripts/ +β”œβ”€β”€ 1password-chronicle-ingestor/ # 1Password Events API ingestion +β”œβ”€β”€ entra-noninteractive-chronicle-ingestor/ # Microsoft Entra non-interactive sign-ins +β”œβ”€β”€ github-chronicle-ingestor/ # GitHub audit log ingestion +β”œβ”€β”€ snowflake-chronicle-ingestor/ # Snowflake ACCOUNT_USAGE logs +β”œβ”€β”€ thinkst-canary-chronicle-ingestor/ # Thinkst Canary audit trail ingestion +``` + +--- + +## 🧠 Configuration Notes + +- All secrets (API tokens, credentials, org URLs) are securely managed via **GitHub Actions Secrets** +- Each `main.py` script handles: + - Authentication to source platform + - API communication and log retrieval + - Pushing logs to Chronicle via the Unstructured Ingestion API + +--- + +## πŸ•’ Scheduling & Execution + +Each connector includes a GitHub Actions workflow that can: + +- Be triggered **manually** +- Run on a **cron schedule** (e.g., every 15 or 30 minutes) + +This model gives users flexibility while avoiding the costs associated with always-on cloud infrastructure. + +--- + +## πŸ‘₯ Contributions + +We welcome community contributions! To propose an enhancement or new connector, please open an issue or submit a pull request. + diff --git a/github_actions/entra-noninteractive-chronicle-ingestor/README.md b/github_actions/entra-noninteractive-chronicle-ingestor/README.md new file mode 100644 index 0000000..73b12c8 --- /dev/null +++ b/github_actions/entra-noninteractive-chronicle-ingestor/README.md @@ -0,0 +1,107 @@ +# Microsoft Entra Non-Interactive Sign-In Ingestor + +This module is a custom Python-based ingestion script designed to forward non-interactive sign-in events from **Microsoft Entra ID** (formerly Azure AD) into **Google Chronicle** using the Unstructured Ingestion API. + +--- + +## πŸ” What It Does + +- Authenticates to Microsoft Graph API (Beta) +- Fetches **non-interactive sign-in events** (e.g., service principal or daemon access) +- Parses, normalizes, and forwards logs to Chronicle +- Executes via GitHub Actions on a scheduled or manual basis + +> πŸ”Ž **Note:** Chronicle currently provides native ingestion only for **interactive** Entra sign-ins. This script fills a critical visibility gap for service-based or daemon-based sign-ins, improving auditability and detection coverage. + +--- + +## πŸ“‚ Included Script + +### `main.py` + +This script: +- Retrieves an OAuth2 token using Microsoft client credentials +- Queries Microsoft Graph Beta API for sign-ins filtered by `signInEventTypes eq 'nonInteractiveUser'` +- Uses `common/ingest.py` to send the data to Chronicle’s Unstructured Ingestion API + +--- + +## πŸ›  Requirements + +- Python 3.7+ +- GitHub repository with GitHub Actions enabled +- Microsoft Graph API access via: + - `GRAPH_TENANT_ID` + - `GRAPH_CLIENT_ID` + - `GRAPH_CLIENT_SECRET` +- Chronicle access: + - `CHRONICLE_CUSTOMER_ID` + - `CHRONICLE_REGION` + - `CHRONICLE_NAMESPACE` (optional) + - `CHRONICLE_CREDENTIALS_JSON` (Base64-encoded service account JSON) + +--- + +## πŸ” GitHub Secrets + +Store these values in your GitHub repository’s **Secrets**: + +- `GRAPH_CLIENT_ID` +- `GRAPH_CLIENT_SECRET` +- `GRAPH_TENANT_ID` +- `CHRONICLE_CUSTOMER_ID` +- `CHRONICLE_REGION` +- `CHRONICLE_NAMESPACE` +- `CHRONICLE_CREDENTIALS_JSON` + +These variables are referenced in both the ingestion script and GitHub Actions workflow. + +--- + +## βš™ GitHub Actions Workflow + +The repo includes a GitHub Actions workflow at: + +``` +.github/workflows/entra.yml +``` + +This workflow: +- Installs dependencies +- Decodes the Chronicle credentials +- Runs the Entra ingestion script + +It runs: +- **Every 15 minutes** via `cron` +- **Manually** via `workflow_dispatch` + +--- + +## πŸ§ͺ Running Locally + +To test locally, export your environment variables and run: + +```bash +export GRAPH_CLIENT_ID=... +export GRAPH_CLIENT_SECRET=... +export GRAPH_TENANT_ID=... +export CHRONICLE_CUSTOMER_ID=... +export CHRONICLE_REGION=... +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/chronicle_sa.json + +python main.py --creds-file /path/to/chronicle_sa.json +``` + +--- + +## πŸ“„ Notes + +- Uses the **Microsoft Graph Beta API** β€” subject to schema changes +- The default filter fetches only **non-interactive** sign-in events +- Intended for users who need deeper visibility into Entra usage beyond native Chronicle coverage + +--- + +## 🀝 Contributions + +Want to enhance this module or add support for other Entra sign-in types? Open a PR or submit an issue! diff --git a/github_actions/entra-noninteractive-chronicle-ingestor/main.py b/github_actions/entra-noninteractive-chronicle-ingestor/main.py new file mode 100644 index 0000000..d24e96d --- /dev/null +++ b/github_actions/entra-noninteractive-chronicle-ingestor/main.py @@ -0,0 +1,61 @@ +# chronicle-scripts/entra-noninteractive-chronicle-ingestor/main.py + +import os +import requests +from common import ingest, utils + +LOG_TYPE = "AZURE_AD" +GRAPH_URL = "https://graph.microsoft.com/beta/auditLogs/signIns" + +def get_token(): + print("[INFO] Requesting access token...") + tenant_id = os.getenv("GRAPH_TENANT_ID") + client_id = os.getenv("GRAPH_CLIENT_ID") + client_secret = os.getenv("GRAPH_CLIENT_SECRET") + + url = f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token" + data = { + "client_id": client_id, + "scope": "https://graph.microsoft.com/.default", + "client_secret": client_secret, + "grant_type": "client_credentials", + } + + response = requests.post(url, data=data) + response.raise_for_status() + return response.json()["access_token"] + +def fetch_signins(access_token, since): + print("[INFO] Fetching non-interactive sign-ins from Microsoft Graph (beta)...") + + headers = { + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json" + } + + filter_time = since.isoformat() + params = { + "$filter": f"createdDateTime ge {filter_time} and signInEventTypes/any(t:t eq 'nonInteractiveUser')", + "$top": 100 + } + + all_logs = [] + response = requests.get(GRAPH_URL, headers=headers, params=params) + response.raise_for_status() + + data = response.json() + all_logs.extend(data.get("value", [])) + + print(f"[INFO] Retrieved {len(all_logs)} non-interactive sign-ins.") + return all_logs + +def main(): + print("[INFO] Starting Microsoft Entra ingestion for non-interactive sign-ins...") + + access_token = get_token() + since = utils.get_last_run_at() # Default 5 mins or POLL_INTERVAL + logs = fetch_signins(access_token, since) + ingest.ingest(logs, LOG_TYPE) + +if __name__ == "__main__": + main() diff --git a/github_actions/entra-noninteractive-chronicle-ingestor/requirements.txt b/github_actions/entra-noninteractive-chronicle-ingestor/requirements.txt new file mode 100644 index 0000000..524b652 --- /dev/null +++ b/github_actions/entra-noninteractive-chronicle-ingestor/requirements.txt @@ -0,0 +1,2 @@ +requests +google-auth>=2.0.0 diff --git a/github_actions/github-chronicle-ingestor/README.md b/github_actions/github-chronicle-ingestor/README.md new file mode 100644 index 0000000..0c55a89 --- /dev/null +++ b/github_actions/github-chronicle-ingestor/README.md @@ -0,0 +1,67 @@ +# GitHub Audit Log Ingestor for Chronicle + +This module ingests GitHub audit logs into **Google Chronicle** using the Unstructured Ingestion API. + +--- + +## πŸ” What It Does + +- Fetches GitHub organization audit logs using the GitHub REST API +- Tracks last run timestamp to avoid duplicate ingestion +- Sends logs to Chronicle for security analysis and correlation +- Supports GitHub Actions automation or local execution + +--- + +## πŸ›  Requirements + +- Python 3.7+ +- GitHub Actions enabled +- Required GitHub Secrets: + - `GITHUB_AUDIT_TOKEN` + - `GITHUB_AUDIT_URL` (e.g., `https://api.github.com/orgs/myorg/audit-log`) + - `CHRONICLE_CUSTOMER_ID` + - `CHRONICLE_REGION` + - `CHRONICLE_NAMESPACE` (optional) + - `CHRONICLE_CREDENTIALS_JSON` (base64-encoded Chronicle SA key) + +--- + +## βš™ GitHub Actions Workflow + +Location: `.github/workflows/github.yml` + +This workflow: +- Runs **manually only** +- Installs dependencies +- Writes Chronicle credentials +- Executes the GitHub ingestion script + +--- + +## πŸ§ͺ Running Locally + +```bash +export GITHUB_AUDIT_TOKEN=... +export GITHUB_AUDIT_URL=https://api.github.com/orgs/myorg/audit-log +export CHRONICLE_CUSTOMER_ID=... +export CHRONICLE_REGION=... +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/chronicle_sa.json + +python main.py --creds-file /path/to/chronicle_sa.json +``` + +--- + +## πŸ“„ Notes + +- Script stores a local `.github_audit_last_run.json` file to track last successful ingestion +- Fetches only events from the last run (default: 24 hours ago if no file exists) +- Use cases include tracking repository, team, and user-level events for security monitoring + +--- + +## 🀝 Contributions + +Feel free to open an issue or submit a PR to improve the ingestor! + diff --git a/github_actions/github-chronicle-ingestor/main.py b/github_actions/github-chronicle-ingestor/main.py new file mode 100644 index 0000000..0eb4c86 --- /dev/null +++ b/github_actions/github-chronicle-ingestor/main.py @@ -0,0 +1,60 @@ +"""Fetch GitHub audit logs and ingest into Chronicle.""" + +from datetime import datetime, timedelta, timezone +import os +import json +import requests + +from common import ingest + +GITHUB_API_URL = os.getenv("GITHUB_AUDIT_URL") +CHRONICLE_DATA_TYPE = "GITHUB" +DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +LAST_RUN_FILE = ".github_audit_last_run.json" + +GITHUB_TOKEN = os.getenv("GITHUB_AUDIT_TOKEN") + +def load_last_run_timestamp() -> datetime: + if os.path.exists(LAST_RUN_FILE): + with open(LAST_RUN_FILE, "r") as f: + data = json.load(f) + last_run_str = data.get("last_run", "") + if last_run_str: + return datetime.strptime(last_run_str, DATE_FORMAT).replace(tzinfo=timezone.utc) + return datetime.now(timezone.utc) - timedelta(days=1) + +def save_last_run_timestamp(timestamp: datetime) -> None: + with open(LAST_RUN_FILE, "w") as f: + json.dump({"last_run": timestamp.strftime(DATE_FORMAT)}, f) + +def fetch_github_audit_logs(since: str): + headers = { + "Authorization": f"token {GITHUB_TOKEN}", + "Accept": "application/vnd.github+json" + } + url = f"{GITHUB_API_URL}?per_page=100&since={since}" + response = requests.get(url, headers=headers) + if response.status_code != 200: + print(f"[ERROR] Failed to retrieve logs: {response.status_code} {response.text}") + response.raise_for_status() + logs = response.json() + print(f"[INFO] Retrieved {len(logs)} GitHub audit logs.") + return logs + +def main(): + if not GITHUB_TOKEN or not GITHUB_API_URL: + raise RuntimeError("GITHUB_AUDIT_TOKEN or GITHUB_AUDIT_URL is not set!") + last_run_time = load_last_run_timestamp() + since = last_run_time.strftime(DATE_FORMAT) + print(f"[INFO] Fetching GitHub audit logs since {since}") + logs = fetch_github_audit_logs(since) + if logs: + ingest.ingest(logs, CHRONICLE_DATA_TYPE) + print(f"[INFO] Successfully ingested {len(logs)} log(s).") + else: + print("[INFO] No new events to ingest.") + save_last_run_timestamp(datetime.now(timezone.utc)) + +if __name__ == "__main__": + main() + diff --git a/github_actions/github-chronicle-ingestor/requirements.txt b/github_actions/github-chronicle-ingestor/requirements.txt new file mode 100644 index 0000000..524b652 --- /dev/null +++ b/github_actions/github-chronicle-ingestor/requirements.txt @@ -0,0 +1,2 @@ +requests +google-auth>=2.0.0 diff --git a/github_actions/snowflake-chronicle-ingestor/README.md b/github_actions/snowflake-chronicle-ingestor/README.md new file mode 100644 index 0000000..fb31d10 --- /dev/null +++ b/github_actions/snowflake-chronicle-ingestor/README.md @@ -0,0 +1,72 @@ +# Snowflake Audit Log Ingestor for Chronicle + +This module ingests audit logs from Snowflake's ACCOUNT_USAGE views into **Google Chronicle**. + +--- + +## πŸ” What It Does + +- Connects to Snowflake using secure credentials +- Fetches data from key `ACCOUNT_USAGE` views like `LOGIN_HISTORY`, `QUERY_HISTORY`, and more +- Transforms and ingests logs into Chronicle using the Unstructured Ingestion API + +--- + +## πŸ›  Requirements + +- Python 3.7+ +- Snowflake account with access to `SNOWFLAKE.ACCOUNT_USAGE` +- GitHub Secrets: + - `SNOWFLAKE_USER` + - `SNOWFLAKE_PASSWORD` + - `SNOWFLAKE_ACCOUNT` + - `SNOWFLAKE_WAREHOUSE` + - `SNOWFLAKE_ROLE` + - `CHRONICLE_CUSTOMER_ID` + - `CHRONICLE_REGION` + - `CHRONICLE_NAMESPACE` (optional) + - `CHRONICLE_CREDENTIALS_JSON` (base64-encoded) + +--- + +## βš™ GitHub Actions Workflow + +Location: `.github/workflows/snowflake.yml` + +This workflow: +- Runs **manually only** +- Uses secrets for Snowflake and Chronicle access +- Executes the ingestion script on demand + +--- + +## πŸ§ͺ Running Locally + +```bash +export SNOWFLAKE_USER=... +export SNOWFLAKE_PASSWORD=... +export SNOWFLAKE_ACCOUNT=... +export SNOWFLAKE_WAREHOUSE=... +export SNOWFLAKE_ROLE=... +export CHRONICLE_CUSTOMER_ID=... +export CHRONICLE_REGION=... +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/chronicle_sa.json + +python main.py --creds-file /path/to/chronicle_sa.json +``` + +--- + +## πŸ“„ Notes + +- Each view is queried independently using a fixed 15-minute lookback window +- Ingested logs include a `"log_source"` field to identify the originating view +- Date/time fields are serialized to ISO format for Chronicle compatibility + +--- + +## 🀝 Contributions + +Have improvements or additional views to include? Submit a PR or open an issue! + + diff --git a/github_actions/snowflake-chronicle-ingestor/main.py b/github_actions/snowflake-chronicle-ingestor/main.py new file mode 100644 index 0000000..f75590c --- /dev/null +++ b/github_actions/snowflake-chronicle-ingestor/main.py @@ -0,0 +1,89 @@ +import os +import snowflake.connector +from datetime import datetime +from common import ingest, utils + +LOG_TYPE = "SNOWFLAKE" + +VIEW_TIMESTAMP_COLUMNS = { + "LOGIN_HISTORY": "EVENT_TIMESTAMP", + "QUERY_HISTORY": "START_TIME", + "ACCESS_HISTORY": "QUERY_START_TIME", + "TASK_HISTORY": "QUERY_START_TIME", + "MATERIALIZED_VIEW_REFRESH_HISTORY": "START_TIME", + "PIPE_USAGE_HISTORY": "START_TIME", + "REPLICATION_USAGE_HISTORY": "START_TIME", + "WAREHOUSE_LOAD_HISTORY": "START_TIME", + "WAREHOUSE_METERING_HISTORY": "START_TIME", +} + +def connect_to_snowflake(): + print("[INFO] Connecting to Snowflake...") + return snowflake.connector.connect( + user=os.getenv("SNOWFLAKE_USER"), + password=os.getenv("SNOWFLAKE_PASSWORD"), + account=os.getenv("SNOWFLAKE_ACCOUNT"), + warehouse=os.getenv("SNOWFLAKE_WAREHOUSE"), + role=os.getenv("SNOWFLAKE_ROLE"), + database="SNOWFLAKE", + schema="ACCOUNT_USAGE" + ) + +def fetch_view_data(conn, view_name, since): + timestamp_column = VIEW_TIMESTAMP_COLUMNS.get(view_name) + if not timestamp_column: + print(f"[WARNING] No timestamp column configured for view {view_name}") + return [] + + query = f""" + SELECT * + FROM SNOWFLAKE.ACCOUNT_USAGE.{view_name} + WHERE {timestamp_column} >= DATEADD(minute, -15, CURRENT_TIMESTAMP()) + """ + + cursor = conn.cursor() + try: + cursor.execute(query) + query_id = cursor.sfqid + print(f"[INFO] Querying: {view_name}") + print(f"[INFO] Snowflake Query ID: {query_id}") + + rows = cursor.fetchall() + columns = [col[0] for col in cursor.description] + logs = [dict(zip(columns, row)) for row in rows] + + for log in logs: + log["log_source"] = view_name + + print(f"[INFO] Retrieved {len(logs)} rows from {view_name}.") + return logs + except Exception as e: + print(f"[ERROR] Fetching from {view_name}: {e}") + return [] + finally: + cursor.close() + +def serialize_for_json(obj): + if isinstance(obj, dict): + return {k: serialize_for_json(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [serialize_for_json(i) for i in obj] + elif isinstance(obj, datetime): + return obj.isoformat() + return obj + +def main(): + print("[INFO] Starting Snowflake audit ingestion...") + conn = connect_to_snowflake() + + all_logs = [] + for view in VIEW_TIMESTAMP_COLUMNS: + logs = fetch_view_data(conn, view, utils.get_last_run_at()) + print(f"[INFO] Ingesting {len(logs)} logs from {view}") + all_logs.extend([serialize_for_json(log) for log in logs]) + + ingest.ingest(all_logs, LOG_TYPE) + +if __name__ == "__main__": + main() + diff --git a/github_actions/snowflake-chronicle-ingestor/requirements.txt b/github_actions/snowflake-chronicle-ingestor/requirements.txt new file mode 100644 index 0000000..bd0d823 --- /dev/null +++ b/github_actions/snowflake-chronicle-ingestor/requirements.txt @@ -0,0 +1,5 @@ +snowflake-connector-python +google-auth +google-auth-oauthlib +google-api-python-client +requests diff --git a/github_actions/thinkst-audit-chronicle-ingestor/README.md b/github_actions/thinkst-audit-chronicle-ingestor/README.md new file mode 100644 index 0000000..883e2e7 --- /dev/null +++ b/github_actions/thinkst-audit-chronicle-ingestor/README.md @@ -0,0 +1,67 @@ +# Thinkst Canary Audit Log Ingestor for Chronicle + +This module ingests audit logs from **Thinkst Canary** into **Google Chronicle** using the Unstructured Ingestion API. + +--- + +## πŸ” What It Does + +- Fetches audit trail logs from a Thinkst Canary Console +- Filters events since the last successful run +- Sends logs to Chronicle for detection and investigation +- Tracks ingestion history using a local timestamp file + +--- + +## πŸ›  Requirements + +- Python 3.7+ +- Valid Thinkst Canary API credentials +- GitHub Secrets: + - `CANARY_CONSOLE_ID` + - `CANARY_AUTH_TOKEN` + - `CHRONICLE_CUSTOMER_ID` + - `CHRONICLE_REGION` + - `CHRONICLE_NAMESPACE` (optional) + - `CHRONICLE_CREDENTIALS_JSON` (base64-encoded) + +--- + +## βš™ GitHub Actions Workflow + +Location: `.github/workflows/thinkst-canary.yml` + +This workflow: +- Is **manual-only** +- Uses GitHub secrets to authenticate to Thinkst and Chronicle +- Calls the script to ingest new audit logs on demand + +--- + +## πŸ§ͺ Running Locally + +```bash +export CANARY_CONSOLE_ID=your-console-id +export CANARY_AUTH_TOKEN=your-auth-token +export CHRONICLE_CUSTOMER_ID=... +export CHRONICLE_REGION=... +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/chronicle_sa.json + +python main.py --creds-file /path/to/chronicle_sa.json +``` + +--- + +## πŸ“„ Notes + +- Uses `.canary_last_run.json` to avoid duplicate ingestion +- Date format must match `"%Y-%m-%d %H:%M:%S UTC+0000"` expected by Thinkst +- Designed for Thinkst customers that want centralized log correlation in Chronicle + +--- + +## 🀝 Contributions + +Have a feature request or bug fix? Open a pull request or issue! + + diff --git a/github_actions/thinkst-audit-chronicle-ingestor/main.py b/github_actions/thinkst-audit-chronicle-ingestor/main.py new file mode 100644 index 0000000..c347e37 --- /dev/null +++ b/github_actions/thinkst-audit-chronicle-ingestor/main.py @@ -0,0 +1,73 @@ +"""Fetch Thinkst Canary audit logs and ingest into Chronicle.""" + +import os +import json +import requests +import datetime + +from common import ingest + +LOG_TYPE = "THINKST_CANARY" +CANARY_AUDIT_URL = "https://{console}.canary.tools/api/v1/audit_trail/fetch" +LAST_RUN_FILE = ".canary_last_run.json" +DATE_FORMAT = "%Y-%m-%d %H:%M:%S UTC+0000" + +def load_last_run_timestamp(): + try: + if os.path.exists(LAST_RUN_FILE): + with open(LAST_RUN_FILE, "r") as f: + data = json.load(f) + ts = data.get("last_run") + if ts: + return datetime.datetime.strptime(ts, DATE_FORMAT).replace(tzinfo=datetime.timezone.utc) + except Exception: + pass + return datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc) - datetime.timedelta(hours=1) + +def save_last_run_timestamp(timestamp): + with open(LAST_RUN_FILE, "w") as f: + json.dump({"last_run": timestamp.strftime(DATE_FORMAT)}, f) + +def get_canary_audit_logs(console_id, auth_token, since): + url = CANARY_AUDIT_URL.format(console=console_id) + params = {"auth_token": auth_token} + response = requests.get(url, params=params) + response.raise_for_status() + + raw_data = response.json() + audit_events = raw_data.get("audit_trail", []) + new_events = [] + + for event in audit_events: + event_time_str = event.get("timestamp", "") + try: + event_time = datetime.datetime.strptime(event_time_str, DATE_FORMAT).replace(tzinfo=datetime.timezone.utc) + if event_time > since: + new_events.append(event) + except Exception: + continue + + return new_events + +def main(req=None): + console_id = os.getenv("CANARY_CONSOLE_ID") + auth_token = os.getenv("CANARY_AUTH_TOKEN") + + if not console_id or not auth_token: + raise RuntimeError("Missing CANARY_CONSOLE_ID or CANARY_AUTH_TOKEN.") + + last_run_time = load_last_run_timestamp() + events = get_canary_audit_logs(console_id, auth_token, last_run_time) + + if events: + ingest.ingest(events, LOG_TYPE) + print(f"[INFO] Ingested {len(events)} events.") + else: + print("[INFO] No new events found.") + + now = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc) + save_last_run_timestamp(now) + +if __name__ == "__main__": + main() + diff --git a/github_actions/thinkst-audit-chronicle-ingestor/requirements.txt b/github_actions/thinkst-audit-chronicle-ingestor/requirements.txt new file mode 100644 index 0000000..524b652 --- /dev/null +++ b/github_actions/thinkst-audit-chronicle-ingestor/requirements.txt @@ -0,0 +1,2 @@ +requests +google-auth>=2.0.0