From fe80fd758453cd95b9c63b578ad509880692015e Mon Sep 17 00:00:00 2001 From: Baher Al Hakim Date: Wed, 18 Mar 2026 15:20:58 +0100 Subject: [PATCH 1/6] feat(quality): implement sender-type style anchors and exemplar cache --- CONTRIBUTING.md | 3 +- README.md | 2 + app/api/feedback_routes.py | 11 ++- app/api/stream_routes.py | 19 +++- app/core/config.py | 11 +++ app/generation/service.py | 83 ++++++++++++++++- docs/RELEASE_GUARDRAILS.md | 70 +++++++++++++++ scripts/setup_wizard.py | 148 ++++++++++++++++++++++--------- tests/test_config.py | 14 +++ tests/test_style_anchor_cache.py | 84 ++++++++++++++++++ var/youos-server.log | 26 ++++++ youos_config.yaml | 13 ++- 12 files changed, 433 insertions(+), 51 deletions(-) create mode 100644 docs/RELEASE_GUARDRAILS.md create mode 100644 tests/test_style_anchor_cache.py create mode 100644 var/youos-server.log diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6cee72c..7b3f473 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,7 +46,8 @@ Configuration is in `pyproject.toml` (line length 100, Python 3.11 target). 3. Add tests for new functionality 4. Ensure all tests pass: `python -m pytest tests/ -q` 5. Ensure linting passes: `ruff check .` -6. Submit a PR with a clear description of the change +6. Run the release checklist in `docs/RELEASE_GUARDRAILS.md` for any user-facing/docs/submission change +7. Submit a PR with a clear description of the change ## Architecture overview diff --git a/README.md b/README.md index 1100053..1606b21 100644 --- a/README.md +++ b/README.md @@ -35,10 +35,12 @@ Gmail (sent mail) Your feedback - Learns your writing style — richer persona: bullet point rate, directness score, sentence length, paragraph style; EWMA-weighted toward recent emails - Persona re-analysis is incremental (recent 90 days × 3 weight), with full weekly refresh; confidence intervals (p25/p75) shown in prompts - Per-sender-type personas: different voice, length, greeting, and closing for internal, external client, and personal contacts +- Sender-type style anchors: explicit prompt slot (`[STYLE ANCHOR — internal|client|personal]`) to stabilize first-draft tone by audience - Per-account corpus isolation — drafts for work emails draw from work history; personal from personal - Greets people by first name, closes in your style — greeting and closing injected from persona config per contact type - Classifies multi-intent (meeting + urgent, etc.), boosts matching exemplars; per-intent reply length calibrated from corpus - Drafts grounded in score-ranked few-shot exemplars (confidence-annotated, thread-deduplicated); exemplar reply text preserved (600 chars), inbound trimmed (400) +- Exemplar cache by intent+sender-type (TTL + feedback-triggered invalidation) improves consistency and reduces repeated ranking churn - Prompt token budget enforced — exemplars auto-trimmed if prompt exceeds 2000 tokens - Confidence thresholds are relative (mean±σ of retrieval scores), not hardcoded - Subject line + topic-aware retrieval; FTS queries expanded with email vocabulary synonyms diff --git a/app/api/feedback_routes.py b/app/api/feedback_routes.py index 47e8148..10844d6 100644 --- a/app/api/feedback_routes.py +++ b/app/api/feedback_routes.py @@ -15,7 +15,7 @@ from app.core.facts_extractor import extract_and_save from app.core.rate_limit import RATE_LIMIT_RESPONSE, draft_limiter from app.db.bootstrap import resolve_sqlite_path -from app.generation.service import DraftRequest, generate_draft +from app.generation.service import DraftRequest, clear_exemplar_cache, generate_draft logger = logging.getLogger(__name__) @@ -198,7 +198,7 @@ def feedback_generate(body: GenerateBody, request: Request) -> dict: """INSERT INTO draft_history (inbound_text, sender, generated_draft, confidence, model_used, retrieval_method) VALUES (?, ?, ?, ?, ?, ?)""", - (body.inbound_text, body.sender, response.draft, response.confidence, response.model_used, response.retrieval_method), + (body.inbound_text, body.sender, response.draft, response.confidence, response.model_used, f"{response.retrieval_method}|cache_hit={int(response.exemplar_cache_hit)}|cache_key={response.exemplar_cache_key or ''}"), ) conn.commit() finally: @@ -213,6 +213,8 @@ def feedback_generate(body: GenerateBody, request: Request) -> dict: "confidence_reason": response.confidence_reason, "confidence_warning": response.confidence == "low", "suggested_subject": response.suggested_subject, + "exemplar_cache_hit": response.exemplar_cache_hit, + "exemplar_cache_key": response.exemplar_cache_key, } @@ -273,9 +275,12 @@ def feedback_submit(body: SubmitBody, request: Request) -> dict: quality_score = max(0.3, min(1.3, quality_score)) conn.execute("UPDATE reply_pairs SET quality_score = ? WHERE id = ?", (round(quality_score, 4), rp_id)) conn.commit() + # Invalidate exemplar cache on successful quality update + clear_exemplar_cache() except Exception: - logger.warning("Failed to update quality_score for reply pair", exc_info=True) + logger.warning("Failed to update quality_score for reply pair or clear cache", exc_info=True) + clear_exemplar_cache() total = conn.execute("SELECT COUNT(*) FROM feedback_pairs").fetchone()[0] finally: conn.close() diff --git a/app/api/stream_routes.py b/app/api/stream_routes.py index 6359155..74e57af 100644 --- a/app/api/stream_routes.py +++ b/app/api/stream_routes.py @@ -19,6 +19,10 @@ _precedent_summary, _score_confidence, assemble_prompt, + _apply_cached_order, + _get_cached_exemplar_ids, + _top_exemplar_source_ids, + _update_exemplar_cache, generate_draft, lookup_sender_profile, ) @@ -59,6 +63,18 @@ def _stream_generate(body: StreamBody, settings): ) reply_pairs = retrieval_response.reply_pairs + + # Apply exemplar caching (read + write) + from app.core.intent import classify_intents_multi + intents = classify_intents_multi(clean_inbound) + detected_intent = intents[0] + + cached_ids, exemplar_cache_hit, exemplar_cache_key = _get_cached_exemplar_ids(detected_intent, sender_type_hint) + reply_pairs = _apply_cached_order(reply_pairs, cached_ids) + + selected_ids = _top_exemplar_source_ids(reply_pairs) + _update_exemplar_cache(detected_intent, sender_type_hint, selected_ids) + confidence, _ = _score_confidence(reply_pairs) precedent_used = [_precedent_summary(rp) for rp in reply_pairs] detected_mode = retrieval_response.detected_mode @@ -80,6 +96,7 @@ def _stream_generate(body: StreamBody, settings): detected_mode=detected_mode, tone_hint=body.tone_hint, sender_context=sender_context, + sender_type=sender_type_hint, user_prompt=body.user_prompt, ) @@ -117,7 +134,7 @@ def _stream_generate(body: StreamBody, settings): except Exception as exc: yield f"data: {json.dumps({'token': f'[generation failed: {exc}]'})}\n\n" - yield f"data: {json.dumps({'done': True, 'confidence': confidence, 'precedent_used': precedent_used})}\n\n" + yield f"data: {json.dumps({'done': True, 'confidence': confidence, 'precedent_used': precedent_used, 'exemplar_cache_hit': exemplar_cache_hit, 'exemplar_cache_key': exemplar_cache_key})}\n\n" @router.post("/stream") diff --git a/app/core/config.py b/app/core/config.py index 4710137..5b36dae 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -130,6 +130,17 @@ def get_autoresearch_iterations(config: dict[str, Any] | None = None) -> int: return int(cfg.get("autoresearch", {}).get("iterations", 80)) +def get_persona_mode_config(sender_type: str, config: dict[str, Any] | None = None) -> dict[str, Any]: + cfg = config or load_config() + return cfg.get("persona", {}).get("modes", {}).get(sender_type, {}) + + +def get_persona_style_anchor(sender_type: str, config: dict[str, Any] | None = None) -> str | None: + cfg = config or load_config() + mode_config = get_persona_mode_config(sender_type, config) + return mode_config.get("style_anchor") + + def get_ollama_config(config: dict[str, Any] | None = None) -> dict[str, Any]: cfg = config or load_config() return cfg.get("model", {}).get("ollama", {}) diff --git a/app/generation/service.py b/app/generation/service.py index 485f755..3ea7625 100644 --- a/app/generation/service.py +++ b/app/generation/service.py @@ -11,7 +11,14 @@ import yaml -from app.core.config import get_account_for_sender, get_base_model, get_model_fallback, get_user_name, get_user_names +from app.core.config import ( + get_account_for_sender, + get_base_model, + get_model_fallback, + get_persona_style_anchor, + get_user_name, + get_user_names, +) from app.core.sender import classify_sender, extract_domain, first_name_from_display_name from app.core.text_utils import strip_quoted_text from app.db.bootstrap import resolve_sqlite_path @@ -24,6 +31,61 @@ logger = logging.getLogger(__name__) +_EXEMPLAR_CACHE_TTL_SECONDS = 30 * 60 +_exemplar_cache: dict[tuple[str, str], dict[str, Any]] = {} + + +def clear_exemplar_cache() -> None: + _exemplar_cache.clear() + + +def _cache_key(intent_hint: str | None, sender_type: str | None) -> tuple[str, str]: + return ((intent_hint or "general").strip().lower(), (sender_type or "unknown").strip().lower()) + + +def _get_cached_exemplar_ids(intent_hint: str | None, sender_type: str | None) -> tuple[list[str], bool, str]: + import time + + key = _cache_key(intent_hint, sender_type) + key_str = f"{key[0]}::{key[1]}" + entry = _exemplar_cache.get(key) + if not entry: + logger.info("Exemplar cache MISS key=%s", key_str) + return [], False, key_str + if time.time() - float(entry.get("ts", 0.0)) > _EXEMPLAR_CACHE_TTL_SECONDS: + _exemplar_cache.pop(key, None) + logger.info("Exemplar cache EXPIRED key=%s", key_str) + return [], False, key_str + ids = [str(x) for x in entry.get("ids", []) if x] + logger.info("Exemplar cache HIT key=%s size=%d", key_str, len(ids)) + return ids, True, key_str + + +def _update_exemplar_cache(intent_hint: str | None, sender_type: str | None, source_ids: list[str]) -> None: + import time + + key = _cache_key(intent_hint, sender_type) + _exemplar_cache[key] = {"ts": time.time(), "ids": source_ids[:10]} + + +def _apply_cached_order(reply_pairs: list[RetrievalMatch], cached_ids: list[str]) -> list[RetrievalMatch]: + if not cached_ids or not reply_pairs: + return reply_pairs + rank = {sid: i for i, sid in enumerate(cached_ids)} + cached = [rp for rp in reply_pairs if rp.source_id in rank] + uncached = [rp for rp in reply_pairs if rp.source_id not in rank] + cached.sort(key=lambda rp: rank.get(rp.source_id, 9999)) + return cached + uncached + + +def _top_exemplar_source_ids(reply_pairs: list[RetrievalMatch], limit: int = 5) -> list[str]: + ranked = sorted( + [rp for rp in reply_pairs if rp.source_id], + key=lambda rp: ((rp.metadata or {}).get("quality_score", 1.0), rp.score), + reverse=True, + ) + return [rp.source_id for rp in ranked[:limit]] + @dataclass(slots=True) class DraftRequest: @@ -56,6 +118,8 @@ class DraftResponse: suggested_subject: str | None = None token_estimate: int | None = None empty_output_retried: bool = False + exemplar_cache_hit: bool = False + exemplar_cache_key: str | None = None def to_dict(self) -> dict[str, Any]: return asdict(self) @@ -297,6 +361,7 @@ def _precedent_summary(match: RetrievalMatch) -> dict[str, Any]: "title": match.title, "snippet": match.snippet, "score": match.score, + "reply_pair_id": match.reply_pair_id, } @@ -657,6 +722,12 @@ def assemble_prompt( persona_block = "\n".join(persona_lines) + style_anchor_block = "" + if sender_type: + style_anchor = get_persona_style_anchor(sender_type) + if style_anchor: + style_anchor_block = f"\n[STYLE ANCHOR — {sender_type}]\n{style_anchor.strip()}\n" + # Build optional context lines context_lines: list[str] = [] if detected_mode: @@ -700,6 +771,7 @@ def assemble_prompt( f"{system.strip()}\n" f"{persona_block}\n" f"{context_block}" + f"{style_anchor_block}" f"{sender_block}" f"{facts_block}" f"{language_block}" @@ -949,6 +1021,13 @@ def generate_draft( detected_mode = request.mode or retrieval_response.detected_mode reply_pairs = retrieval_response.reply_pairs + + cached_ids, exemplar_cache_hit, exemplar_cache_key = _get_cached_exemplar_ids(detected_intent, sender_type_hint) + reply_pairs = _apply_cached_order(reply_pairs, cached_ids) + + selected_ids = _top_exemplar_source_ids(reply_pairs) + _update_exemplar_cache(detected_intent, sender_type_hint, selected_ids) + # Build score stats dict from retrieval response score_stats = None if retrieval_response.mean_score is not None: @@ -1145,4 +1224,6 @@ def generate_draft( suggested_subject=suggested_subject, token_estimate=token_estimate, empty_output_retried=empty_output_retried, + exemplar_cache_hit=exemplar_cache_hit, + exemplar_cache_key=exemplar_cache_key, ) diff --git a/docs/RELEASE_GUARDRAILS.md b/docs/RELEASE_GUARDRAILS.md new file mode 100644 index 0000000..5be0acb --- /dev/null +++ b/docs/RELEASE_GUARDRAILS.md @@ -0,0 +1,70 @@ +# YouOS Release Guardrails (Development, Docs, Submission) + +This checklist is mandatory before any public submission/update (ClawHub, GitHub release notes, listing refreshes). + +## 1) Metadata consistency (must match reality) + +- `SKILL.md` must describe YouOS as a **full local Python app**, not instruction-only. +- `clawhub.json` must declare runtime requirements: + - `requires.bins`: `python3`, `gog` + - `requires.platform`: `darwin` + - `requires.arch`: `arm64` +- Version must be consistent across: + - `pyproject.toml` + - `clawhub.json` + - `app/main.py` + - `app/api/stats_routes.py` + - UI footers/templates + - `CHANGELOG.md` + +## 2) Privacy/security defaults (no risky defaults) + +- Default model fallback must be local-first safe: + - `model.fallback: none` by default (external fallback opt-in) +- Default web bind must be local only: + - `server.host: 127.0.0.1` +- Setup flow must not leave auth wide-open: + - if user leaves PIN empty, generate a PIN automatically +- Docs must explicitly state: + - external fallback may send email/context externally if enabled + - strict local-only mode: set `model.fallback: none` + +## 3) Documentation integrity + +- Public docs/UI text must match shipped behavior. +- Remove stale references to removed features. +- Keep install path explicit (manual pip) and prerequisites clear. +- Keep examples aligned with current defaults (`127.0.0.1`, current version). + +## 4) CI quality gate (required) + +Before push/merge: +- `ruff check .` passes +- tests pass (at minimum changed scopes + known CI-sensitive suites) +- if modifying training/export filters, run: + - `tests/test_export_quality_gate.py` + - `tests/test_finetune_improvements.py` + +## 5) Submission package hygiene + +Use a review-friendly package that excludes non-essential noise: +- exclude caches, logs, runtime state, local instances, binaries/screenshots when form requires text-only +- include only essentials for scanner understanding: + - `SKILL.md`, `clawhub.json`, `pyproject.toml`, `README.md`, `PRIVACY.md`, `app/`, needed `scripts/`, required `configs/` + +## 6) Release process contract + +- Every upload increments version (no reuse). +- Update `CHANGELOG.md` with concise, accurate bullets. +- Verify GitHub `main` contains the release commit before submission. +- If CI fails, patch and re-run before re-submitting. + +## 7) Non-commit local state + +Never commit local runtime state: +- `youos_config.yaml` (instance/local values) +- `var/`, `instances/`, `.venv/`, caches + +--- + +If a change conflicts with this document, update this document in the same PR and explain why. diff --git a/scripts/setup_wizard.py b/scripts/setup_wizard.py index 8f83616..4fa4478 100644 --- a/scripts/setup_wizard.py +++ b/scripts/setup_wizard.py @@ -10,6 +10,8 @@ import sqlite3 import subprocess import sys +import secrets # For random PIN generation +from datetime import datetime from pathlib import Path ROOT_DIR = Path(__file__).resolve().parents[1] @@ -255,19 +257,71 @@ def _verify_accounts(emails: list[str]) -> list[str]: def _run_ingestion(config: dict) -> dict: - """Run ingestion for configured accounts. Returns stats.""" + """Run ingestion for configured accounts. Returns stats. + + Supports quick-start foreground ingest + background full ingest. + """ emails = config.get("user", {}).get("emails", []) if not emails: print("No email accounts configured. Skipping ingestion.") return {"threads": 0, "reply_pairs": 0} - months = config.get("ingestion", {}).get("initial_months", 12) + ingestion_cfg = config.get("ingestion", {}) + mode = ingestion_cfg.get("mode", "balanced") + months = int(ingestion_cfg.get("initial_months", 12)) + max_threads = int(ingestion_cfg.get("max_threads", 0)) + quick_start = bool(ingestion_cfg.get("quick_start_enabled", False)) + quick_months = int(ingestion_cfg.get("quick_start_months", min(3, months))) + quick_max_threads = int(ingestion_cfg.get("quick_start_max_threads", 200)) + + def _query_for_months(m: int) -> str: + return f"in:sent newer_than:{m}m" + + def _run_foreground(email: str, *, query: str, thread_cap: int, timeout_s: int = 1800) -> tuple[int, int]: + print(f"\nIngesting {email} ({query}, max_threads={thread_cap})...") + local_threads = 0 + local_pairs = 0 + result = subprocess.run( + [ + sys.executable, + str(ROOT_DIR / "scripts" / "ingest_gmail_threads.py"), + "--live", + "--account", + email, + "--query", + query, + "--max-threads", + str(thread_cap), + ], + capture_output=True, + text=True, + timeout=timeout_s, + ) + print(result.stdout) + if result.returncode == 0: + for line in result.stdout.splitlines(): + if "threads" in line.lower() and "reply" in line.lower(): + import re + + thread_match = re.search(r"(\d+)\s*threads?", line) + pair_match = re.search(r"(\d+)\s*reply.?pairs?", line) + if thread_match: + local_threads += int(thread_match.group(1)) + if pair_match: + local_pairs += int(pair_match.group(1)) + else: + print(f" WARN: Ingestion for {email} had issues") + if result.stderr: + print(f" {result.stderr[:300]}") + return local_threads, local_pairs print() print("--- Corpus Ingestion ---") print(f" Accounts: {', '.join(emails)}") - print(f" Range: last {months} months of sent mail") - print(" Estimated time: 10-30 minutes depending on volume") + print(f" Ingestion mode: {mode}") + print(f" Full range: last {months} months | max_threads={max_threads}") + if quick_start: + print(f" Quick start: last {quick_months} months | max_threads={quick_max_threads}") print() proceed = input("Start ingestion? [Y/n] ").strip().lower() @@ -278,46 +332,52 @@ def _run_ingestion(config: dict) -> dict: total_threads = 0 total_pairs = 0 - for email in emails: - print(f"\nIngesting {email}...") - try: - result = subprocess.run( - [ - sys.executable, - str(ROOT_DIR / "scripts" / "ingest_gmail_threads.py"), - "--live", - "--account", - email, - "--query", - "in:sent", - "--max-threads", - "0", - ], - capture_output=True, - text=True, - timeout=1800, - ) - print(result.stdout) - if result.returncode == 0: - # Parse counts from output - for line in result.stdout.splitlines(): - if "threads" in line.lower() and "reply" in line.lower(): - import re - - thread_match = re.search(r"(\d+)\s*threads?", line) - pair_match = re.search(r"(\d+)\s*reply.?pairs?", line) - if thread_match: - total_threads += int(thread_match.group(1)) - if pair_match: - total_pairs += int(pair_match.group(1)) - else: - print(f" WARN: Ingestion for {email} had issues") - if result.stderr: - print(f" {result.stderr[:200]}") - except subprocess.TimeoutExpired: - print(f" TIMEOUT: Ingestion for {email} timed out after 30 minutes") - except Exception as exc: - print(f" ERROR: {exc}") + try: + if quick_start: + print("\nQuick start enabled: running a smaller ingest so you can use YouOS faster...") + for email in emails: + t, p = _run_foreground(email, query=_query_for_months(quick_months), thread_cap=quick_max_threads, timeout_s=1200) + total_threads += t + total_pairs += p + + print(f"\nQuick-start corpus: {total_threads} threads | {total_pairs} reply pairs") + print("You can start drafting now while full ingestion continues in the background.") + + full_query = _query_for_months(months) + log_dir = ROOT_DIR / "var" + log_dir.mkdir(parents=True, exist_ok=True) + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + + for email in emails: + safe_email = email.replace("@", "_at_").replace(".", "_") + log_path = log_dir / f"ingest-background-{safe_email}-{stamp}.log" + with open(log_path, "w", encoding="utf-8") as logf: + proc = subprocess.Popen( + [ + sys.executable, + str(ROOT_DIR / "scripts" / "ingest_gmail_threads.py"), + "--live", + "--account", + email, + "--query", + full_query, + "--max-threads", + str(max_threads), + ], + stdout=logf, + stderr=subprocess.STDOUT, + ) + print(f" Background ingest started for {email} (pid={proc.pid})") + print(f" Log: {log_path}") + else: + for email in emails: + t, p = _run_foreground(email, query=_query_for_months(months), thread_cap=max_threads, timeout_s=1800) + total_threads += t + total_pairs += p + except subprocess.TimeoutExpired: + print(" TIMEOUT: Ingestion timed out") + except Exception as exc: + print(f" ERROR: {exc}") print(f"\nCorpus built: {total_threads} threads | {total_pairs} reply pairs") return {"threads": total_threads, "reply_pairs": total_pairs} diff --git a/tests/test_config.py b/tests/test_config.py index 56b4960..d46cf83 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -103,3 +103,17 @@ def test_save_and_reload(tmp_path): save_config(config, path) reloaded = _load_raw_config(path) assert reloaded["user"]["name"] == "Test" + + +def test_get_persona_style_anchor(): + from app.core.config import get_persona_style_anchor + + cfg = { + "persona": { + "modes": { + "client": {"style_anchor": "Polished and reassuring."}, + } + } + } + assert get_persona_style_anchor("client", cfg) == "Polished and reassuring." + assert get_persona_style_anchor("internal", cfg) is None diff --git a/tests/test_style_anchor_cache.py b/tests/test_style_anchor_cache.py new file mode 100644 index 0000000..cdbb64e --- /dev/null +++ b/tests/test_style_anchor_cache.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from app.generation.service import ( + _apply_cached_order, + _cache_key, + _get_cached_exemplar_ids, + _top_exemplar_source_ids, + _update_exemplar_cache, + assemble_prompt, + clear_exemplar_cache, +) +from app.retrieval.service import RetrievalMatch + + +def _rp(source_id: str, score: float = 5.0, quality_score: float = 1.0) -> RetrievalMatch: + return RetrievalMatch( + result_type="reply_pair", + score=score, + lexical_score=score, + metadata_score=0.0, + source_type="gmail", + source_id=source_id, + account_email=None, + title=None, + author=None, + external_uri=None, + thread_id=None, + created_at=None, + updated_at=None, + inbound_text=f"inbound {source_id}", + reply_text=f"reply {source_id}", + snippet=f"snippet {source_id}", + metadata={"quality_score": quality_score}, + ) + + +def test_style_anchor_is_included_for_sender_type(monkeypatch): + monkeypatch.setattr( + "app.generation.service.get_persona_style_anchor", + lambda sender_type: "Use crisp, executive language." if sender_type == "client" else None, + ) + prompt = assemble_prompt( + inbound_message="Can we align on next steps?", + reply_pairs=[], + persona={"style": {"voice": "direct"}}, + prompts={"system_prompt": "You are YouOS."}, + sender_type="client", + ) + assert "[STYLE ANCHOR — client]" in prompt + assert "Use crisp, executive language." in prompt + + +def test_exemplar_cache_hit_miss_and_key(): + clear_exemplar_cache() + ids, hit, key = _get_cached_exemplar_ids("follow_up", "client") + assert ids == [] + assert hit is False + assert key == "follow_up::client" + + _update_exemplar_cache("follow_up", "client", ["a", "b"]) + ids2, hit2, key2 = _get_cached_exemplar_ids("follow_up", "client") + assert hit2 is True + assert ids2 == ["a", "b"] + assert key2 == "follow_up::client" + + +def test_apply_cached_order_reorders_matches(): + rps = [_rp("x"), _rp("y"), _rp("z")] + ordered = _apply_cached_order(rps, ["z", "x"]) + assert [rp.source_id for rp in ordered] == ["z", "x", "y"] + + +def test_top_exemplar_source_ids_prefers_quality_then_score(): + rps = [ + _rp("low", score=9.0, quality_score=0.5), + _rp("high", score=6.0, quality_score=1.2), + _rp("mid", score=7.0, quality_score=1.0), + ] + ids = _top_exemplar_source_ids(rps, limit=2) + assert ids == ["high", "mid"] + + +def test_cache_key_normalization(): + assert _cache_key(" Follow_Up ", " Client ") == ("follow_up", "client") diff --git a/var/youos-server.log b/var/youos-server.log new file mode 100644 index 0000000..948d158 --- /dev/null +++ b/var/youos-server.log @@ -0,0 +1,26 @@ +INFO: Started server process [63604] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:8765 (Press CTRL+C to quit) +INFO: 100.96.144.103:53296 - "GET /feedback HTTP/1.1" 200 OK +INFO: 100.96.144.103:53296 - "GET /api/config HTTP/1.1" 200 OK +INFO: 100.96.144.103:53297 - "GET /api/config HTTP/1.1" 200 OK +INFO: 100.96.144.103:53317 - "GET /review-queue/next-stream?batch_size=10 HTTP/1.1" 200 OK +INFO: 100.96.144.103:54830 - "GET /feedback HTTP/1.1" 200 OK +INFO: 100.96.144.103:54831 - "GET /api/config HTTP/1.1" 200 OK +INFO: 100.96.144.103:54830 - "GET /api/config HTTP/1.1" 200 OK +INFO: 100.96.144.103:54834 - "GET /favicon.ico HTTP/1.1" 404 Not Found +INFO: 100.96.144.103:55078 - "GET /api/facts HTTP/1.1" 200 OK +INFO: 100.96.144.103:55078 - "GET /api/facts?type=user_pref HTTP/1.1" 200 OK +INFO: 100.96.144.103:55104 - "POST /api/facts HTTP/1.1" 200 OK +INFO: 100.96.144.103:55104 - "GET /api/facts?type=user_pref HTTP/1.1" 200 OK +INFO: 100.96.144.103:55147 - "POST /api/facts HTTP/1.1" 200 OK +INFO: 100.96.144.103:55147 - "GET /api/facts?type=user_pref HTTP/1.1" 200 OK +INFO: 100.96.144.103:55163 - "POST /api/facts HTTP/1.1" 200 OK +INFO: 100.96.144.103:55163 - "GET /api/facts?type=user_pref HTTP/1.1" 200 OK +INFO: 100.96.144.103:55193 - "POST /api/facts HTTP/1.1" 200 OK +INFO: 100.96.144.103:55193 - "GET /api/facts?type=user_pref HTTP/1.1" 200 OK +INFO: Shutting down +INFO: Waiting for application shutdown. +INFO: Application shutdown complete. +INFO: Finished server process [63604] diff --git a/youos_config.yaml b/youos_config.yaml index f3efc48..47c6cec 100644 --- a/youos_config.yaml +++ b/youos_config.yaml @@ -7,6 +7,10 @@ benchmarks: ingestion: accounts: [] initial_months: 12 + mode: balanced # new: balanced, quick_start, deep_history + quick_start_enabled: true + quick_start_months: 3 + quick_start_max_threads: 200 last_ingest_at: {} model: adapter_path: models/adapters/latest @@ -26,8 +30,15 @@ persona: {name}' custom_constraints: [] greeting_style: Hi [name], + modes: + internal: + style_anchor: Prefer concise, execution-focused replies with minimal ceremony and direct next steps. + client: + style_anchor: Prefer polished, reassuring, professional replies that clarify ownership, timeline, and concrete deliverables. + personal: + style_anchor: Prefer warm, natural, low-formality replies that still stay crisp and thoughtful. server: - host: 0.0.0.0 + host: 127.0.0.1 pin: '' port: 8901 tailscale: From d336c5ba3e4d3efd5c9417b61e75ce1dc9fb8e3f Mon Sep 17 00:00:00 2001 From: Baher Al Hakim Date: Wed, 18 Mar 2026 16:53:23 +0100 Subject: [PATCH 2/6] feat(YouOS): Implement persistent exemplar cache and quickstart default; fix syntax --- app/api/feedback_routes.py | 4 +- app/api/stream_routes.py | 4 +- app/db/bootstrap.py | 13 ++++++ app/generation/service.py | 93 ++++++++++++++++++++++++++++++-------- scripts/setup_wizard.py | 9 +++- youos_config.yaml | 2 +- 6 files changed, 101 insertions(+), 24 deletions(-) diff --git a/app/api/feedback_routes.py b/app/api/feedback_routes.py index 10844d6..554dd43 100644 --- a/app/api/feedback_routes.py +++ b/app/api/feedback_routes.py @@ -276,11 +276,11 @@ def feedback_submit(body: SubmitBody, request: Request) -> dict: conn.execute("UPDATE reply_pairs SET quality_score = ? WHERE id = ?", (round(quality_score, 4), rp_id)) conn.commit() # Invalidate exemplar cache on successful quality update - clear_exemplar_cache() + clear_exemplar_cache(database_url=request.app.state.settings.database_url) except Exception: logger.warning("Failed to update quality_score for reply pair or clear cache", exc_info=True) - clear_exemplar_cache() + clear_exemplar_cache(database_url=request.app.state.settings.database_url) total = conn.execute("SELECT COUNT(*) FROM feedback_pairs").fetchone()[0] finally: conn.close() diff --git a/app/api/stream_routes.py b/app/api/stream_routes.py index 74e57af..3d39b1f 100644 --- a/app/api/stream_routes.py +++ b/app/api/stream_routes.py @@ -69,11 +69,11 @@ def _stream_generate(body: StreamBody, settings): intents = classify_intents_multi(clean_inbound) detected_intent = intents[0] - cached_ids, exemplar_cache_hit, exemplar_cache_key = _get_cached_exemplar_ids(detected_intent, sender_type_hint) + cached_ids, exemplar_cache_hit, exemplar_cache_key = _get_cached_exemplar_ids(detected_intent, sender_type_hint, database_url=settings.database_url) reply_pairs = _apply_cached_order(reply_pairs, cached_ids) selected_ids = _top_exemplar_source_ids(reply_pairs) - _update_exemplar_cache(detected_intent, sender_type_hint, selected_ids) + _update_exemplar_cache(detected_intent, sender_type_hint, selected_ids, database_url=settings.database_url) confidence, _ = _score_confidence(reply_pairs) precedent_used = [_precedent_summary(rp) for rp in reply_pairs] diff --git a/app/db/bootstrap.py b/app/db/bootstrap.py index 5a99f8f..a3ff114 100644 --- a/app/db/bootstrap.py +++ b/app/db/bootstrap.py @@ -27,6 +27,7 @@ def bootstrap_database() -> Path: _migrate_sender_profiles(connection) _migrate_memory(connection) _migrate_review_streaks(connection) + _migrate_exemplar_cache(connection) _populate_fts(connection) connection.commit() finally: @@ -147,3 +148,15 @@ def _populate_fts(connection: sqlite3.Connection) -> None: ) except Exception: pass + + +def _migrate_exemplar_cache(connection: sqlite3.Connection) -> None: + """Create persistent exemplar cache table if it doesn't exist.""" + connection.execute(""" + CREATE TABLE IF NOT EXISTS exemplar_cache ( + cache_key TEXT PRIMARY KEY, + source_ids_json TEXT NOT NULL, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + """) + connection.execute("CREATE INDEX IF NOT EXISTS idx_exemplar_cache_updated ON exemplar_cache(updated_at)") diff --git a/app/generation/service.py b/app/generation/service.py index 3ea7625..1b9b25a 100644 --- a/app/generation/service.py +++ b/app/generation/service.py @@ -5,6 +5,7 @@ import re import sqlite3 import subprocess +import time from dataclasses import asdict, dataclass from pathlib import Path from typing import Any @@ -35,37 +36,93 @@ _exemplar_cache: dict[tuple[str, str], dict[str, Any]] = {} -def clear_exemplar_cache() -> None: +def clear_exemplar_cache(*, database_url: str | None = None) -> None: _exemplar_cache.clear() + if database_url: + try: + db_path = resolve_sqlite_path(database_url) + conn = sqlite3.connect(db_path) + try: + conn.execute("DELETE FROM exemplar_cache") + conn.commit() + finally: + conn.close() + except Exception: + logger.warning("Failed to clear persistent exemplar cache", exc_info=True) def _cache_key(intent_hint: str | None, sender_type: str | None) -> tuple[str, str]: return ((intent_hint or "general").strip().lower(), (sender_type or "unknown").strip().lower()) -def _get_cached_exemplar_ids(intent_hint: str | None, sender_type: str | None) -> tuple[list[str], bool, str]: - import time - +def _get_cached_exemplar_ids(intent_hint: str | None, sender_type: str | None, *, database_url: str | None = None) -> tuple[list[str], bool, str]: key = _cache_key(intent_hint, sender_type) key_str = f"{key[0]}::{key[1]}" + + # 1) In-memory fast path entry = _exemplar_cache.get(key) - if not entry: - logger.info("Exemplar cache MISS key=%s", key_str) - return [], False, key_str - if time.time() - float(entry.get("ts", 0.0)) > _EXEMPLAR_CACHE_TTL_SECONDS: + if entry: + if time.time() - float(entry.get("ts", 0.0)) <= _EXEMPLAR_CACHE_TTL_SECONDS: + ids = [str(x) for x in entry.get("ids", []) if x] + logger.info("Exemplar cache HIT(mem) key=%s size=%d", key_str, len(ids)) + return ids, True, key_str _exemplar_cache.pop(key, None) - logger.info("Exemplar cache EXPIRED key=%s", key_str) - return [], False, key_str - ids = [str(x) for x in entry.get("ids", []) if x] - logger.info("Exemplar cache HIT key=%s size=%d", key_str, len(ids)) - return ids, True, key_str + # 2) Persistent fallback + if database_url: + try: + db_path = resolve_sqlite_path(database_url) + conn = sqlite3.connect(db_path) + try: + row = conn.execute( + "SELECT source_ids_json, strftime('%s', updated_at) FROM exemplar_cache WHERE cache_key = ?", + (key_str,), + ).fetchone() + if row: + source_ids_json, updated_epoch = row + updated_epoch = int(updated_epoch or 0) + if updated_epoch and (time.time() - updated_epoch) <= _EXEMPLAR_CACHE_TTL_SECONDS: + ids = [str(x) for x in json.loads(source_ids_json or "[]") if x] + _exemplar_cache[key] = {"ts": time.time(), "ids": ids[:10]} + logger.info("Exemplar cache HIT(db) key=%s size=%d", key_str, len(ids)) + return ids, True, key_str + conn.execute("DELETE FROM exemplar_cache WHERE cache_key = ?", (key_str,)) + conn.commit() + finally: + conn.close() + except Exception: + logger.warning("Exemplar cache DB read failed for key=%s", key_str, exc_info=True) + + logger.info("Exemplar cache MISS key=%s", key_str) + return [], False, key_str -def _update_exemplar_cache(intent_hint: str | None, sender_type: str | None, source_ids: list[str]) -> None: - import time +def _update_exemplar_cache(intent_hint: str | None, sender_type: str | None, source_ids: list[str], *, database_url: str | None = None) -> None: key = _cache_key(intent_hint, sender_type) - _exemplar_cache[key] = {"ts": time.time(), "ids": source_ids[:10]} + key_str = f"{key[0]}::{key[1]}" + ids = [sid for sid in source_ids[:10] if sid] + _exemplar_cache[key] = {"ts": time.time(), "ids": ids} + + if database_url: + try: + db_path = resolve_sqlite_path(database_url) + conn = sqlite3.connect(db_path) + try: + conn.execute( + """ + INSERT INTO exemplar_cache(cache_key, source_ids_json, updated_at) + VALUES (?, ?, CURRENT_TIMESTAMP) + ON CONFLICT(cache_key) DO UPDATE SET + source_ids_json=excluded.source_ids_json, + updated_at=CURRENT_TIMESTAMP + """, + (key_str, json.dumps(ids)), + ) + conn.commit() + finally: + conn.close() + except Exception: + logger.warning("Exemplar cache DB write failed for key=%s", key_str, exc_info=True) def _apply_cached_order(reply_pairs: list[RetrievalMatch], cached_ids: list[str]) -> list[RetrievalMatch]: @@ -1022,11 +1079,11 @@ def generate_draft( detected_mode = request.mode or retrieval_response.detected_mode reply_pairs = retrieval_response.reply_pairs - cached_ids, exemplar_cache_hit, exemplar_cache_key = _get_cached_exemplar_ids(detected_intent, sender_type_hint) + cached_ids, exemplar_cache_hit, exemplar_cache_key = _get_cached_exemplar_ids(detected_intent, sender_type_hint, database_url=database_url) reply_pairs = _apply_cached_order(reply_pairs, cached_ids) selected_ids = _top_exemplar_source_ids(reply_pairs) - _update_exemplar_cache(detected_intent, sender_type_hint, selected_ids) + _update_exemplar_cache(detected_intent, sender_type_hint, selected_ids, database_url=database_url) # Build score stats dict from retrieval response score_stats = None diff --git a/scripts/setup_wizard.py b/scripts/setup_wizard.py index 4fa4478..1c5f8be 100644 --- a/scripts/setup_wizard.py +++ b/scripts/setup_wizard.py @@ -31,7 +31,7 @@ def _print_banner(): print("YouOS learns how YOU write email - from your own sent history.") print("It runs entirely on your Mac. Your data never leaves your machine.") print() - print("This setup takes about 15 minutes (mostly waiting for ingestion).") + print("This setup takes about 5 minutes to first draft (full ingestion continues in background).") print("Let's get started.") print() @@ -799,6 +799,13 @@ def main() -> None: config["user"]["internal_domains"] = identity["internal_domains"] config.setdefault("ingestion", {}) config["ingestion"]["accounts"] = identity["emails"] + # Quickstart-first onboarding defaults + config["ingestion"].setdefault("mode", "quick_start") + config["ingestion"].setdefault("initial_months", 12) + config["ingestion"].setdefault("max_threads", 0) + config["ingestion"].setdefault("quick_start_enabled", True) + config["ingestion"].setdefault("quick_start_months", 3) + config["ingestion"].setdefault("quick_start_max_threads", 200) config.setdefault("review", {})["batch_size"] = 10 config["review"].setdefault("draft_model", "claude") # 'claude', 'local', or 'auto' diff --git a/youos_config.yaml b/youos_config.yaml index 47c6cec..bc0cc52 100644 --- a/youos_config.yaml +++ b/youos_config.yaml @@ -7,7 +7,7 @@ benchmarks: ingestion: accounts: [] initial_months: 12 - mode: balanced # new: balanced, quick_start, deep_history + mode: quick_start # default onboarding path: quick first value, background full ingest quick_start_enabled: true quick_start_months: 3 quick_start_max_threads: 200 From 4a58db3a0505e153fb7c40d990527b6fe1285768 Mon Sep 17 00:00:00 2001 From: Baher Al Hakim Date: Wed, 18 Mar 2026 16:53:40 +0100 Subject: [PATCH 3/6] feat(YouOS): Display edit-reduction metrics in dashboard --- templates/stats.html | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/templates/stats.html b/templates/stats.html index 643faab..5211c7e 100644 --- a/templates/stats.html +++ b/templates/stats.html @@ -265,6 +265,29 @@

System Health

? outcome.low_edit_pct.toFixed(1) + '%' : 'N/A'; document.getElementById('highRatingPct').textContent = outcome.high_rating_pct != null ? outcome.high_rating_pct.toFixed(1) + '%' : 'N/A'; + + // Outcome deltas (newly added) + var outcomeDeltas = d.outcome_deltas || {}; + if (outcomeDeltas.edit_distance_delta != null || outcomeDeltas.high_rating_delta != null) { + document.getElementById('outcomeDeltasSection').style.display = 'block'; + if (outcomeDeltas.edit_distance_delta != null) { + var deltaPct = (outcomeDeltas.edit_distance_delta * 100).toFixed(1); + var sign = deltaPct > 0 ? '+' : ''; + document.getElementById('editDistDelta').textContent = sign + deltaPct + '%'; + } else { + document.getElementById('editDistDelta').textContent = 'N/A'; + } + if (outcomeDeltas.high_rating_delta != null) { + var deltaPctHr = (outcomeDeltas.high_rating_delta * 100).toFixed(1); + var signHr = deltaPctHr > 0 ? '+' : ''; + document.getElementById('highRatingDelta').textContent = signHr + deltaPctHr + '%'; + } else { + document.getElementById('highRatingDelta').textContent = 'N/A'; + } + document.getElementById('recentPairsCount').textContent = (outcomeDeltas.recent_window_count || 0).toLocaleString(); + document.getElementById('prevPairsCount').textContent = (outcomeDeltas.previous_window_count || 0).toLocaleString(); + } + document.getElementById('embeddingPct').textContent = d.corpus.embedding_pct != null ? d.corpus.embedding_pct.toFixed(1) + '%' : 'N/A'; From f4fe6695f6383469f40a58ff4b49ec7bcc7a745f Mon Sep 17 00:00:00 2001 From: Baher Al Hakim Date: Wed, 18 Mar 2026 17:12:41 +0100 Subject: [PATCH 4/6] docs: add changelog since yesterday submission for ClawHub upload --- CHANGELOG_SINCE_YESTERDAY.md | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 CHANGELOG_SINCE_YESTERDAY.md diff --git a/CHANGELOG_SINCE_YESTERDAY.md b/CHANGELOG_SINCE_YESTERDAY.md new file mode 100644 index 0000000..220983d --- /dev/null +++ b/CHANGELOG_SINCE_YESTERDAY.md @@ -0,0 +1,37 @@ +# Changelog Since Yesterday's Submission + +Assuming yesterday’s ClawHub submission corresponds to commit `6942cbd` (release prep for v0.1.10). + +## Included commits (`6942cbd..4a58db3`) + +### 1) CI safety fix +- `75f71c5` — **fix(ci): narrow low-signal filter to avoid dropping valid training pairs** + +### 2) Draft quality improvements +- `fe80fd7` — **feat(quality): implement sender-type style anchors and exemplar cache** + - Adds sender-type style anchoring in draft prompting. + - Adds exemplar cache logic for more consistent draft selection. + +### 3) Persistence + onboarding defaults +- `d336c5b` — **feat(YouOS): Implement persistent exemplar cache and quickstart default; fix syntax** + - Persists exemplar cache in SQLite (`exemplar_cache` table) across restarts. + - Wires cache read/write/clear through API + generation paths. + - Sets quickstart-first onboarding defaults. + - Fixes syntax issue in generation service. + +### 4) Dashboard metrics visibility +- `4a58db3` — **feat(YouOS): Display edit-reduction metrics in dashboard** + - Surfaces edit-distance and high-rating deltas in stats UI. + +## Test status (targeted) +- `.venv/bin/pytest -q tests/test_style_anchor_cache.py tests/test_config.py tests/test_setup_wizard.py tests/test_generation_improvements.py` +- Result: **35 passed** + +## Files touched in this delta +- `app/api/feedback_routes.py` +- `app/api/stream_routes.py` +- `app/db/bootstrap.py` +- `app/generation/service.py` +- `scripts/setup_wizard.py` +- `templates/stats.html` +- `youos_config.yaml` From b1ba71a75b9dffe11dfd8da9e8842efd39d174ed Mon Sep 17 00:00:00 2001 From: Baher Al Hakim Date: Wed, 18 Mar 2026 17:38:06 +0100 Subject: [PATCH 5/6] chore(release): add default ClawHub release-bundle prep script --- PUBLISHING.md | 13 +++++-- scripts/prepare_clawhub_release.sh | 61 ++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 4 deletions(-) create mode 100755 scripts/prepare_clawhub_release.sh diff --git a/PUBLISHING.md b/PUBLISHING.md index 4ac699b..b8b305f 100644 --- a/PUBLISHING.md +++ b/PUBLISHING.md @@ -5,12 +5,17 @@ - GitHub account authenticated - All CI checks passing -## Steps -1. Bump version in clawhub.json and CHANGELOG.md +## Steps (default release prep flow) +1. Bump version in `clawhub.json`, `pyproject.toml`, and `CHANGELOG.md` 2. Commit: `git commit -m "chore: bump version to X.Y.Z"` 3. Tag: `git tag vX.Y.Z && git push origin vX.Y.Z` -4. Publish: `clawhub publish ./` -5. Verify on https://clawhub.com/skills/youos +4. Build a **physically clean** release folder: + - `./scripts/prepare_clawhub_release.sh` + - Optional custom output: `./scripts/prepare_clawhub_release.sh ~/Documents/youos-release-X.Y.Z` +5. Publish from that folder (not repo root): + - `cd ~/Documents/youos-release-X.Y.Z` + - `clawhub publish ./` +6. Verify on https://clawhub.com/skills/youos ## What clawhub publish does - Reads SKILL.md and clawhub.json diff --git a/scripts/prepare_clawhub_release.sh b/scripts/prepare_clawhub_release.sh new file mode 100755 index 0000000..c188ed7 --- /dev/null +++ b/scripts/prepare_clawhub_release.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Prepare a physically clean ClawHub release folder. +# Default output: ~/Documents/youos-release- + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +if [[ ! -f "clawhub.json" ]]; then + echo "Error: clawhub.json not found in $ROOT_DIR" >&2 + exit 1 +fi + +VERSION="$(python3 - <<'PY' +import json +from pathlib import Path +j=json.loads(Path('clawhub.json').read_text()) +print(j.get('version','').strip()) +PY +)" + +if [[ -z "$VERSION" ]]; then + echo "Error: could not read version from clawhub.json" >&2 + exit 1 +fi + +OUT_DIR="${1:-$HOME/Documents/youos-release-${VERSION}}" + +echo "Preparing ClawHub release bundle" +echo " source : $ROOT_DIR" +echo " version: $VERSION" +echo " output : $OUT_DIR" + +rm -rf "$OUT_DIR" +mkdir -p "$OUT_DIR" + +# Start from repo content, then apply strict excludes. +rsync -a \ + --exclude '.git/' \ + --exclude '.venv/' \ + --exclude '__pycache__/' \ + --exclude '*.pyc' \ + --exclude '.DS_Store' \ + --exclude 'node_modules/' \ + --exclude-from '.clawhubignore' \ + "$ROOT_DIR/" "$OUT_DIR/" + +# Extra hygiene: remove common local caches even if they slipped through. +find "$OUT_DIR" -name '.DS_Store' -delete || true +find "$OUT_DIR" -name '__pycache__' -type d -prune -exec rm -rf {} + || true +find "$OUT_DIR" -name '*.pyc' -delete || true + +# Sanity report +for p in .git .venv .github tests fixtures var instances data .pytest_cache .ruff_cache .herenow gif-frames youos.egg-info; do + if [[ -e "$OUT_DIR/$p" ]]; then + echo "WARN: unexpected path still present: $p" + fi +done + +echo "Done. Bundle ready at: $OUT_DIR" From 85a0aa0c1ebad735730c48049ab46a65c17bcca3 Mon Sep 17 00:00:00 2001 From: Baher Al Hakim Date: Wed, 18 Mar 2026 17:51:51 +0100 Subject: [PATCH 6/6] chore(release): enforce minimal ClawHub allowlist bundle and bump to v0.1.11 --- PUBLISHING.md | 3 +- clawhub.json | 2 +- pyproject.toml | 2 +- scripts/prepare_clawhub_release.sh | 50 ++++++++++++++++++++---------- 4 files changed, 37 insertions(+), 20 deletions(-) diff --git a/PUBLISHING.md b/PUBLISHING.md index b8b305f..2a776bf 100644 --- a/PUBLISHING.md +++ b/PUBLISHING.md @@ -9,9 +9,10 @@ 1. Bump version in `clawhub.json`, `pyproject.toml`, and `CHANGELOG.md` 2. Commit: `git commit -m "chore: bump version to X.Y.Z"` 3. Tag: `git tag vX.Y.Z && git push origin vX.Y.Z` -4. Build a **physically clean** release folder: +4. Build a **minimal allowlist release folder** (default): - `./scripts/prepare_clawhub_release.sh` - Optional custom output: `./scripts/prepare_clawhub_release.sh ~/Documents/youos-release-X.Y.Z` + - This script includes only: `app/`, `clawhub.json`, `configs/`, `PRIVACY.md`, `pyproject.toml`, `README.md`, `scripts/`, `SKILL.md` 5. Publish from that folder (not repo root): - `cd ~/Documents/youos-release-X.Y.Z` - `clawhub publish ./` diff --git a/clawhub.json b/clawhub.json index 2942f40..19388f4 100644 --- a/clawhub.json +++ b/clawhub.json @@ -1,6 +1,6 @@ { "name": "youos", - "version": "0.1.10", + "version": "0.1.11", "displayName": "YouOS \u2014 Personal Email Copilot", "description": "Learns your email writing style from your Gmail history and drafts replies that sound like you. Self-improving via LoRA fine-tuning and autoresearch. Runs locally on Apple Silicon with optional external fallback that can be disabled for strict local-only mode.", "author": "DrBaher", diff --git a/pyproject.toml b/pyproject.toml index 43fad7b..5c47753 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "youos" -version = "0.1.10" +version = "0.1.11" description = "Your personal AI email copilot — learns your writing style from Gmail and drafts replies that sound like you." readme = "README.md" requires-python = ">=3.11" diff --git a/scripts/prepare_clawhub_release.sh b/scripts/prepare_clawhub_release.sh index c188ed7..73a07cd 100755 --- a/scripts/prepare_clawhub_release.sh +++ b/scripts/prepare_clawhub_release.sh @@ -35,27 +35,43 @@ echo " output : $OUT_DIR" rm -rf "$OUT_DIR" mkdir -p "$OUT_DIR" -# Start from repo content, then apply strict excludes. -rsync -a \ - --exclude '.git/' \ - --exclude '.venv/' \ - --exclude '__pycache__/' \ - --exclude '*.pyc' \ - --exclude '.DS_Store' \ - --exclude 'node_modules/' \ - --exclude-from '.clawhubignore' \ - "$ROOT_DIR/" "$OUT_DIR/" - -# Extra hygiene: remove common local caches even if they slipped through. +# Strict allowlist (minimal bundle that passed ClawHub checks) +ALLOWED=( + "app" + "clawhub.json" + "configs" + "PRIVACY.md" + "pyproject.toml" + "README.md" + "scripts" + "SKILL.md" +) + +for item in "${ALLOWED[@]}"; do + if [[ -e "$ROOT_DIR/$item" ]]; then + rsync -a "$ROOT_DIR/$item" "$OUT_DIR/" + else + echo "WARN: missing allowlisted item: $item" + fi +done + +# Extra hygiene inside copied tree. find "$OUT_DIR" -name '.DS_Store' -delete || true find "$OUT_DIR" -name '__pycache__' -type d -prune -exec rm -rf {} + || true find "$OUT_DIR" -name '*.pyc' -delete || true -# Sanity report -for p in .git .venv .github tests fixtures var instances data .pytest_cache .ruff_cache .herenow gif-frames youos.egg-info; do - if [[ -e "$OUT_DIR/$p" ]]; then - echo "WARN: unexpected path still present: $p" +# Final strict check: only allowlisted top-level entries remain. +for entry in $(cd "$OUT_DIR" && ls -1A); do + keep=false + for allowed in "${ALLOWED[@]}"; do + if [[ "$entry" == "$allowed" ]]; then + keep=true + break + fi + done + if [[ "$keep" == false ]]; then + echo "WARN: unexpected top-level entry: $entry" fi done -echo "Done. Bundle ready at: $OUT_DIR" +echo "Done. Minimal bundle ready at: $OUT_DIR"