diff --git a/parallel_web_tools/cli/commands.py b/parallel_web_tools/cli/commands.py index 042ed0b..fb8e164 100644 --- a/parallel_web_tools/cli/commands.py +++ b/parallel_web_tools/cli/commands.py @@ -1,8 +1,10 @@ """CLI commands for Parallel.""" +import csv import json import logging import os +import tempfile from typing import Any import click @@ -39,6 +41,22 @@ # ============================================================================= +def parse_comma_separated(values: tuple[str, ...]) -> list[str]: + """Parse a tuple of values that may contain comma-separated items. + + Supports both repeated flags and comma-separated values: + --flag a,b --flag c -> ['a', 'b', 'c'] + --flag a --flag b -> ['a', 'b'] + --flag "a,b,c" -> ['a', 'b', 'c'] + """ + result = [] + for value in values: + # Split by comma and strip whitespace + parts = [p.strip() for p in value.split(",")] + result.extend(p for p in parts if p) # Skip empty strings + return result + + def write_json_output(data: dict[str, Any], output_file: str | None, output_json: bool) -> None: """Write output data to file and/or stdout as JSON. @@ -132,6 +150,51 @@ def build_config_from_args( } +def parse_inline_data(data_json: str) -> tuple[str, list[dict[str, str]]]: + """Parse inline JSON data and write to a temporary CSV file. + + Args: + data_json: JSON string containing array of objects + + Returns: + Tuple of (temp_csv_path, inferred_source_columns) + + Raises: + click.BadParameter: If JSON is invalid or not an array of objects + """ + try: + data = json.loads(data_json) + except json.JSONDecodeError as e: + raise click.BadParameter(f"Invalid JSON data: {e}") from e + + if not isinstance(data, list): + raise click.BadParameter("Data must be a JSON array") + + if len(data) == 0: + raise click.BadParameter("Data array cannot be empty") + + if not isinstance(data[0], dict): + raise click.BadParameter("Data must be an array of objects") + + # Infer columns from the first row + columns = list(data[0].keys()) + if not columns: + raise click.BadParameter("Data objects must have at least one field") + + # Create source_columns with inferred descriptions + source_columns = [{"name": col, "description": f"The {col} field"} for col in columns] + + # Write to a temporary CSV file + temp_file = tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False, newline="") + writer = csv.DictWriter(temp_file, fieldnames=columns) + writer.writeheader() + for row in data: + writer.writerow(row) + temp_file.close() + + return temp_file.name, source_columns + + def suggest_from_intent( intent: str, source_columns: list[dict[str, str]] | None = None, @@ -256,8 +319,8 @@ def logout_cmd(): "--mode", type=click.Choice(["one-shot", "agentic"]), default="one-shot", help="Search mode", show_default=True ) @click.option("--max-results", type=int, default=10, help="Maximum results", show_default=True) -@click.option("--include-domains", multiple=True, help="Only search these domains") -@click.option("--exclude-domains", multiple=True, help="Exclude these domains") +@click.option("--include-domains", multiple=True, help="Only search these domains (comma-separated or repeated)") +@click.option("--exclude-domains", multiple=True, help="Exclude these domains (comma-separated or repeated)") @click.option("--after-date", help="Only results after this date (YYYY-MM-DD)") @click.option("-o", "--output", "output_file", type=click.Path(), help="Save results to file (JSON)") @click.option("--json", "output_json", is_flag=True, help="Output as JSON") @@ -291,9 +354,9 @@ def search( source_policy: dict[str, Any] = {} if include_domains: - source_policy["include_domains"] = list(include_domains) + source_policy["include_domains"] = parse_comma_separated(include_domains) if exclude_domains: - source_policy["exclude_domains"] = list(exclude_domains) + source_policy["exclude_domains"] = parse_comma_separated(exclude_domains) if after_date: source_policy["after_date"] = after_date if source_policy: @@ -450,6 +513,7 @@ def enrich(): @click.option("--enriched-columns", help="Enriched columns as JSON") @click.option("--intent", help="Natural language description (AI suggests columns)") @click.option("--processor", type=click.Choice(AVAILABLE_PROCESSORS), help="Processor to use") +@click.option("--data", "inline_data", help="Inline JSON data array (alternative to --source)") def enrich_run( config_file: str | None, source_type: str | None, @@ -459,23 +523,60 @@ def enrich_run( enriched_columns: str | None, intent: str | None, processor: str | None, + inline_data: str | None, ): - """Run data enrichment from YAML config or CLI arguments.""" - base_args = [source_type, source, target, source_columns] - has_cli_args = any(arg is not None for arg in base_args) or enriched_columns or intent + """Run data enrichment from YAML config or CLI arguments. - if config_file and has_cli_args: - console.print("[bold red]Error: Provide either a config file OR CLI arguments, not both.[/bold red]") - raise click.Abort() + You can provide data in three ways: - if not config_file and not has_cli_args: - console.print("[bold red]Error: Provide a config file or CLI arguments.[/bold red]") - raise click.Abort() + \b + 1. YAML config file: + parallel-cli enrich run config.yaml - if has_cli_args: - validate_enrich_args(source_type, source, target, source_columns, enriched_columns, intent) + \b + 2. CLI arguments with source file: + parallel-cli enrich run --source-type csv --source data.csv ... + + \b + 3. Inline JSON data (no CSV file needed): + parallel-cli enrich run --data '[{"company": "Google"}, {"company": "Apple"}]' \\ + --target output.csv --intent "Find the CEO" + """ + temp_csv_path: str | None = None try: + # Handle inline data - creates a temp CSV and infers source columns + if inline_data: + if source: + console.print("[bold red]Error: Use --data OR --source, not both.[/bold red]") + raise click.Abort() + if source_type and source_type != "csv": + console.print("[bold red]Error: --data only works with CSV output (--source-type csv).[/bold red]") + raise click.Abort() + + temp_csv_path, inferred_cols = parse_inline_data(inline_data) + source = temp_csv_path + source_type = "csv" + + # Use inferred columns if not explicitly provided + if not source_columns: + source_columns = json.dumps(inferred_cols) + console.print(f"[dim]Inferred {len(inferred_cols)} source column(s) from data[/dim]") + + base_args = [source_type, source, target, source_columns] + has_cli_args = any(arg is not None for arg in base_args) or enriched_columns or intent + + if config_file and has_cli_args: + console.print("[bold red]Error: Provide either a config file OR CLI arguments, not both.[/bold red]") + raise click.Abort() + + if not config_file and not has_cli_args: + console.print("[bold red]Error: Provide a config file or CLI arguments.[/bold red]") + raise click.Abort() + + if has_cli_args: + validate_enrich_args(source_type, source, target, source_columns, enriched_columns, intent) + if config_file: console.print(f"[bold cyan]Running enrichment from {config_file}...[/bold cyan]\n") run_enrichment(config_file) @@ -519,6 +620,10 @@ def enrich_run( except Exception as e: console.print(f"[bold red]Error during enrichment: {e}[/bold red]") raise + finally: + # Clean up temp file if we created one + if temp_csv_path and os.path.exists(temp_csv_path): + os.unlink(temp_csv_path) @enrich.command(name="plan") diff --git a/tests/test_cli.py b/tests/test_cli.py index d70098e..ceb00fb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,6 +11,8 @@ build_config_from_args, main, parse_columns, + parse_comma_separated, + parse_inline_data, suggest_from_intent, ) @@ -21,6 +23,45 @@ def runner(): return CliRunner() +class TestParseCommaSeparated: + """Tests for parse_comma_separated helper function.""" + + def test_single_value(self): + """Should handle single value.""" + result = parse_comma_separated(("example.com",)) + assert result == ["example.com"] + + def test_comma_separated(self): + """Should split comma-separated values.""" + result = parse_comma_separated(("google.com,github.com",)) + assert result == ["google.com", "github.com"] + + def test_repeated_flags(self): + """Should handle repeated flags.""" + result = parse_comma_separated(("google.com", "github.com")) + assert result == ["google.com", "github.com"] + + def test_mixed_usage(self): + """Should handle mix of comma-separated and repeated.""" + result = parse_comma_separated(("google.com,github.com", "twitter.com")) + assert result == ["google.com", "github.com", "twitter.com"] + + def test_whitespace_handling(self): + """Should trim whitespace around values.""" + result = parse_comma_separated(("google.com , github.com",)) + assert result == ["google.com", "github.com"] + + def test_empty_tuple(self): + """Should return empty list for empty tuple.""" + result = parse_comma_separated(()) + assert result == [] + + def test_skips_empty_strings(self): + """Should skip empty strings from trailing commas.""" + result = parse_comma_separated(("google.com,",)) + assert result == ["google.com"] + + class TestParseColumns: """Tests for parse_columns helper function.""" @@ -94,6 +135,78 @@ def test_build_config(self): assert len(config["enriched_columns"]) == 1 +class TestParseInlineData: + """Tests for parse_inline_data helper function.""" + + def test_parse_valid_data(self): + """Should parse valid JSON array and create temp CSV.""" + data = '[{"company": "Google", "industry": "Tech"}, {"company": "Apple", "industry": "Tech"}]' + csv_path, source_columns = parse_inline_data(data) + + try: + # Verify temp file was created + assert os.path.exists(csv_path) + assert csv_path.endswith(".csv") + + # Verify source columns were inferred + assert len(source_columns) == 2 + col_names = [c["name"] for c in source_columns] + assert "company" in col_names + assert "industry" in col_names + + # Verify CSV content + import csv as csv_module + + with open(csv_path) as f: + reader = csv_module.DictReader(f) + rows = list(reader) + assert len(rows) == 2 + assert rows[0]["company"] == "Google" + assert rows[1]["company"] == "Apple" + finally: + os.unlink(csv_path) + + def test_parse_single_item(self): + """Should work with a single item array.""" + data = '[{"name": "Test"}]' + csv_path, source_columns = parse_inline_data(data) + + try: + assert os.path.exists(csv_path) + assert len(source_columns) == 1 + assert source_columns[0]["name"] == "name" + finally: + os.unlink(csv_path) + + def test_parse_invalid_json(self): + """Should raise BadParameter for invalid JSON.""" + from click import BadParameter + + with pytest.raises(BadParameter, match="Invalid JSON"): + parse_inline_data("not valid json") + + def test_parse_not_array(self): + """Should raise BadParameter for non-array JSON.""" + from click import BadParameter + + with pytest.raises(BadParameter, match="must be a JSON array"): + parse_inline_data('{"name": "test"}') + + def test_parse_empty_array(self): + """Should raise BadParameter for empty array.""" + from click import BadParameter + + with pytest.raises(BadParameter, match="cannot be empty"): + parse_inline_data("[]") + + def test_parse_not_objects(self): + """Should raise BadParameter for array of non-objects.""" + from click import BadParameter + + with pytest.raises(BadParameter, match="array of objects"): + parse_inline_data('["a", "b", "c"]') + + class TestMainCLI: """Tests for the main CLI group.""" @@ -160,6 +273,14 @@ def test_search_help(self, runner): assert "Search the web" in result.output assert "--json" in result.output + def test_search_help_shows_comma_separated(self, runner): + """Should mention comma-separated in domain options help.""" + result = runner.invoke(main, ["search", "--help"]) + assert result.exit_code == 0 + assert "--include-domains" in result.output + assert "--exclude-domains" in result.output + assert "comma-separated" in result.output + def test_search_no_args(self, runner): """Should error without objective or query.""" result = runner.invoke(main, ["search"]) @@ -251,6 +372,71 @@ def test_enrich_run_both_enriched_and_intent(self, runner): assert result.exit_code != 0 assert "either" in result.output.lower() or "not both" in result.output.lower() + def test_enrich_run_help_shows_data_option(self, runner): + """Should show --data option in help.""" + result = runner.invoke(main, ["enrich", "run", "--help"]) + assert result.exit_code == 0 + assert "--data" in result.output + assert "Inline JSON data" in result.output + + def test_enrich_run_data_and_source_error(self, runner): + """Should error when both --data and --source provided.""" + result = runner.invoke( + main, + [ + "enrich", + "run", + "--data", + '[{"company": "Google"}]', + "--source", + "input.csv", + "--target", + "output.csv", + "--intent", + "Find CEO", + ], + ) + assert result.exit_code != 0 + assert "data" in result.output.lower() and "source" in result.output.lower() + + def test_enrich_run_data_with_non_csv_error(self, runner): + """Should error when --data used with non-csv source type.""" + result = runner.invoke( + main, + [ + "enrich", + "run", + "--data", + '[{"company": "Google"}]', + "--source-type", + "duckdb", + "--target", + "output.csv", + "--intent", + "Find CEO", + ], + ) + assert result.exit_code != 0 + assert "csv" in result.output.lower() + + def test_enrich_run_data_invalid_json(self, runner): + """Should error with invalid JSON data.""" + result = runner.invoke( + main, + [ + "enrich", + "run", + "--data", + "not valid json", + "--target", + "output.csv", + "--intent", + "Find CEO", + ], + ) + assert result.exit_code != 0 + assert "invalid" in result.output.lower() or "json" in result.output.lower() + class TestEnrichPlanCommand: """Tests for the enrich plan command."""