From 1a235a00f043ccb50f893d7a1c32200527b0f53d Mon Sep 17 00:00:00 2001 From: Matt Harris Date: Fri, 23 Jan 2026 17:24:32 -0500 Subject: [PATCH 01/14] fix: secret update hanging due to missing stdin input When updating an existing secret, gcloud secrets versions add with --data-file=- was called without providing stdin input, causing it to hang indefinitely. Now uses Popen with communicate() to pass the API key via stdin, same as the create case. --- .../integrations/bigquery/deploy.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/parallel_web_tools/integrations/bigquery/deploy.py b/parallel_web_tools/integrations/bigquery/deploy.py index 4704f85..a0ad536 100644 --- a/parallel_web_tools/integrations/bigquery/deploy.py +++ b/parallel_web_tools/integrations/bigquery/deploy.py @@ -150,10 +150,17 @@ def deploy_bigquery_integration( result = _run_command(["gcloud", "secrets", "describe", secret_name, "--project", project_id], check=False) if result.returncode == 0: - # Update existing secret - _run_command( - ["gcloud", "secrets", "versions", "add", secret_name, "--data-file=-", "--project", project_id], check=True + # Update existing secret - use Popen to provide api_key via stdin + process = subprocess.Popen( + ["gcloud", "secrets", "versions", "add", secret_name, "--data-file=-", "--project", project_id], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, ) + stdout, stderr = process.communicate(input=api_key) + if process.returncode != 0: + raise RuntimeError(f"Failed to update secret: {stderr}") else: # Create new secret process = subprocess.Popen( @@ -172,7 +179,9 @@ def deploy_bigquery_integration( stderr=subprocess.PIPE, text=True, ) - process.communicate(input=api_key) + stdout, stderr = process.communicate(input=api_key) + if process.returncode != 0: + raise RuntimeError(f"Failed to create secret: {stderr}") secret_resource = f"projects/{project_id}/secrets/{secret_name}/versions/latest" From 0192e1e399e704276b7969beba3b7a2eceb05852 Mon Sep 17 00:00:00 2001 From: Matt Harris Date: Fri, 23 Jan 2026 17:28:51 -0500 Subject: [PATCH 02/14] fix: use python312 runtime for cloud function (package requires >=3.12) --- parallel_web_tools/integrations/bigquery/deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parallel_web_tools/integrations/bigquery/deploy.py b/parallel_web_tools/integrations/bigquery/deploy.py index a0ad536..bfb263b 100644 --- a/parallel_web_tools/integrations/bigquery/deploy.py +++ b/parallel_web_tools/integrations/bigquery/deploy.py @@ -196,7 +196,7 @@ def deploy_bigquery_integration( "deploy", function_name, "--gen2", - "--runtime=python311", + "--runtime=python312", "--region", region, "--source", From c077cf09547d86ea643f8dfc9e9aedf9be0dcdd1 Mon Sep 17 00:00:00 2001 From: Matt Harris Date: Fri, 23 Jan 2026 17:44:05 -0500 Subject: [PATCH 03/14] fix: include stdout in error messages (bq outputs errors to stdout) --- parallel_web_tools/integrations/bigquery/deploy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/parallel_web_tools/integrations/bigquery/deploy.py b/parallel_web_tools/integrations/bigquery/deploy.py index bfb263b..0021fd3 100644 --- a/parallel_web_tools/integrations/bigquery/deploy.py +++ b/parallel_web_tools/integrations/bigquery/deploy.py @@ -19,7 +19,9 @@ def _run_command(cmd: list[str], check: bool = True) -> subprocess.CompletedProc """Run a shell command and return the result.""" result = subprocess.run(cmd, capture_output=True, text=True) if check and result.returncode != 0: - raise RuntimeError(f"Command failed: {' '.join(cmd)}\n{result.stderr}") + # Include both stdout and stderr as some tools output errors to stdout + error_output = result.stderr or result.stdout or "(no output)" + raise RuntimeError(f"Command failed: {' '.join(cmd)}\n{error_output}") return result From a448160b83eb886a70e9691c308db3a56bd222fc Mon Sep 17 00:00:00 2001 From: Matt Harris Date: Fri, 23 Jan 2026 17:44:59 -0500 Subject: [PATCH 04/14] fix: move --format=json flag before positional argument in bq command --- parallel_web_tools/integrations/bigquery/deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parallel_web_tools/integrations/bigquery/deploy.py b/parallel_web_tools/integrations/bigquery/deploy.py index 0021fd3..d9c707a 100644 --- a/parallel_web_tools/integrations/bigquery/deploy.py +++ b/parallel_web_tools/integrations/bigquery/deploy.py @@ -285,8 +285,8 @@ def deploy_bigquery_integration( "bq", "show", "--connection", - f"{project_id}.{region}.{connection_id}", "--format=json", + f"{project_id}.{region}.{connection_id}", ] ) connection_info = json.loads(result.stdout) From 44d7708adbf40ec464c7961228b9c327a58f516e Mon Sep 17 00:00:00 2001 From: Matt Harris Date: Fri, 23 Jan 2026 18:01:47 -0500 Subject: [PATCH 05/14] fix: add project_id flag to bq query and add progress output --- parallel_web_tools/integrations/bigquery/deploy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parallel_web_tools/integrations/bigquery/deploy.py b/parallel_web_tools/integrations/bigquery/deploy.py index d9c707a..1b49a17 100644 --- a/parallel_web_tools/integrations/bigquery/deploy.py +++ b/parallel_web_tools/integrations/bigquery/deploy.py @@ -347,7 +347,8 @@ def deploy_bigquery_integration( for statement in sql.split(";"): statement = statement.strip() if statement.startswith("CREATE"): - _run_command(["bq", "query", "--use_legacy_sql=false", statement + ";"]) + print(" Creating function...") + _run_command(["bq", "query", "--use_legacy_sql=false", f"--project_id={project_id}", statement + ";"]) print("\nDeployment complete!") From 76b0d6a2e842f2e2b24922c46243d169f4b06969 Mon Sep 17 00:00:00 2001 From: Matt Harris Date: Fri, 23 Jan 2026 18:08:58 -0500 Subject: [PATCH 06/14] fix: strip SQL comments before detecting CREATE statements --- .../integrations/bigquery/deploy.py | 23 +++++++++++++++---- pyproject.toml | 9 ++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/parallel_web_tools/integrations/bigquery/deploy.py b/parallel_web_tools/integrations/bigquery/deploy.py index 1b49a17..e467c94 100644 --- a/parallel_web_tools/integrations/bigquery/deploy.py +++ b/parallel_web_tools/integrations/bigquery/deploy.py @@ -344,11 +344,24 @@ def deploy_bigquery_integration( ) # Execute each CREATE FUNCTION statement - for statement in sql.split(";"): - statement = statement.strip() - if statement.startswith("CREATE"): - print(" Creating function...") - _run_command(["bq", "query", "--use_legacy_sql=false", f"--project_id={project_id}", statement + ";"]) + # Strip SQL comments (lines starting with --) before checking for CREATE + def strip_sql_comments(sql_text: str) -> str: + lines = [line for line in sql_text.split("\n") if not line.strip().startswith("--")] + return "\n".join(lines).strip() + + statements = [] + for chunk in sql.split(";"): + clean = strip_sql_comments(chunk) + if clean.startswith("CREATE"): + statements.append(clean) + + if not statements: + print(" Warning: No CREATE statements found in SQL template") + for statement in statements: + # Extract function name for logging + func_name = statement.split("`")[1] if "`" in statement else "unknown" + print(f" Creating {func_name}...") + _run_command(["bq", "query", "--use_legacy_sql=false", f"--project_id={project_id}", statement + ";"]) print("\nDeployment complete!") diff --git a/pyproject.toml b/pyproject.toml index f48848d..0ffc02a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,6 +88,15 @@ dev = [ [tool.hatch.build.targets.wheel] packages = ["parallel_web_tools"] +artifacts = [ + "*.sql", + "*.txt", # requirements.txt for cloud function +] + +[tool.hatch.build.targets.sdist] +include = [ + "/parallel_web_tools", +] [tool.pyrefly] project-includes = [ From f9305bfe64c651d985588987a2c6a5da94680d82 Mon Sep 17 00:00:00 2001 From: Matt Harris Date: Fri, 23 Jan 2026 18:09:40 -0500 Subject: [PATCH 07/14] fix: check for gcloud and bq CLI tools before deployment --- .../integrations/bigquery/deploy.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/parallel_web_tools/integrations/bigquery/deploy.py b/parallel_web_tools/integrations/bigquery/deploy.py index e467c94..205883b 100644 --- a/parallel_web_tools/integrations/bigquery/deploy.py +++ b/parallel_web_tools/integrations/bigquery/deploy.py @@ -114,6 +114,21 @@ def deploy_bigquery_integration( Raises: RuntimeError: If any deployment step fails or user declines confirmation. """ + # Check for required CLI tools + import shutil + + missing_tools = [] + if not shutil.which("gcloud"): + missing_tools.append("gcloud") + if not shutil.which("bq"): + missing_tools.append("bq") + + if missing_tools: + raise RuntimeError( + f"Required CLI tools not found: {', '.join(missing_tools)}\n" + "Please install the Google Cloud SDK: https://cloud.google.com/sdk/docs/install" + ) + print(f"Checking for existing resources in {project_id}...") # Check for existing resources From ad6e312f7bbcda783609034158961e5b16b11a21 Mon Sep 17 00:00:00 2001 From: Matt Harris Date: Fri, 23 Jan 2026 18:11:49 -0500 Subject: [PATCH 08/14] fix: remove parallel_enrich_company convenience function, keep only generic parallel_enrich --- .../bigquery/sql/create_functions.sql | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/parallel_web_tools/integrations/bigquery/sql/create_functions.sql b/parallel_web_tools/integrations/bigquery/sql/create_functions.sql index c7195af..d97d570 100644 --- a/parallel_web_tools/integrations/bigquery/sql/create_functions.sql +++ b/parallel_web_tools/integrations/bigquery/sql/create_functions.sql @@ -20,30 +20,3 @@ OPTIONS ( endpoint = '{function_url}', user_defined_context = [("processor", "lite-fast")] ); - --- Convenience function for company enrichment -CREATE OR REPLACE FUNCTION `{project_id}.{dataset_id}.parallel_enrich_company`( - company_name STRING, - company_website STRING, - fields STRING -) -RETURNS STRING -AS ( - `{project_id}.{dataset_id}.parallel_enrich`( - JSON_OBJECT('company_name', company_name, 'website', company_website), - fields - ) -); - --- Example queries: --- --- SELECT parallel_enrich( --- JSON_OBJECT('company_name', 'Google', 'website', 'google.com'), --- JSON_ARRAY('CEO name', 'Founding year', 'Brief description') --- ); --- --- SELECT parallel_enrich_company( --- 'Apple', --- 'apple.com', --- JSON_ARRAY('CEO name', 'Market cap', 'Industry') --- ); From 121e53e95b008372385b7de5fe1e0f86e0a8d6ae Mon Sep 17 00:00:00 2001 From: Matt Harris Date: Fri, 23 Jan 2026 18:12:38 -0500 Subject: [PATCH 09/14] fix: wrap JSON_OBJECT/JSON_ARRAY with TO_JSON_STRING in example query --- parallel_web_tools/integrations/bigquery/deploy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parallel_web_tools/integrations/bigquery/deploy.py b/parallel_web_tools/integrations/bigquery/deploy.py index 205883b..c0c78a4 100644 --- a/parallel_web_tools/integrations/bigquery/deploy.py +++ b/parallel_web_tools/integrations/bigquery/deploy.py @@ -387,8 +387,8 @@ def strip_sql_comments(sql_text: str) -> str: "function_url": function_url, "example_query": f""" SELECT `{project_id}.{dataset_id}.parallel_enrich`( - JSON_OBJECT('company_name', 'Google', 'website', 'google.com'), - JSON_ARRAY('CEO name', 'Founding year', 'Brief description') + TO_JSON_STRING(JSON_OBJECT('company_name', 'Google', 'website', 'google.com')), + TO_JSON_STRING(JSON_ARRAY('CEO name', 'Founding year', 'Brief description')) ) as enriched_data; """.strip(), } From 0f9e65f1365fcfc70cf4a1561dc14fae03c8af09 Mon Sep 17 00:00:00 2001 From: Matt Harris Date: Fri, 23 Jan 2026 18:13:02 -0500 Subject: [PATCH 10/14] fix: use JSON type for function parameters and return value (cleaner API) --- parallel_web_tools/integrations/bigquery/deploy.py | 4 ++-- .../integrations/bigquery/sql/create_functions.sql | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/parallel_web_tools/integrations/bigquery/deploy.py b/parallel_web_tools/integrations/bigquery/deploy.py index c0c78a4..205883b 100644 --- a/parallel_web_tools/integrations/bigquery/deploy.py +++ b/parallel_web_tools/integrations/bigquery/deploy.py @@ -387,8 +387,8 @@ def strip_sql_comments(sql_text: str) -> str: "function_url": function_url, "example_query": f""" SELECT `{project_id}.{dataset_id}.parallel_enrich`( - TO_JSON_STRING(JSON_OBJECT('company_name', 'Google', 'website', 'google.com')), - TO_JSON_STRING(JSON_ARRAY('CEO name', 'Founding year', 'Brief description')) + JSON_OBJECT('company_name', 'Google', 'website', 'google.com'), + JSON_ARRAY('CEO name', 'Founding year', 'Brief description') ) as enriched_data; """.strip(), } diff --git a/parallel_web_tools/integrations/bigquery/sql/create_functions.sql b/parallel_web_tools/integrations/bigquery/sql/create_functions.sql index d97d570..7d1b016 100644 --- a/parallel_web_tools/integrations/bigquery/sql/create_functions.sql +++ b/parallel_web_tools/integrations/bigquery/sql/create_functions.sql @@ -10,11 +10,12 @@ -- {function_url} - Deployed Cloud Function URL -- Main enrichment function +-- Accepts JSON input and returns JSON output CREATE OR REPLACE FUNCTION `{project_id}.{dataset_id}.parallel_enrich`( - input_data STRING, - output_columns STRING + input_data JSON, + output_columns JSON ) -RETURNS STRING +RETURNS JSON REMOTE WITH CONNECTION `{project_id}.{location}.{connection_id}` OPTIONS ( endpoint = '{function_url}', From 44acdc4e1a9806b5a79b77f3eb4c47a55728d2d7 Mon Sep 17 00:00:00 2001 From: Matt Harris Date: Fri, 23 Jan 2026 18:16:25 -0500 Subject: [PATCH 11/14] docs: add example showing how to parse JSON result with JSON_VALUE --- parallel_web_tools/integrations/bigquery/deploy.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/parallel_web_tools/integrations/bigquery/deploy.py b/parallel_web_tools/integrations/bigquery/deploy.py index 205883b..5f0f188 100644 --- a/parallel_web_tools/integrations/bigquery/deploy.py +++ b/parallel_web_tools/integrations/bigquery/deploy.py @@ -386,10 +386,23 @@ def strip_sql_comments(sql_text: str) -> str: "dataset_id": dataset_id, "function_url": function_url, "example_query": f""" +-- Basic usage SELECT `{project_id}.{dataset_id}.parallel_enrich`( JSON_OBJECT('company_name', 'Google', 'website', 'google.com'), JSON_ARRAY('CEO name', 'Founding year', 'Brief description') ) as enriched_data; + +-- Parsing the JSON result +SELECT + JSON_VALUE(enriched_data, '$.ceo_name') as ceo_name, + JSON_VALUE(enriched_data, '$.founding_year') as founding_year, + JSON_VALUE(enriched_data, '$.brief_description') as description +FROM ( + SELECT `{project_id}.{dataset_id}.parallel_enrich`( + JSON_OBJECT('company_name', 'Google', 'website', 'google.com'), + JSON_ARRAY('CEO name', 'Founding year', 'Brief description') + ) as enriched_data +); """.strip(), } From 6baa4cad26beef7eb0133d14a8d1e11306ce71eb Mon Sep 17 00:00:00 2001 From: Matt Harris Date: Fri, 23 Jan 2026 18:17:55 -0500 Subject: [PATCH 12/14] docs: add basis/citations to example query --- parallel_web_tools/integrations/bigquery/deploy.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/parallel_web_tools/integrations/bigquery/deploy.py b/parallel_web_tools/integrations/bigquery/deploy.py index 5f0f188..099ed71 100644 --- a/parallel_web_tools/integrations/bigquery/deploy.py +++ b/parallel_web_tools/integrations/bigquery/deploy.py @@ -386,17 +386,18 @@ def strip_sql_comments(sql_text: str) -> str: "dataset_id": dataset_id, "function_url": function_url, "example_query": f""" --- Basic usage +-- Basic usage (returns JSON with enriched fields and basis/citations) SELECT `{project_id}.{dataset_id}.parallel_enrich`( JSON_OBJECT('company_name', 'Google', 'website', 'google.com'), JSON_ARRAY('CEO name', 'Founding year', 'Brief description') ) as enriched_data; --- Parsing the JSON result +-- Parsing the JSON result into columns SELECT JSON_VALUE(enriched_data, '$.ceo_name') as ceo_name, JSON_VALUE(enriched_data, '$.founding_year') as founding_year, - JSON_VALUE(enriched_data, '$.brief_description') as description + JSON_VALUE(enriched_data, '$.brief_description') as description, + JSON_QUERY(enriched_data, '$.basis') as citations FROM ( SELECT `{project_id}.{dataset_id}.parallel_enrich`( JSON_OBJECT('company_name', 'Google', 'website', 'google.com'), From c6d3666a40a989455530148f93b753c971b1a872 Mon Sep 17 00:00:00 2001 From: Matt Harris Date: Fri, 23 Jan 2026 18:44:49 -0500 Subject: [PATCH 13/14] use json --- .../integrations/bigquery/cloud_function/main.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/parallel_web_tools/integrations/bigquery/cloud_function/main.py b/parallel_web_tools/integrations/bigquery/cloud_function/main.py index e5d185c..1ae2e09 100644 --- a/parallel_web_tools/integrations/bigquery/cloud_function/main.py +++ b/parallel_web_tools/integrations/bigquery/cloud_function/main.py @@ -175,22 +175,22 @@ def parallel_enrich(request: Request): logger.info(f"Using batch processing for {len(inputs)} inputs") batch_results = _process_batch(inputs, common_output_cols, processor) - replies = [""] * len(calls) + replies: list[dict[str, Any] | None] = [None] * len(calls) for error_call in error_calls: - replies[error_call["index"]] = json.dumps({"error": error_call["error"]}) + replies[error_call["index"]] = {"error": error_call["error"]} for j, valid_call in enumerate(valid_calls): - replies[valid_call["index"]] = json.dumps(batch_results[j]) + replies[valid_call["index"]] = batch_results[j] return jsonify({"replies": replies}) # Fallback: process individually - replies = [""] * len(calls) + replies: list[dict[str, Any] | None] = [None] * len(calls) for error_call in error_calls: - replies[error_call["index"]] = json.dumps({"error": error_call["error"]}) + replies[error_call["index"]] = {"error": error_call["error"]} for valid_call in valid_calls: result = _process_batch([valid_call["input_data"]], valid_call["output_columns"], processor) - replies[valid_call["index"]] = json.dumps(result[0] if result else {"error": "No result"}) + replies[valid_call["index"]] = result[0] if result else {"error": "No result"} return jsonify({"replies": replies}) From 9785c88b765de02bb7157edc6224a8ff471995c5 Mon Sep 17 00:00:00 2001 From: Matt Harris Date: Fri, 23 Jan 2026 18:45:28 -0500 Subject: [PATCH 14/14] bump to 0.0.4 --- parallel_web_tools/__init__.py | 2 +- .../integrations/bigquery/cloud_function/requirements.txt | 2 +- pyproject.toml | 2 +- tests/test_cli.py | 2 +- uv.lock | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/parallel_web_tools/__init__.py b/parallel_web_tools/__init__.py index eb41914..ea07545 100644 --- a/parallel_web_tools/__init__.py +++ b/parallel_web_tools/__init__.py @@ -27,7 +27,7 @@ run_tasks, ) -__version__ = "0.0.3" +__version__ = "0.0.4" __all__ = [ # Auth diff --git a/parallel_web_tools/integrations/bigquery/cloud_function/requirements.txt b/parallel_web_tools/integrations/bigquery/cloud_function/requirements.txt index c82b529..fe3b239 100644 --- a/parallel_web_tools/integrations/bigquery/cloud_function/requirements.txt +++ b/parallel_web_tools/integrations/bigquery/cloud_function/requirements.txt @@ -1,5 +1,5 @@ # Cloud Function dependencies for BigQuery Remote Function functions-framework>=3.0.0 flask>=3.0.0 -parallel-web-tools>=0.0.3 +parallel-web-tools>=0.0.4 google-cloud-secret-manager>=2.20.0 diff --git a/pyproject.toml b/pyproject.toml index 0ffc02a..4441f8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "parallel-web-tools" -version = "0.0.3" +version = "0.0.4" description = "Parallel Tools: CLI and data enrichment utilities for the Parallel API" readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_cli.py b/tests/test_cli.py index 7ceaf9d..d70098e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -112,7 +112,7 @@ def test_version(self, runner): """Should show version.""" result = runner.invoke(main, ["--version"]) assert result.exit_code == 0 - assert "0.0.3" in result.output + assert "0.0.4" in result.output class TestAuthCommand: diff --git a/uv.lock b/uv.lock index f901fff..29ab4ef 100644 --- a/uv.lock +++ b/uv.lock @@ -1057,7 +1057,7 @@ wheels = [ [[package]] name = "parallel-web-tools" -version = "0.0.3" +version = "0.0.4" source = { editable = "." } dependencies = [ { name = "pandas" },