diff --git a/python/pyspark/errors/error-conditions.json b/python/pyspark/errors/error-conditions.json index 2a3bcd7240e78..33730f757dbd9 100644 --- a/python/pyspark/errors/error-conditions.json +++ b/python/pyspark/errors/error-conditions.json @@ -507,6 +507,11 @@ "Variant binary is malformed. Please check the data source is valid." ] }, + "MASTER_URL_INVALID": { + "message": [ + "Master must either be yarn or start with spark, k8s, or local." + ] + }, "MASTER_URL_NOT_SET": { "message": [ "A master URL must be set in your configuration." diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py index 11bd1ded42b96..241424c954b2d 100644 --- a/python/pyspark/sql/session.py +++ b/python/pyspark/sql/session.py @@ -508,6 +508,11 @@ def getOrCreate(self) -> "SparkSession": if url is None and is_api_mode_connect: url = opts.get("spark.master", os.environ.get("MASTER", "local")) + if url.startswith("sc://"): + raise PySparkRuntimeError( + errorClass="MASTER_URL_INVALID", + messageParameters={}, + ) if url is None: raise PySparkRuntimeError(