From 40a0e9fff18b04ee468657ec4514575fd0e062f1 Mon Sep 17 00:00:00 2001 From: Quentin Le Helloco Date: Thu, 4 Dec 2025 16:32:43 +0100 Subject: [PATCH] Fix pyspark python diff version between driver/worker --- cluster_pack/spark/spark_config_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster_pack/spark/spark_config_builder.py b/cluster_pack/spark/spark_config_builder.py index 8e3c370..b274d72 100644 --- a/cluster_pack/spark/spark_config_builder.py +++ b/cluster_pack/spark/spark_config_builder.py @@ -24,7 +24,7 @@ def add_packaged_environment(ssb: SparkSession.Builder, archive: str) -> None: if _get_value(ssb, "spark.submit.deployMode") == "cluster": os.environ["PYSPARK_DRIVER_PYTHON"] = usage.interpreter_cmd else: - os.environ["PYSPARK_DRIVER_PYTHON"] = "python" + os.environ["PYSPARK_DRIVER_PYTHON"] = usage.interpreter_cmd if not _get_value(ssb, "spark.master") == "yarn": _logger.info("Not running on yarn. Adding archive to spark.files")