From d18e9823e5058268beafecfe78d755a794831da5 Mon Sep 17 00:00:00 2001 From: Valentin Heiserer Date: Thu, 11 Dec 2025 20:51:00 +0100 Subject: [PATCH] add Spark configuration setup --- Aufgabe 11/sparkstart.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 Aufgabe 11/sparkstart.py diff --git a/Aufgabe 11/sparkstart.py b/Aufgabe 11/sparkstart.py new file mode 100644 index 0000000..bdb7010 --- /dev/null +++ b/Aufgabe 11/sparkstart.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- + +""" +Erzeugen einer Spark-Konfiguration +""" + +from pyspark import SparkConf, SparkContext +from pyspark.sql import SparkSession + +# connect to cluster +conf = SparkConf().setMaster("spark://193.174.205.250:7077").setAppName("HeisererValentin") +conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") +conf.set("spark.executor.memory", '32g') +conf.set("spark.driver.memory", '8g') +conf.set("spark.cores.max", "40") +scon = SparkContext(conf=conf) + + +spark = SparkSession \ + .builder \ + .appName("Python Spark SQL") \ + .getOrCreate()