mirror of
https://github.com/Vale54321/BigData.git
synced 2025-12-15 19:29:34 +01:00
add Spark configuration setup
This commit is contained in:
22
Aufgabe 11/sparkstart.py
Normal file
22
Aufgabe 11/sparkstart.py
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
"""
|
||||||
|
Erzeugen einer Spark-Konfiguration
|
||||||
|
"""
|
||||||
|
|
||||||
|
from pyspark import SparkConf, SparkContext
|
||||||
|
from pyspark.sql import SparkSession
|
||||||
|
|
||||||
|
# connect to cluster
|
||||||
|
conf = SparkConf().setMaster("spark://193.174.205.250:7077").setAppName("HeisererValentin")
|
||||||
|
conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
|
||||||
|
conf.set("spark.executor.memory", '32g')
|
||||||
|
conf.set("spark.driver.memory", '8g')
|
||||||
|
conf.set("spark.cores.max", "40")
|
||||||
|
scon = SparkContext(conf=conf)
|
||||||
|
|
||||||
|
|
||||||
|
spark = SparkSession \
|
||||||
|
.builder \
|
||||||
|
.appName("Python Spark SQL") \
|
||||||
|
.getOrCreate()
|
||||||
Reference in New Issue
Block a user