mirror of
https://github.com/Vale54321/BigData.git
synced 2025-12-11 09:59:33 +01:00
22 lines
576 B
Python
22 lines
576 B
Python
# -*- coding: utf-8 -*-
|
|
|
|
"""
|
|
Erzeugen einer Spark-Konfiguration
|
|
"""
|
|
|
|
from pyspark import SparkConf, SparkContext
|
|
from pyspark.sql import SparkSession
|
|
|
|
# connect to cluster
|
|
conf = SparkConf().setMaster("spark://193.174.205.250:7077").setAppName("HeisererValentin")
|
|
conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
|
|
conf.set("spark.executor.memory", '32g')
|
|
conf.set("spark.driver.memory", '8g')
|
|
conf.set("spark.cores.max", "40")
|
|
scon = SparkContext(conf=conf)
|
|
|
|
spark = SparkSession \
|
|
.builder \
|
|
.appName("Python Spark SQL") \
|
|
.getOrCreate()
|