mirror of
https://github.com/Vale54321/BigData.git
synced 2025-12-15 11:29:32 +01:00
add 9 + 10
This commit is contained in:
209
Aufgabe 10/Aufgabe10.py
Normal file
209
Aufgabe 10/Aufgabe10.py
Normal file
@@ -0,0 +1,209 @@
|
||||
from sparkstart import scon, spark
|
||||
import ghcnd_stations
|
||||
import matplotlib.pyplot as plt
|
||||
import time
|
||||
|
||||
# a) Scatterplot: alle Stationen (lon/lat)
|
||||
def plot_all_stations(spark):
|
||||
q = """
|
||||
SELECT stationname, latitude, longitude
|
||||
FROM cdc_stations
|
||||
WHERE latitude IS NOT NULL AND longitude IS NOT NULL
|
||||
"""
|
||||
t0 = time.time()
|
||||
rows = spark.sql(q).collect()
|
||||
t1 = time.time()
|
||||
print(f"Ausfuehrungszeit (SQL): {t1 - t0:.3f}s -- Rows: {len(rows)}")
|
||||
|
||||
lats = [r['latitude'] for r in rows]
|
||||
lons = [r['longitude'] for r in rows]
|
||||
names = [r['stationname'] for r in rows]
|
||||
|
||||
plt.figure(figsize=(8,6))
|
||||
plt.scatter(lons, lats, s=10, alpha=0.6)
|
||||
plt.xlabel('Longitude')
|
||||
plt.ylabel('Latitude')
|
||||
plt.title('Alle CDC-Stationen (Scatter)')
|
||||
plt.grid(True)
|
||||
plt.show()
|
||||
|
||||
|
||||
# b) Scatterplot: Stationsdauer in Jahren als Marker-Size
|
||||
def plot_station_duration(spark, size_factor=20):
|
||||
q = """
|
||||
SELECT
|
||||
stationname,
|
||||
latitude,
|
||||
longitude,
|
||||
(CAST(CASE WHEN length(to_date) >= 4 THEN substr(to_date,1,4) ELSE year(current_date()) END AS INT)
|
||||
- CAST(substr(from_date,1,4) AS INT)) AS years
|
||||
FROM cdc_stations
|
||||
WHERE latitude IS NOT NULL AND longitude IS NOT NULL
|
||||
"""
|
||||
t0 = time.time()
|
||||
rows = spark.sql(q).collect()
|
||||
t1 = time.time()
|
||||
print(f"Ausfuehrungszeit (SQL): {t1 - t0:.3f}s -- Rows: {len(rows)}")
|
||||
|
||||
lats = [r['latitude'] for r in rows]
|
||||
lons = [r['longitude'] for r in rows]
|
||||
years = [r['years'] if r['years'] is not None else 0 for r in rows]
|
||||
sizes = [max(5, (y+1) * size_factor) for y in years]
|
||||
|
||||
plt.figure(figsize=(8,6))
|
||||
plt.scatter(lons, lats, s=sizes, alpha=0.6)
|
||||
plt.xlabel('Longitude')
|
||||
plt.ylabel('Latitude')
|
||||
plt.title('CDC-Stationen: Dauer der Verfuegbarkeit (Größe ~ Jahre)')
|
||||
plt.grid(True)
|
||||
plt.show()
|
||||
|
||||
def plot_frost_distribution_year(spark, year):
|
||||
q = f"""
|
||||
WITH daily_max AS (
|
||||
SELECT stationid, date, MAX(tt_tu) AS max_temp
|
||||
FROM cdc_hourly
|
||||
WHERE length(date) >= 8 AND substr(date,1,4) = '{year}'
|
||||
GROUP BY stationid, date
|
||||
),
|
||||
station_frost AS (
|
||||
SELECT dm.stationid, SUM(CASE WHEN dm.max_temp < 0 THEN 1 ELSE 0 END) AS frostdays
|
||||
FROM daily_max dm
|
||||
GROUP BY dm.stationid
|
||||
)
|
||||
SELECT sf.frostdays, COUNT(*) AS stations
|
||||
FROM station_frost sf
|
||||
GROUP BY sf.frostdays
|
||||
ORDER BY sf.frostdays
|
||||
"""
|
||||
t0 = time.time()
|
||||
rows = spark.sql(q).collect()
|
||||
t1 = time.time()
|
||||
print(f"Ausfuehrungszeit (SQL): {t1 - t0:.3f}s -- Distinct frostdays: {len(rows)}")
|
||||
|
||||
x = [r['frostdays'] for r in rows]
|
||||
y = [r['stations'] for r in rows]
|
||||
|
||||
plt.figure(figsize=(8,5))
|
||||
plt.bar(x, y)
|
||||
plt.xlabel('Anzahl Frosttage im Jahr ' + str(year))
|
||||
plt.ylabel('Anzahl Stationen')
|
||||
plt.title(f'Verteilung der Frosttage pro Station im Jahr {year}')
|
||||
plt.grid(True)
|
||||
plt.show()
|
||||
|
||||
|
||||
# c2) Frosttage Zeitreihe für eine Station mit 5- und 20-Jahres Durchschnitt (SQL window)
|
||||
def plot_station_frost_timeseries(spark, station_name):
|
||||
q = f"""
|
||||
WITH daily_max AS (
|
||||
SELECT stationid, date, MAX(tt_tu) AS max_temp
|
||||
FROM cdc_hourly
|
||||
GROUP BY stationid, date
|
||||
),
|
||||
yearly AS (
|
||||
SELECT
|
||||
dm.stationid,
|
||||
CAST(substr(dm.date,1,4) AS INT) AS year,
|
||||
SUM(CASE WHEN dm.max_temp < 0 THEN 1 ELSE 0 END) AS frostdays
|
||||
FROM daily_max dm
|
||||
GROUP BY dm.stationid, CAST(substr(dm.date,1,4) AS INT)
|
||||
),
|
||||
station_yearly AS (
|
||||
SELECT
|
||||
y.year,
|
||||
y.frostdays,
|
||||
AVG(y.frostdays) OVER (ORDER BY y.year ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) AS avg5,
|
||||
AVG(y.frostdays) OVER (ORDER BY y.year ROWS BETWEEN 19 PRECEDING AND CURRENT ROW) AS avg20
|
||||
FROM yearly y
|
||||
JOIN cdc_stations s ON y.stationid = s.stationid
|
||||
WHERE trim(upper(s.stationname)) = '{station_name.upper()}'
|
||||
ORDER BY y.year
|
||||
)
|
||||
SELECT * FROM station_yearly
|
||||
"""
|
||||
t0 = time.time()
|
||||
rows = spark.sql(q).collect()
|
||||
t1 = time.time()
|
||||
print(f"Ausfuehrungszeit (SQL): {t1 - t0:.3f}s -- Years: {len(rows)}")
|
||||
|
||||
if not rows:
|
||||
print(f"Keine Daten f\u00fcr Station '{station_name}'.")
|
||||
return
|
||||
|
||||
years = [r['year'] for r in rows]
|
||||
frostdays = [r['frostdays'] for r in rows]
|
||||
avg5 = [r['avg5'] for r in rows]
|
||||
avg20 = [r['avg20'] for r in rows]
|
||||
|
||||
plt.figure(figsize=(10,5))
|
||||
plt.plot(years, frostdays, label='Frosttage (Jahr)')
|
||||
plt.plot(years, avg5, label='5-Jahres-Durchschnitt')
|
||||
plt.plot(years, avg20, label='20-Jahres-Durchschnitt')
|
||||
plt.xlabel('Jahr')
|
||||
plt.ylabel('Anzahl Frosttage')
|
||||
plt.title(f'Frosttage f\u00fcr Station {station_name}')
|
||||
plt.legend()
|
||||
plt.grid(True)
|
||||
plt.show()
|
||||
|
||||
|
||||
# d) Korrelation Hoehe vs. Frosttage pro Jahr
|
||||
def plot_height_frost_correlation(spark):
|
||||
q = """
|
||||
WITH daily_max AS (
|
||||
SELECT stationid, date, MAX(tt_tu) AS max_temp
|
||||
FROM cdc_hourly
|
||||
GROUP BY stationid, date
|
||||
),
|
||||
yearly AS (
|
||||
SELECT
|
||||
dm.stationid,
|
||||
CAST(substr(dm.date,1,4) AS INT) AS year,
|
||||
SUM(CASE WHEN dm.max_temp < 0 THEN 1 ELSE 0 END) AS frostdays
|
||||
FROM daily_max dm
|
||||
GROUP BY dm.stationid, CAST(substr(dm.date,1,4) AS INT)
|
||||
),
|
||||
joined AS (
|
||||
SELECT y.year, s.height, y.frostdays
|
||||
FROM yearly y
|
||||
JOIN cdc_stations s ON y.stationid = s.stationid
|
||||
),
|
||||
yearly_corr AS (
|
||||
SELECT year, corr(height, frostdays) AS corr
|
||||
FROM joined
|
||||
GROUP BY year
|
||||
ORDER BY year
|
||||
)
|
||||
SELECT year, corr FROM yearly_corr WHERE corr IS NOT NULL
|
||||
"""
|
||||
t0 = time.time()
|
||||
rows = spark.sql(q).collect()
|
||||
t1 = time.time()
|
||||
print(f"Ausfuehrungszeit (SQL): {t1 - t0:.3f}s -- Years with corr: {len(rows)}")
|
||||
|
||||
if not rows:
|
||||
print("Keine Korrelationsdaten verfügbar.")
|
||||
return
|
||||
|
||||
years = [r['year'] for r in rows]
|
||||
corr = [r['corr'] for r in rows]
|
||||
|
||||
plt.figure(figsize=(10,5))
|
||||
plt.bar(years, corr)
|
||||
plt.xlabel('Jahr')
|
||||
plt.ylabel('Korrelationskoeffizient (height vs frostdays)')
|
||||
plt.title('Korrelation Hoehe vs. Frosttage pro Jahr')
|
||||
plt.grid(True)
|
||||
plt.show()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
ghcnd_stations.read_ghcnd_from_parquet(spark)
|
||||
|
||||
plot_all_stations(spark)
|
||||
plot_station_duration(spark)
|
||||
plot_frost_distribution_year(spark, '2010')
|
||||
plot_station_frost_timeseries(spark, 'KEMPTEN')
|
||||
plot_height_frost_correlation(spark)
|
||||
pass
|
||||
Reference in New Issue
Block a user