Merge pull request #135 from technologiestiftung/staging

feat: add district to trees layer, fix: weather data old entries

harvester/src/mapbox_tree_update.py

@@ -63,15 +63,16 @@ def generate_trees_csv(temp_dir, db_conn):
                     ST_X(geom) AS lng,
                     trees.radolan_sum,
                     trees.pflanzjahr,
-                    COALESCE(SUM(w.amount), 0) AS watering_sum
+                    COALESCE(SUM(w.amount), 0) AS watering_sum,
+                    trees.bezirk as district
                 FROM
                     trees
                 LEFT JOIN
                     trees_watered w ON w.tree_id = trees.id AND w.timestamp >= CURRENT_DATE - INTERVAL '30 days' AND DATE_TRUNC('day', w.timestamp) < CURRENT_DATE
                 WHERE
                     ST_CONTAINS(ST_SetSRID ((SELECT ST_EXTENT (geometry) FROM radolan_geometry), 4326), trees.geom)
                 GROUP BY
-                    trees.id, trees.lat, trees.lng, trees.radolan_sum, trees.pflanzjahr;
+                    trees.id, trees.lat, trees.lng, trees.radolan_sum, trees.pflanzjahr, trees.bezirk;
             """
         )
         trees = cur.fetchall()
@@ -87,7 +88,9 @@ def generate_trees_csv(temp_dir, db_conn):
         logging.info(f"Creating trees.csv file for {len(trees)} trees...")
 
         # Build CSV file with all trees in it
-        header = "id,lat,lng,radolan_sum,age,watering_sum,total_water_sum_liters"
+        header = (
+            "id,lat,lng,radolan_sum,age,watering_sum,total_water_sum_liters,district"
+        )
         lines = []
         for tree in tqdm(trees):
             id = tree[0]
@@ -111,7 +114,9 @@ def generate_trees_csv(temp_dir, db_conn):
             watering_sum = float(tree[5])
             total_water_sum_liters = (radolan_sum / 10.0) + watering_sum
 
-            line = f"{id}, {lat}, {lng}, {radolan_sum}, {age}, {watering_sum}, {total_water_sum_liters}"
+            district = tree[6]
+
+            line = f"{id}, {lat}, {lng}, {radolan_sum}, {age}, {watering_sum}, {total_water_sum_liters}, {district}"
             lines.append(line)
         trees_csv = "\n".join([header] + lines)
         trees_csv_full_path = os.path.join(temp_dir, "trees.csv")

harvester/src/run_daily_weather.py

@@ -52,13 +52,13 @@
 
 today = datetime.date.today()
 
-# Calculate the date one year ago
-one_year_ago = today - datetime.timedelta(days=3 * 365)
+# Calculate the date some years ago
+x_years_ago = today - datetime.timedelta(days=3 * 365 + 31)
 
-# Generate a list of all dates between one year ago and today
+# Generate a list of all dates between x years ago and today
 date_list = [
-    one_year_ago + datetime.timedelta(days=x)
-    for x in range((today - one_year_ago).days + 1)
+    x_years_ago + datetime.timedelta(days=x)
+    for x in range((today - x_years_ago).days + 1)
 ]
 
 print(f"📅 Fetching weather data for {len(date_list)} days...")
@@ -67,6 +67,22 @@
     cur.execute("SELECT measure_day, day_finished FROM daily_weather_data;")
     weather_days_in_db = cur.fetchall()
 
+outdated_weather_data = [
+    data_point_in_db
+    for data_point_in_db in weather_days_in_db
+    if data_point_in_db[0].date() < x_years_ago
+]
+logging.info(
+    f"🌦 Deleting {len(outdated_weather_data)} outdated weather data entries..."
+)
+for data_point in outdated_weather_data:
+    with database_connection.cursor() as cur:
+        cur.execute(
+            "DELETE FROM daily_weather_data WHERE measure_day = %s", [data_point[0]]
+        )
+database_connection.commit()
+
+
 for date in date_list:
     today = datetime.date.today()
 
@@ -75,15 +91,24 @@
         for data_point_in_db in weather_days_in_db
         if data_point_in_db[0].date() == date
     ]
+
     if existing_weather_in_db_for_this_day != []:
         logging.info(f"🌦 Weather data for {date} already exists in the database...")
-        if existing_weather_in_db_for_this_day[0][1] == False:
+
+        unfinished_weather_data = [
+            data_point_in_db
+            for data_point_in_db in existing_weather_in_db_for_this_day
+            if data_point_in_db[1] == False
+        ]
+
+        if unfinished_weather_data != []:
             logging.info(
                 f"🌦 Weather data for {date} was not finished in last run, updating now..."
             )
             with database_connection.cursor() as cur:
+                logging.info(f"🌦 Deleting old weather data for {date}...")
                 cur.execute(
-                    "DELETE FROM daily_weather_data WHERE measure_day = %s", [today]
+                    "DELETE FROM daily_weather_data WHERE measure_day = %s", [date]
                 )
             database_connection.commit()
         else: