past_data_api_calls.py

import codecs
import csv
import http.client
import os
import re
import sys
import urllib.request
from datetime import date, timedelta
from io import StringIO

import pandas as pd


def pollution_data():
    particles = ["NO2", "O3"]
    stations = ["NL10636", "NL10639", "NL10643"]
    last_year_date = date.today() - timedelta(days=365)
    start_date = last_year_date - timedelta(days=7)
    end_date = last_year_date + timedelta(days=3)
    date_list = [start_date + timedelta(days=x) for x in range((end_date - start_date).days + 1)]
    for current_date in date_list:
        today = current_date.isoformat() + "T09:00:00Z"
        yesterday = (current_date - timedelta(1)).isoformat() + "T09:00:00Z"
        for particle in particles:
            all_dataframes = []  # Reset for each particle
            for station in stations:
                conn = http.client.HTTPSConnection("api.luchtmeetnet.nl")
                payload = ''
                headers = {}
                conn.request("GET", f"/open_api/measurements?station_number={station}&formula={particle}&page=1&order_by=timestamp_measured&order_direction=desc&end={today}&start={yesterday}", payload, headers)
                res = conn.getresponse()
                data = res.read()
                decoded_data = data.decode("utf-8")
                df = pd.read_csv(StringIO(decoded_data))
                df = df.filter(like='value')
                all_dataframes.append(df)
            if all_dataframes:
                combined_data = pd.concat(all_dataframes, ignore_index=True)
                combined_data.to_csv(f'{particle}_{today}.csv', index=False)

def delete_csv(csvs):
    for csv_file in csvs:
        if(os.path.exists(csv_file) and os.path.isfile(csv_file)): 
            os.remove(csv_file)

def clean_values():
    particles = ["NO2", "O3"]
    csvs = []
    NO2 = []
    O3 = []
    last_year_date = date.today() - timedelta(days=365)
    start_date = last_year_date - timedelta(days=7)
    end_date = last_year_date + timedelta(days=3)
    date_list = [start_date + timedelta(days=x) for x in range((end_date - start_date).days + 1)]
    for current_date in date_list:
        today = current_date.isoformat() + "T09:00:00Z"
        for particle in particles:
            name = f'{particle}_{today}.csv'
            csvs.append(name)
    for csv_file in csvs:
        if not os.path.exists(csv_file):
            continue  # Skip if the file doesn't exist
        values = []  # Reset values for each CSV file
        # Open the CSV file and read the values
        with open(csv_file, 'r') as file:
            reader = csv.reader(file)
            for row in reader:
                for value in row:
                    # Use regular expressions to extract numeric part
                    cleaned_value = re.findall(r"[-+]?\d*\.\d+|\d+", value)
                    if cleaned_value:  # If we successfully extract a number
                        values.append(float(cleaned_value[0]))  # Convert the first match to float

        # Compute the average if the values list is not empty
        if values:
            avg = sum(values) / len(values)
            if "NO2" in csv_file:
                NO2.append(avg)
            else:
                O3.append(avg)
    delete_csv(csvs)
    return NO2, O3

def add_columns():
    file_path = 'weather_data.csv'
    df = pd.read_csv(file_path)

    df.insert(1, 'NO2', None)
    df.insert(2, 'O3', None)
    df.insert(10, 'weekday', None)

    return df

def scale(data):
    df = data
    columns = list(df.columns)

    columns.insert(3, columns.pop(6))
    df = df[columns]

    columns.insert(5, columns.pop(9))
    df = df[columns]

    columns.insert(9, columns.pop(6))
    df = df[columns]

    df = df.rename(columns={
        'datetime':'date',
        'windspeed': 'wind_speed',
        'temp': 'mean_temp',
        'solarradiation':'global_radiation',
        'precip':'percipitation',
        'sealevelpressure':'pressure',
        'visibility':'minimum_visibility'
    })

    df['date'] = pd.to_datetime(df['date'])
    df['weekday'] = df['date'].dt.day_name()

    df = df.sort_values(by='date').reset_index(drop=True)

    df['wind_speed'] = (df['wind_speed'] / 3.6) * 10
    df['mean_temp'] = df['mean_temp'] * 10
    df['minimum_visibility'] = df['minimum_visibility'] * 10
    df['percipitation'] = df['percipitation'] * 10
    df['pressure'] = df['pressure']

    df['wind_speed'] = df['wind_speed'].astype(int)
    df['mean_temp'] = df['mean_temp'].astype(int)
    df['minimum_visibility'] = df['minimum_visibility'].astype(int)
    df['percipitation'] = df['percipitation'].astype(int)
    df['pressure'] = df['pressure'].astype(int)
    df['humidity'] = df['humidity'].astype(int)
    df['global_radiation'] = df['global_radiation'].astype(int)

    return df

def insert_pollution(NO2, O3, data):
    df = data
    df['NO2'] = NO2
    df['O3'] = O3
    return df

def weather_data():
    # Get last year's same day
    last_year_date = date.today() - timedelta(days=365)
    # Start date is 7 days prior
    start_date = (last_year_date - timedelta(days=7)).isoformat()
    # End date is 3 days ahead
    end_date = (last_year_date + timedelta(days=3)).isoformat()
    try: 
        ResultBytes = urllib.request.urlopen(f"https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/timeline/Utrecht/{start_date}/{end_date}?unitGroup=metric&elements=datetime%2Cwindspeed%2Ctemp%2Csolarradiation%2Cprecip%2Cpressure%2Cvisibility%2Chumidity&include=days&key=7Y6AY56M6RWVNHQ3SAVHNJWFS&maxStations=1&contentType=csv")
        
        # Parse the results as CSV
        CSVText = csv.reader(codecs.iterdecode(ResultBytes, 'utf-8'))
        # Saving the CSV content to a file
        current_dir = os.path.dirname(os.path.realpath(__file__))
        file_path = os.path.join(current_dir, 'weather_data.csv')
        with open(file_path, 'w', newline='', encoding='utf-8') as csvfile:
              csv_writer = csv.writer(csvfile)
              csv_writer.writerows(CSVText)
    
    except urllib.error.HTTPError  as e:
        ErrorInfo= e.read().decode() 
        print('Error code: ', e.code, ErrorInfo)
        sys.exit()
    except  urllib.error.URLError as e:
        ErrorInfo= e.read().decode() 
        print('Error code: ', e.code,ErrorInfo)
        sys.exit()

def weather_data():
    # Set up dates for last year: 7 days before today last year, and 3 days ahead of this day last year
    today_last_year = date.today() - timedelta(365)
    start_last_year = today_last_year - timedelta(8)
    end_last_year = today_last_year + timedelta(2)
    
    try: 
        # API call with new date range for last year
        ResultBytes = urllib.request.urlopen(f"https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/timeline/Utrecht/{start_last_year}/{end_last_year}?unitGroup=metric&elements=datetime%2Cwindspeed%2Ctemp%2Csolarradiation%2Cprecip%2Cpressure%2Cvisibility%2Chumidity&include=days&key=7Y6AY56M6RWVNHQ3SAVHNJWFS&maxStations=1&contentType=csv")
        
        # Parse the results as CSV
        CSVText = csv.reader(codecs.iterdecode(ResultBytes, 'utf-8'))
        # Saving the CSV content to a file
        current_dir = os.path.dirname(os.path.realpath(__file__))
        file_path = os.path.join(current_dir, 'weather_data.csv')
        with open(file_path, 'w', newline='', encoding='utf-8') as csvfile:
            csv_writer = csv.writer(csvfile)
            csv_writer.writerows(CSVText)
    
    except urllib.error.HTTPError as e:
        ErrorInfo = e.read().decode() 
        print('Error code: ', e.code, ErrorInfo)
        sys.exit()
    except urllib.error.URLError as e:
        ErrorInfo = e.read().decode() 
        print('Error code: ', e.code, ErrorInfo)
        sys.exit()


def get_past_data():
    weather_data()
    pollution_data()
    NO2, O3 = clean_values()
    df = add_columns()
    scaled_df = scale(df)
    output_df = insert_pollution(NO2, O3, scaled_df)
    os.remove('weather_data.csv')
    return output_df