{ "cells": [ { "cell_type": "markdown", "id": "119e8e41", "metadata": {}, "source": [ "### What this script does\n", "\n", "This script processes 10-minute merged mast, radiometer, and sonic data to compute turbulent momentum fluxes and update each daily merged_data_10min.csv file with new stress components.\n", "\n", "It automatically:\n", "\n", "- Loops over all target months (2024-03, 2024-04, 2024-05) and their valid dates.\n", "\n", "- Searches each day’s folder inside your local data directory for:\n", "\n", " - merged_data_10min.csv\n", "\n", " - Raw .dat files containing high-frequency wind components (Ux, Uy, Uz).\n", "\n", " - Rotates the wind components by a fixed theta angle (116°) to align them with the mean wind direction.\n", "\n", " - Divides each .dat file into ~10-minute chunks (12 000 samples each), and for each chunk computes:\n", "\n", " - uw_flux_corr → covariance of u′ and w′\n", "\n", " - vw_flux_corr → covariance of v′ and w′\n", "\n", " - uv_flux_corr → covariance of u′ and v′\n", " (these represent corrected turbulent fluxes).\n", "\n", "- Merges these fluxes into the corresponding daily merged_data_10min.csv.\n", "\n", "- Computes corrected stress components using air density (rho_air_Tv):\n", "\n", " - tau_xz_corr = −ρ * uw_flux_corr\n", "\n", " - tau_yz_corr = −ρ * vw_flux_corr\n", "\n", " - tau_xy_corr = −ρ * uv_flux_corr\n", "\n", "- Saves the updated CSV back to disk.\n", "\n", "Edit before running:\n", "Base directory containing the 10-min merged data\n", "base_dir = r\"C:\\path\\to\\your\\Sonic\"\n" ] }, { "cell_type": "code", "execution_count": null, "id": "6a43b622", "metadata": {}, "outputs": [], "source": [ "import os\n", "import pandas as pd\n", "import numpy as np\n", "from datetime import datetime, timedelta" ] }, { "cell_type": "code", "execution_count": null, "id": "b4461157", "metadata": {}, "outputs": [], "source": [ "# Base directory containing the 10 min-merged mast/radiometer/sonic data\n", "# ⚠️ Edit this path to match your local setup before running.\n", "# Example: base_dir = r\"D:\\MyThesis\\data\\Sonic\"\n", "base_dir = r\"C:\\path\\to\\your\\Sonic\"\n", "\n", "months = ['2024-03', '2024-04', '2024-05']\n", "theta_deg = 116\n", "theta_rad = np.deg2rad(theta_deg)\n", "\n", "for month_str in months:\n", " month_dt = datetime.strptime(month_str, '%Y-%m')\n", " for day in range(1, 32):\n", " try:\n", " date_dt = month_dt.replace(day=day)\n", " date_str = date_dt.strftime('%Y-%m-%d') # '2024-05-01\n", "\n", " except ValueError:\n", " continue # skip invalid dates\n", "\n", " data_dir = os.path.join(base_dir, month_str, date_str)\n", " \n", " merged_file_path = os.path.join(data_dir, 'merged_data_10min.csv')\n", " print(f\"Checking {data_dir}\")\n", "\n", " if not os.path.exists(merged_file_path):\n", " continue\n", "\n", " if not any(fname.endswith('.dat') for fname in os.listdir(data_dir)):\n", " continue\n", "\n", " timestamps_10min = []\n", " uw_flux_10min_corr = []\n", " vw_flux_10min_corr = []\n", " uv_flux_10min_corr = []\n", "\n", " for filename in os.listdir(data_dir):\n", " if filename.endswith('.dat'):\n", " file_path = os.path.join(data_dir, filename)\n", " data = pd.read_csv(file_path, skiprows=1, delimiter=',', encoding='latin1', low_memory=False)\n", " data.dropna(inplace=True)\n", "\n", " data['TIMESTAMP'] = pd.to_datetime(data['TIMESTAMP'], errors='coerce')\n", " for var in ['Ux', 'Uy', 'Uz']:\n", " data[var] = pd.to_numeric(data[var], errors='coerce')\n", "\n", " data['Ux_corrected'] = data['Ux'] * np.cos(theta_rad) + data['Uy'] * np.sin(theta_rad)\n", " data['Uy_corrected'] = -data['Ux'] * np.sin(theta_rad) + data['Uy'] * np.cos(theta_rad)\n", "\n", " for i in range(0, len(data), 12000):\n", " chunk = data.iloc[i:i+12000]\n", " if chunk.empty:\n", " continue\n", "\n", " u_mean = chunk['Ux_corrected'].mean()\n", " v_mean = chunk['Uy_corrected'].mean()\n", " w_mean = chunk['Uz'].mean()\n", "\n", " u_prime = chunk['Ux_corrected'] - u_mean\n", " v_prime = chunk['Uy_corrected'] - v_mean\n", " w_prime = chunk['Uz'] - w_mean\n", "\n", " uw_flux_corr = (u_prime * w_prime).mean()\n", " vw_flux_corr = (v_prime * w_prime).mean()\n", " uv_flux_corr = (u_prime * v_prime).mean()\n", "\n", " timestamps_10min.append(chunk['TIMESTAMP'].iloc[0].floor('10T'))\n", " uw_flux_10min_corr.append(uw_flux_corr)\n", " vw_flux_10min_corr.append(vw_flux_corr)\n", " uv_flux_10min_corr.append(uv_flux_corr)\n", "\n", " if not timestamps_10min:\n", " continue\n", "\n", " flux_df = pd.DataFrame({\n", " 'TIMESTAMP': timestamps_10min,\n", " 'uw_flux_corr': uw_flux_10min_corr,\n", " 'vw_flux_corr': vw_flux_10min_corr,\n", " 'uv_flux_corr': uv_flux_10min_corr,\n", " })\n", "\n", " try:\n", " merged_df = pd.read_csv(merged_file_path, parse_dates=['TIMESTAMP'])\n", " merged_df = pd.merge(merged_df, flux_df, on='TIMESTAMP', how='left')\n", "\n", " # Compute corrected stress components\n", " merged_df['tau_xz_corr'] = -merged_df['rho_air_Tv'] * merged_df['uw_flux_corr']\n", " merged_df['tau_yz_corr'] = -merged_df['rho_air_Tv'] * merged_df['vw_flux_corr']\n", " merged_df['tau_xy_corr'] = -merged_df['rho_air_Tv'] * merged_df['uv_flux_corr']\n", "\n", " merged_df.to_csv(merged_file_path, index=False)\n", " print(f'Updated: {merged_file_path}')\n", " except Exception as e:\n", " print(f\"Error processing {merged_file_path}: {e}\")" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.7" } }, "nbformat": 4, "nbformat_minor": 5 }