migrate: new data dir

.gitignore

@@ -8,4 +8,5 @@
 !*/
 
 dist
-build
+build
+!.gitkeep

CONSTANT.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class DATA_BASEPATH:
+    balloon = "./data/balloon"
+    cosmic= "./data/cosmic"
+    radar= "./data/radar"
+    saber= "./data/saber"
+    tidi= "./data/tidi"

backend.py

@@ -1,7 +1,6 @@
 from matplotlib import pyplot as plt
 import cosmic
 import tidi
-from utils import *
 import saber
 import radar
 import balloon

balloon/__init__.py

@@ -1,4 +1,5 @@
 from quart import Blueprint
+from CONSTANT import DATA_BASEPATH
 import balloon.extract_wave
 import balloon.read_data
 import balloon.plot_once
@@ -6,11 +7,13 @@ import balloon.plot_year
 from quart import  jsonify, request, send_file
 from balloon.plot_once_backend import render_by_mode_single
 from balloon.plot_year_backend import get_all_modes, render_based_on_mode
-from utils import *
+from balloon.utils import *
 
-all_year_data = pd.read_parquet("./balloon/ballon_data_lin.parquet")
+BASE_PATH_BALLOON = DATA_BASEPATH.balloon
 
-all_ballon_files = glob.glob("./balloon/data/**/**.nc", recursive=True)
+all_year_data = pd.read_parquet(f"{BASE_PATH_BALLOON}/ballon_data_lin.parquet")
+
+all_ballon_files = glob.glob(f"{BASE_PATH_BALLOON}/**/**.nc", recursive=True)
 
 def get_dataframe_between_year(start_year, end_year):
     res = all_year_data

cosmic/single.py

@@ -7,6 +7,8 @@ from scipy.optimize import curve_fit
 import netCDF4 as nc
 import matplotlib.pyplot as plt
 
+from CONSTANT import DATA_BASEPATH
+
 # 模块1: 数据读取与预处理
 
 
@@ -377,8 +379,9 @@ def plot_results_mean_ktemp_Ptz(mean_ktemp_Ptz, heights):
 class SingleCosmicWavePlot:
     
     def __init__(self, year, day):
+        BASE_PATH_COSMIC = DATA_BASEPATH.cosmic
         
-        base_folder_path = f"./cosmic/data/{year}"
+        base_folder_path = f"./{BASE_PATH_COSMIC}/{year}"
         day_num = day
         # 模块1调用
         final_df = read_and_preprocess_data(base_folder_path, day_num)

cosmic/temp_render.py

@@ -8,13 +8,15 @@ import matplotlib.pyplot as plt
 # 定义拟合函数
 # 解决绘图中中文不能显示的问题
 import matplotlib
+
+from CONSTANT import DATA_BASEPATH
 # 设置中文显示和负号正常显示
 matplotlib.rcParams['font.sans-serif'] = ['SimHei']  # 显示中文
 matplotlib.rcParams['axes.unicode_minus'] = False  # 正常显示负号
 # 读取一年的数据文件
 
 
-def temp_render(path:str = "./cosmic/cosmic.txt", T = 16):
+def temp_render(path:str = f"{DATA_BASEPATH.cosmic}/cosmic.txt", T = 16):
     def u_func(x, *params):
         a1, b1, a2, b2, a3, b3, a4, b4, a5, b5, a6, b6, a7, b7, a8, b8, a9, b9 = params
         return (

file_stuff.py

@@ -1,11 +0,0 @@
-import pandas as pd
-
-
-all_year_data = pd.read_parquet("./cache/ballon_data_lin.parquet")
-
-
-def get_dataframe_between_year(start_year, end_year):
-    res = all_year_data
-    filtered_res = res[(res['file_name'].str.extract(r'LIN-(\d{4})')[0].astype(int) >= start_year) &
-                       (res['file_name'].str.extract(r'LIN-(\d{4})')[0].astype(int) <= end_year)]
-    return filtered_res

radar/__init__.py

@@ -3,11 +3,13 @@ import glob
 from io import BytesIO
 from quart import Blueprint, request, send_file
 from matplotlib import pyplot as plt
+from CONSTANT import DATA_BASEPATH
 from radar.plot_original import final_render_v2
 from radar.plot_prod import final_plot_v2
+from quart.utils import run_sync
 
-
+BASE_PATH_RADAR = DATA_BASEPATH.radar
-globed_all_files = glob.glob("./radar/data/**/**.txt", recursive=True)
+globed_all_files = glob.glob(f"{BASE_PATH_RADAR}/**/**.txt", recursive=True)
 
 
 radar_module = Blueprint("Radar", __name__)
@@ -39,7 +41,7 @@ async def render_v1():
     model_name = request.args.get('model_name')
     mode = request.args.get('mode')
     
-    renderer = final_render_v2(int(H), int(year), station, wind_type)
+    renderer = await run_sync(final_render_v2)(int(H), int(year), station, wind_type)
     if mode=="day":
         day = request.args.get('day')
         renderer.render_day(day,model_name)
@@ -78,7 +80,7 @@ async def render_v2():
         month_range = (1, 12)
     
 
-    buffer = final_plot_v2(int(year), station, model_name, month_range)
+    buffer = await run_sync(final_plot_v2)(int(year), station, model_name, month_range)
     buffer.seek(0)
 
     return await send_file(buffer, mimetype='image/png')

radar/plot_original.py

@@ -17,6 +17,8 @@ from datetime import datetime
 
 # 解决绘图中中文不能显示的问题
 import matplotlib
+
+from CONSTANT import DATA_BASEPATH
 # 设置中文显示和负号正常显示
 matplotlib.rcParams['font.sans-serif'] = ['SimHei']  # 显示中文
 matplotlib.rcParams['axes.unicode_minus'] = False  # 正常显示负号
@@ -377,7 +379,8 @@ def get_final_df(H, year, station):
 # # station = '黑龙江漠河站'
 
     # 文件路径设置
-    file_dir = rf'./radar/data/{station}/{year}'  # 数据文件路径
+    
+    file_dir = rf'{DATA_BASEPATH.radar}/{station}/{year}'  # 数据文件路径
     # output_dir = rf'./out\{station}\{year}'  # 参数txt、图片输出路径
     # os.makedirs(output_dir, exist_ok=True)  # 确保输出路径存在
 
@@ -461,7 +464,8 @@ class final_render_v2():
     
     @staticmethod
     def get_all_pathes():
-        result =  glob.glob(f"./radar/data/**/*.txt", recursive=True)
+        # 
+        result =  glob.glob(f"{DATA_BASEPATH.radar}/**/*.txt", recursive=True)
         # normalize path
         result = [os.path.normpath(path).replace("\\", "/") for path in result]
         return result

radar/plot_prod.py

@@ -10,6 +10,8 @@ import seaborn as sns
 
 # 解决绘图中中文不能显示的问题
 import matplotlib
+
+from CONSTANT import DATA_BASEPATH
 matplotlib.rcParams['font.sans-serif'] = ['SimHei']
 matplotlib.rcParams['axes.unicode_minus'] = False
 
@@ -170,7 +172,7 @@ def final_plot_v2(year, station, model_name, month_range=(1,12)):
     - model_name: str，模型名称（如 '潮汐波', '2日行星波', '5日行星波', '10日行星波', '16日行星波'）
     """
     # 配置文件路径
-    file_dir = rf'./radar/data/{station}/{year}'
+    file_dir = rf'{DATA_BASEPATH.radar}/{station}/{year}'
     # output_dir = rf'./radar/tmp\{station}\{year}\时空'
     # os.makedirs(output_dir, exist_ok=True)
 

saber/__init__.py

@@ -2,6 +2,7 @@ import glob
 from io import BytesIO
 from quart import Blueprint, request, send_file
 from matplotlib import pyplot as plt
+from CONSTANT import DATA_BASEPATH
 from saber.process import DataProcessor
 from saber.render import Renderer
 from saber.utils import *
@@ -48,7 +49,7 @@ async def extract_payload():
     return await  send_file(buffer, mimetype="image/png")
 
 
-_all_saber_files = glob.glob("./saber/data/**/**.nc", recursive=True)
+_all_saber_files = glob.glob(f"{DATA_BASEPATH.saber}/**/**.nc", recursive=True)
 
 
 @saber_module.route("/metadata")
@@ -59,10 +60,10 @@ def get_files():
 
 
 @saber_module.route("/metadata/list_days")
-def get_days():
+async def get_days():
     path = request.args.get("path")
 
-    ncfile = data_nc_load(path)
+    ncfile = await run_sync(data_nc_load)(path)
 
     return ncfile.date_time.tolist()
 
@@ -73,8 +74,8 @@ async def do_plot_wave_day_fitting():
     day = request.args.get("day")
     height = request.args.get("height_no")
 
-    ncfile = data_nc_load(path)
+    ncfile = await run_sync(data_nc_load)(path)
-    processor = get_processer()
+    processor = await run_sync(get_processer)()
     data = await run_sync(processor.process_day)(ncfile, int(day))
     await run_sync(renderer.plot_wave_fitting)(data, int(height))
     return await  extract_payload()
@@ -86,8 +87,8 @@ async def do_day_fft_ifft_plot():
     day = request.args.get("day")
     cycle_no = request.args.get("cycle_no")
 
-    ncfile = data_nc_load(path)
+    ncfile = await run_sync(data_nc_load)(path)
-    processor = get_processer()
+    processor = await run_sync(get_processer)()
     data = await run_sync(processor.process_day)(ncfile, int(day))
     # 高度滤波处理
     await run_sync(renderer.day_fft_ifft_plot)(wave_data=data, cycle_no=int(cycle_no))
@@ -100,8 +101,8 @@ async def do_day_cycle_power_wave_plot():
     day = request.args.get("day")
     cycle_no = request.args.get("cycle_no")
 
-    ncfile = data_nc_load(path)
+    ncfile = await run_sync(data_nc_load)(path)
-    processor = get_processer()
+    processor = await run_sync(get_processer)()
     data = await run_sync(processor.process_day)(ncfile, int(day))
     await run_sync(renderer.day_cycle_power_wave_plot)(wave_data=data, cycle_no=int(cycle_no))
     return await extract_payload()
@@ -112,8 +113,8 @@ async def do_month_power_wave_plot():
     path = request.args.get("path")
     month = request.args.get("month")
 
-    ncfile = data_nc_load(path)
+    ncfile = await run_sync(data_nc_load)(path)
-    processor = get_processer()
+    processor = await run_sync(get_processer)()
     data =await run_sync(processor.process_month)(ncfile)
     await run_sync(renderer.month_power_wave_plot)(wave_data=data, date_time=ncfile.date_time)
     return await  extract_payload()
@@ -122,9 +123,9 @@ async def do_month_power_wave_plot():
 @saber_module.route("/render/year_power_wave_plot")
 async def do_year_power_wave_plot():
     year = request.args.get("year")
-    processor = get_processer()
+    processor = await run_sync(get_processer)()
     data = await run_sync(processor.process_year)([
-        data_nc_load(path) for path in glob.glob(f"./saber/data/{year}/*.nc")
+        data_nc_load(path) for path in glob.glob(f"{DATA_BASEPATH.saber}/{year}/*.nc")
     ])
     await run_sync(renderer.year_power_wave_plot)(year_wave=data)
     return await extract_payload()

saber/archive/saber_render.py

@@ -2,6 +2,7 @@ import numpy as np
 import matplotlib.pyplot as plt
 import matplotlib.dates as mdates
 
+from CONSTANT import DATA_BASEPATH
 from saber.utils import *
 # from matplotlib.colors import LinearSegmentedColormap
 # 设置字体为支持中文的字体
@@ -694,7 +695,7 @@ def year_power_wave_plot(year, path, latitude_min, latitude_max, altitude_min, a
 # 7 主程序，运行数据,并输出主要统计分析图                                                                                          -
 # -----------------------------------------------------------------------------------------------------------------------------
 # 7-1 挑选某一天进行运行主要程序
-file_path = "./SABER/data\\2016\\SABER_Temp_O3_April2016_v2.0.nc"
+file_path = f"{DATA_BASEPATH.saber}\\2016\\SABER_Temp_O3_April2016_v2.0.nc"
 # day_read=2018113,
 day_read = 2016094
 # 初始化某一天、某个纬度、高度范围等参数
@@ -776,7 +777,7 @@ def filename_read(year, month_num, path):
 
 
 year = 2018
-path = "./saber/data"
+path = f"{DATA_BASEPATH.saber}"
 # 初始化某一天、某个纬度、高度范围等参数
 latitude_min = 30.0
 latitude_max = 40.0

saber/process.py

@@ -2,6 +2,7 @@ from dataclasses import dataclass
 from datetime import datetime
 from typing import Dict, List, Optional, Tuple
 import numpy as np
+from CONSTANT import DATA_BASEPATH
 from saber.utils import *
 
 # lat_range=(latitude_min, latitude_max),
@@ -224,7 +225,7 @@ class DataProcessor:
 
 if __name__ == "__main__":
     year = 2018
-    path = "./saber/data/2012/SABER_Temp_O3_April2012_v2.0.nc"
+    path = f"{DATA_BASEPATH.saber}/2012/SABER_Temp_O3_April2012_v2.0.nc"
     # 初始化某一天、某个纬度、高度范围等参数
     latitude_min = 30.0
     latitude_max = 40.0

tidi/__init__.py

@@ -4,6 +4,7 @@ from quart import Blueprint, request, send_file
 from quart.utils import run_sync
 from matplotlib import pyplot as plt
 
+from CONSTANT import DATA_BASEPATH
 from tidi.plot import TidiPlotv2
 from tidi.staged.plot import tidi_render
 
@@ -12,7 +13,7 @@ tidi_module = Blueprint("Tidi", __name__)
 
 @tidi_module.route('/metadata')
 def get_all_years():
-    res = glob.glob("./tidi/data/**/**.txt", recursive=True)
+    res = glob.glob(f"{DATA_BASEPATH.tidi}/**/**.txt", recursive=True)
     # search for the folder name that is year
 
     return {

tidi/plot.py

@@ -5,6 +5,8 @@ from scipy.io import loadmat
 from scipy.optimize import curve_fit
 import matplotlib.pyplot as plt
 import seaborn as sns
+
+from CONSTANT import DATA_BASEPATH
 # ---------------------------------------------------------------------------------------
 # -----vzonal----------------------------------------------------------------------------
 
@@ -12,12 +14,13 @@ import seaborn as sns
 def process_vzonal_day(day, year=2015):
     try:
         # 读取数              据
-        height_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Height.mat")
+        base_path = DATA_BASEPATH.tidi
-        lat_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Lat.mat")
+        height_data = loadmat(rf"{base_path}/{year}/{day:03d}_Height.mat")
-        lon_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Lon.mat")
+        lat_data = loadmat(rf"{base_path}/{year}/{day:03d}_Lat.mat")
+        lon_data = loadmat(rf"{base_path}/{year}/{day:03d}_Lon.mat")
         vmeridional_data = loadmat(
-            rf"./tidi/data/{year}/{day:03d}_VMerdional.mat")
+            rf"{base_path}/{year}/{day:03d}_VMerdional.mat")
-        vzonal_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Vzonal.mat")
+        vzonal_data = loadmat(rf"{base_path}/{year}/{day:03d}_Vzonal.mat")
 
         # 将数据转换为DataFrame
         height_df = pd.DataFrame(height_data['Height'])
@@ -380,12 +383,14 @@ def process_vzonal_day(day, year=2015):
 def process_vmeridional_day(day, year=2015):
     try:
         # 读取数据
-        height_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Height.mat")
+        base_path = DATA_BASEPATH.tidi
-        lat_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Lat.mat")
+        
-        lon_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Lon.mat")
+        height_data = loadmat(rf"{base_path}/{year}/{day:03d}_Height.mat")
+        lat_data = loadmat(rf"{base_path}/{year}/{day:03d}_Lat.mat")
+        lon_data = loadmat(rf"{base_path}/{year}/{day:03d}_Lon.mat")
         vmeridional_data = loadmat(
-            rf"./tidi/data/{year}/{day:03d}_VMerdional.mat")
+            rf"{base_path}/{year}/{day:03d}_VMerdional.mat")
-        vzonal_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Vzonal.mat")
+        vzonal_data = loadmat(rf"{base_path}/{year}/{day:03d}_Vzonal.mat")
 
         # 将数据转换为DataFrame
         height_df = pd.DataFrame(height_data['Height'])
@@ -755,11 +760,11 @@ def day_to_month(day):
 class TidiPlotv2:
     def __init__(self, year):
         self.year = year
-        
+        cache_path = f"{DATA_BASEPATH.tidi}/cache"
-        if os.path.exists(f"./tidi/cache/{year}/all_days_vzonal_results.parquet") \
+        if os.path.exists(f"{cache_path}/{year}/all_days_vzonal_results.parquet") \
-        and os.path.exists(f"./tidi/cache/{year}/all_days_vmeridional_results.parquet"):
+        and os.path.exists(f"{cache_path}/{year}/all_days_vmeridional_results.parquet"):
-            all_days_vzonal_results = pd.read_parquet(f"./tidi/cache/{year}/all_days_vzonal_results.parquet")
+            all_days_vzonal_results = pd.read_parquet(f"{cache_path}/{year}/all_days_vzonal_results.parquet")
-            all_days_vmeridional_results = pd.read_parquet(f"./tidi/cache/{year}/all_days_vmeridional_results.parquet")
+            all_days_vmeridional_results = pd.read_parquet(f"{cache_path}/{year}/all_days_vmeridional_results.parquet")
         else:
     
             all_days_vzonal_results = pd.DataFrame()
@@ -784,11 +789,11 @@ class TidiPlotv2:
             
             # cache the results
             # if dir not exists, create it
-            if not os.path.exists(f"./tidi/cache/{year}"):
+            if not os.path.exists(f"{cache_path}/{year}"):
-                os.makedirs(f"./tidi/cache/{year}")
+                os.makedirs(f"{cache_path}/{year}")
             
-            all_days_vzonal_results.to_parquet(f"./tidi/cache/{year}/all_days_vzonal_results.parquet")
+            all_days_vzonal_results.to_parquet(f"{cache_path}/{year}/all_days_vzonal_results.parquet")
-            all_days_vmeridional_results.to_parquet(f"./tidi/cache/{year}/all_days_vmeridional_results.parquet")
+            all_days_vmeridional_results.to_parquet(f"{cache_path}/{year}/all_days_vmeridional_results.parquet")
         
         self.all_days_vzonal_results = all_days_vzonal_results
         self.all_days_vmeridional_results = all_days_vmeridional_results
@@ -837,7 +842,7 @@ class TidiPlotv2:
         months = self.months
         '''绘图'''
         plt.clf()
-        plt.rcParams['font.family'] = 'SimSun'  # 宋体
+        plt.rcParams['font.family'] = 'SimHei'  # 宋体
         plt.rcParams['font.size'] = 12           # 中文字号
         plt.rcParams['axes.unicode_minus'] = False  # 正确显示负号
         plt.rcParams['font.sans-serif'] = 'Times New Roman'  # 新罗马

tidi/staged/plot.py

@@ -10,6 +10,8 @@ import matplotlib.pyplot as plt
 
 # 解决绘图中中文不能显示的问题
 import matplotlib
+
+from CONSTANT import DATA_BASEPATH
 # 设置中文显示和负号正常显示
 matplotlib.rcParams['font.sans-serif'] = ['SimHei']  # 显示中文
 matplotlib.rcParams['axes.unicode_minus'] = False  # 正常显示负号
@@ -38,7 +40,7 @@ def tidi_render(mode, year, k, T=10):
         raise ValueError('mode must be V_Zonal or V_Meridional')
     if k not in  list(range(-4, 5)):
         raise ValueError('k must be in range(-4, 5)')
-    df = pd.read_csv(f'./tidi/data/{year}/TIDI_{mode}_data.txt',  sep='\s+')
+    df = pd.read_csv(f'{DATA_BASEPATH.tidi}/{year}/TIDI_{mode}_data.txt',  sep='\s+')
     # 删除有 NaN 的行
     # V_Meridional
     df = df.dropna(subset=[mode])

tidi/staged/plot_old.py

@@ -806,7 +806,7 @@ months = [day_to_month(day) for day in dates]
 
 
 '''绘图'''
-plt.rcParams['font.family'] = 'SimSun'  # 宋体
+plt.rcParams['font.family'] = 'SimHei'  # 宋体
 plt.rcParams['font.size'] = 12           # 中文字号
 plt.rcParams['axes.unicode_minus'] = False  # 正确显示负号
 plt.rcParams['font.sans-serif'] = 'Times New Roman'  # 新罗马

tidi/staged/process.py

@@ -3,6 +3,8 @@ import numpy as np
 from scipy.io import loadmat
 import pandas as pd
 
+from CONSTANT import DATA_BASEPATH
+
 #第156行代码，纬度可以自己选择，但是都是每5°做一个选择
 #70km-120km每2.5km一个，共21个，第六个就是82.5km高度的情况，高度也可以自己选择
 
@@ -15,7 +17,7 @@ vmeridional_list = []
 vzonal_list = []
 
 # 文件路径
-base_path = "./tidi/data/2022/"
+base_path = f"{DATA_BASEPATH.tidi}/2022/"
 
 # 循环读取从第1天到第365天的数据
 for day in range(1, 366):  # 365天数据，确保循环次数为365