diff --git a/.gitignore b/.gitignore index c5f9266..f9091dc 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ !*/ dist -build \ No newline at end of file +build +!.gitkeep \ No newline at end of file diff --git a/CONSTANT.py b/CONSTANT.py new file mode 100644 index 0000000..93740cd --- /dev/null +++ b/CONSTANT.py @@ -0,0 +1,10 @@ +from dataclasses import dataclass + + +@dataclass +class DATA_BASEPATH: + balloon = "./data/balloon" + cosmic= "./data/cosmic" + radar= "./data/radar" + saber= "./data/saber" + tidi= "./data/tidi" diff --git a/backend.py b/backend.py index b601d4e..24fa054 100644 --- a/backend.py +++ b/backend.py @@ -1,7 +1,6 @@ from matplotlib import pyplot as plt import cosmic import tidi -from utils import * import saber import radar import balloon diff --git a/balloon/__init__.py b/balloon/__init__.py index 5faeeb3..657a025 100644 --- a/balloon/__init__.py +++ b/balloon/__init__.py @@ -1,4 +1,5 @@ from quart import Blueprint +from CONSTANT import DATA_BASEPATH import balloon.extract_wave import balloon.read_data import balloon.plot_once @@ -6,11 +7,13 @@ import balloon.plot_year from quart import jsonify, request, send_file from balloon.plot_once_backend import render_by_mode_single from balloon.plot_year_backend import get_all_modes, render_based_on_mode -from utils import * +from balloon.utils import * -all_year_data = pd.read_parquet("./balloon/ballon_data_lin.parquet") +BASE_PATH_BALLOON = DATA_BASEPATH.balloon -all_ballon_files = glob.glob("./balloon/data/**/**.nc", recursive=True) +all_year_data = pd.read_parquet(f"{BASE_PATH_BALLOON}/ballon_data_lin.parquet") + +all_ballon_files = glob.glob(f"{BASE_PATH_BALLOON}/**/**.nc", recursive=True) def get_dataframe_between_year(start_year, end_year): res = all_year_data diff --git a/utils.py b/balloon/utils.py similarity index 100% rename from utils.py rename to balloon/utils.py diff --git a/cosmic/single.py b/cosmic/single.py index 8694c39..4f04767 100644 --- a/cosmic/single.py +++ b/cosmic/single.py @@ -7,6 +7,8 @@ from scipy.optimize import curve_fit import netCDF4 as nc import matplotlib.pyplot as plt +from CONSTANT import DATA_BASEPATH + # 模块1: 数据读取与预处理 @@ -377,8 +379,9 @@ def plot_results_mean_ktemp_Ptz(mean_ktemp_Ptz, heights): class SingleCosmicWavePlot: def __init__(self, year, day): + BASE_PATH_COSMIC = DATA_BASEPATH.cosmic - base_folder_path = f"./cosmic/data/{year}" + base_folder_path = f"./{BASE_PATH_COSMIC}/{year}" day_num = day # 模块1调用 final_df = read_and_preprocess_data(base_folder_path, day_num) diff --git a/cosmic/temp_render.py b/cosmic/temp_render.py index 0ec7ded..7d1e242 100644 --- a/cosmic/temp_render.py +++ b/cosmic/temp_render.py @@ -8,13 +8,15 @@ import matplotlib.pyplot as plt # 定义拟合函数 # 解决绘图中中文不能显示的问题 import matplotlib + +from CONSTANT import DATA_BASEPATH # 设置中文显示和负号正常显示 matplotlib.rcParams['font.sans-serif'] = ['SimHei'] # 显示中文 matplotlib.rcParams['axes.unicode_minus'] = False # 正常显示负号 # 读取一年的数据文件 -def temp_render(path:str = "./cosmic/cosmic.txt", T = 16): +def temp_render(path:str = f"{DATA_BASEPATH.cosmic}/cosmic.txt", T = 16): def u_func(x, *params): a1, b1, a2, b2, a3, b3, a4, b4, a5, b5, a6, b6, a7, b7, a8, b8, a9, b9 = params return ( diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/balloon/.gitkeep b/data/balloon/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/cosmic/.gitkeep b/data/cosmic/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/radar/.gitkeep b/data/radar/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/saber/.gitkeep b/data/saber/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/tidi/.gitkeep b/data/tidi/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/file_stuff.py b/file_stuff.py deleted file mode 100644 index 200670d..0000000 --- a/file_stuff.py +++ /dev/null @@ -1,11 +0,0 @@ -import pandas as pd - - -all_year_data = pd.read_parquet("./cache/ballon_data_lin.parquet") - - -def get_dataframe_between_year(start_year, end_year): - res = all_year_data - filtered_res = res[(res['file_name'].str.extract(r'LIN-(\d{4})')[0].astype(int) >= start_year) & - (res['file_name'].str.extract(r'LIN-(\d{4})')[0].astype(int) <= end_year)] - return filtered_res diff --git a/radar/__init__.py b/radar/__init__.py index 2a1921e..f090d59 100644 --- a/radar/__init__.py +++ b/radar/__init__.py @@ -3,11 +3,13 @@ import glob from io import BytesIO from quart import Blueprint, request, send_file from matplotlib import pyplot as plt +from CONSTANT import DATA_BASEPATH from radar.plot_original import final_render_v2 from radar.plot_prod import final_plot_v2 +from quart.utils import run_sync - -globed_all_files = glob.glob("./radar/data/**/**.txt", recursive=True) +BASE_PATH_RADAR = DATA_BASEPATH.radar +globed_all_files = glob.glob(f"{BASE_PATH_RADAR}/**/**.txt", recursive=True) radar_module = Blueprint("Radar", __name__) @@ -39,7 +41,7 @@ async def render_v1(): model_name = request.args.get('model_name') mode = request.args.get('mode') - renderer = final_render_v2(int(H), int(year), station, wind_type) + renderer = await run_sync(final_render_v2)(int(H), int(year), station, wind_type) if mode=="day": day = request.args.get('day') renderer.render_day(day,model_name) @@ -78,7 +80,7 @@ async def render_v2(): month_range = (1, 12) - buffer = final_plot_v2(int(year), station, model_name, month_range) + buffer = await run_sync(final_plot_v2)(int(year), station, model_name, month_range) buffer.seek(0) return await send_file(buffer, mimetype='image/png') diff --git a/radar/plot_original.py b/radar/plot_original.py index 7882aca..3e7b4da 100644 --- a/radar/plot_original.py +++ b/radar/plot_original.py @@ -17,6 +17,8 @@ from datetime import datetime # 解决绘图中中文不能显示的问题 import matplotlib + +from CONSTANT import DATA_BASEPATH # 设置中文显示和负号正常显示 matplotlib.rcParams['font.sans-serif'] = ['SimHei'] # 显示中文 matplotlib.rcParams['axes.unicode_minus'] = False # 正常显示负号 @@ -377,7 +379,8 @@ def get_final_df(H, year, station): # # station = '黑龙江漠河站' # 文件路径设置 - file_dir = rf'./radar/data/{station}/{year}' # 数据文件路径 + + file_dir = rf'{DATA_BASEPATH.radar}/{station}/{year}' # 数据文件路径 # output_dir = rf'./out\{station}\{year}' # 参数txt、图片输出路径 # os.makedirs(output_dir, exist_ok=True) # 确保输出路径存在 @@ -461,7 +464,8 @@ class final_render_v2(): @staticmethod def get_all_pathes(): - result = glob.glob(f"./radar/data/**/*.txt", recursive=True) + # + result = glob.glob(f"{DATA_BASEPATH.radar}/**/*.txt", recursive=True) # normalize path result = [os.path.normpath(path).replace("\\", "/") for path in result] return result diff --git a/radar/plot_prod.py b/radar/plot_prod.py index 4ca25aa..89f7707 100644 --- a/radar/plot_prod.py +++ b/radar/plot_prod.py @@ -10,6 +10,8 @@ import seaborn as sns # 解决绘图中中文不能显示的问题 import matplotlib + +from CONSTANT import DATA_BASEPATH matplotlib.rcParams['font.sans-serif'] = ['SimHei'] matplotlib.rcParams['axes.unicode_minus'] = False @@ -170,7 +172,7 @@ def final_plot_v2(year, station, model_name, month_range=(1,12)): - model_name: str,模型名称(如 '潮汐波', '2日行星波', '5日行星波', '10日行星波', '16日行星波') """ # 配置文件路径 - file_dir = rf'./radar/data/{station}/{year}' + file_dir = rf'{DATA_BASEPATH.radar}/{station}/{year}' # output_dir = rf'./radar/tmp\{station}\{year}\时空' # os.makedirs(output_dir, exist_ok=True) diff --git a/saber/__init__.py b/saber/__init__.py index 558b356..0da10db 100644 --- a/saber/__init__.py +++ b/saber/__init__.py @@ -2,6 +2,7 @@ import glob from io import BytesIO from quart import Blueprint, request, send_file from matplotlib import pyplot as plt +from CONSTANT import DATA_BASEPATH from saber.process import DataProcessor from saber.render import Renderer from saber.utils import * @@ -48,7 +49,7 @@ async def extract_payload(): return await send_file(buffer, mimetype="image/png") -_all_saber_files = glob.glob("./saber/data/**/**.nc", recursive=True) +_all_saber_files = glob.glob(f"{DATA_BASEPATH.saber}/**/**.nc", recursive=True) @saber_module.route("/metadata") @@ -59,10 +60,10 @@ def get_files(): @saber_module.route("/metadata/list_days") -def get_days(): +async def get_days(): path = request.args.get("path") - ncfile = data_nc_load(path) + ncfile = await run_sync(data_nc_load)(path) return ncfile.date_time.tolist() @@ -73,8 +74,8 @@ async def do_plot_wave_day_fitting(): day = request.args.get("day") height = request.args.get("height_no") - ncfile = data_nc_load(path) - processor = get_processer() + ncfile = await run_sync(data_nc_load)(path) + processor = await run_sync(get_processer)() data = await run_sync(processor.process_day)(ncfile, int(day)) await run_sync(renderer.plot_wave_fitting)(data, int(height)) return await extract_payload() @@ -86,8 +87,8 @@ async def do_day_fft_ifft_plot(): day = request.args.get("day") cycle_no = request.args.get("cycle_no") - ncfile = data_nc_load(path) - processor = get_processer() + ncfile = await run_sync(data_nc_load)(path) + processor = await run_sync(get_processer)() data = await run_sync(processor.process_day)(ncfile, int(day)) # 高度滤波处理 await run_sync(renderer.day_fft_ifft_plot)(wave_data=data, cycle_no=int(cycle_no)) @@ -100,8 +101,8 @@ async def do_day_cycle_power_wave_plot(): day = request.args.get("day") cycle_no = request.args.get("cycle_no") - ncfile = data_nc_load(path) - processor = get_processer() + ncfile = await run_sync(data_nc_load)(path) + processor = await run_sync(get_processer)() data = await run_sync(processor.process_day)(ncfile, int(day)) await run_sync(renderer.day_cycle_power_wave_plot)(wave_data=data, cycle_no=int(cycle_no)) return await extract_payload() @@ -112,8 +113,8 @@ async def do_month_power_wave_plot(): path = request.args.get("path") month = request.args.get("month") - ncfile = data_nc_load(path) - processor = get_processer() + ncfile = await run_sync(data_nc_load)(path) + processor = await run_sync(get_processer)() data =await run_sync(processor.process_month)(ncfile) await run_sync(renderer.month_power_wave_plot)(wave_data=data, date_time=ncfile.date_time) return await extract_payload() @@ -122,9 +123,9 @@ async def do_month_power_wave_plot(): @saber_module.route("/render/year_power_wave_plot") async def do_year_power_wave_plot(): year = request.args.get("year") - processor = get_processer() + processor = await run_sync(get_processer)() data = await run_sync(processor.process_year)([ - data_nc_load(path) for path in glob.glob(f"./saber/data/{year}/*.nc") + data_nc_load(path) for path in glob.glob(f"{DATA_BASEPATH.saber}/{year}/*.nc") ]) await run_sync(renderer.year_power_wave_plot)(year_wave=data) return await extract_payload() diff --git a/saber/archive/saber_render.py b/saber/archive/saber_render.py index 9dd7e4f..a8e3776 100644 --- a/saber/archive/saber_render.py +++ b/saber/archive/saber_render.py @@ -2,6 +2,7 @@ import numpy as np import matplotlib.pyplot as plt import matplotlib.dates as mdates +from CONSTANT import DATA_BASEPATH from saber.utils import * # from matplotlib.colors import LinearSegmentedColormap # 设置字体为支持中文的字体 @@ -694,7 +695,7 @@ def year_power_wave_plot(year, path, latitude_min, latitude_max, altitude_min, a # 7 主程序,运行数据,并输出主要统计分析图 - # ----------------------------------------------------------------------------------------------------------------------------- # 7-1 挑选某一天进行运行主要程序 -file_path = "./SABER/data\\2016\\SABER_Temp_O3_April2016_v2.0.nc" +file_path = f"{DATA_BASEPATH.saber}\\2016\\SABER_Temp_O3_April2016_v2.0.nc" # day_read=2018113, day_read = 2016094 # 初始化某一天、某个纬度、高度范围等参数 @@ -776,7 +777,7 @@ def filename_read(year, month_num, path): year = 2018 -path = "./saber/data" +path = f"{DATA_BASEPATH.saber}" # 初始化某一天、某个纬度、高度范围等参数 latitude_min = 30.0 latitude_max = 40.0 diff --git a/saber/process.py b/saber/process.py index ba3bdcd..7e5f233 100644 --- a/saber/process.py +++ b/saber/process.py @@ -2,6 +2,7 @@ from dataclasses import dataclass from datetime import datetime from typing import Dict, List, Optional, Tuple import numpy as np +from CONSTANT import DATA_BASEPATH from saber.utils import * # lat_range=(latitude_min, latitude_max), @@ -224,7 +225,7 @@ class DataProcessor: if __name__ == "__main__": year = 2018 - path = "./saber/data/2012/SABER_Temp_O3_April2012_v2.0.nc" + path = f"{DATA_BASEPATH.saber}/2012/SABER_Temp_O3_April2012_v2.0.nc" # 初始化某一天、某个纬度、高度范围等参数 latitude_min = 30.0 latitude_max = 40.0 diff --git a/tidi/__init__.py b/tidi/__init__.py index c67d88f..df69a97 100644 --- a/tidi/__init__.py +++ b/tidi/__init__.py @@ -4,6 +4,7 @@ from quart import Blueprint, request, send_file from quart.utils import run_sync from matplotlib import pyplot as plt +from CONSTANT import DATA_BASEPATH from tidi.plot import TidiPlotv2 from tidi.staged.plot import tidi_render @@ -12,7 +13,7 @@ tidi_module = Blueprint("Tidi", __name__) @tidi_module.route('/metadata') def get_all_years(): - res = glob.glob("./tidi/data/**/**.txt", recursive=True) + res = glob.glob(f"{DATA_BASEPATH.tidi}/**/**.txt", recursive=True) # search for the folder name that is year return { diff --git a/tidi/plot.py b/tidi/plot.py index d3b9dd8..1aa7588 100644 --- a/tidi/plot.py +++ b/tidi/plot.py @@ -5,6 +5,8 @@ from scipy.io import loadmat from scipy.optimize import curve_fit import matplotlib.pyplot as plt import seaborn as sns + +from CONSTANT import DATA_BASEPATH # --------------------------------------------------------------------------------------- # -----vzonal---------------------------------------------------------------------------- @@ -12,12 +14,13 @@ import seaborn as sns def process_vzonal_day(day, year=2015): try: # 读取数 据 - height_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Height.mat") - lat_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Lat.mat") - lon_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Lon.mat") + base_path = DATA_BASEPATH.tidi + height_data = loadmat(rf"{base_path}/{year}/{day:03d}_Height.mat") + lat_data = loadmat(rf"{base_path}/{year}/{day:03d}_Lat.mat") + lon_data = loadmat(rf"{base_path}/{year}/{day:03d}_Lon.mat") vmeridional_data = loadmat( - rf"./tidi/data/{year}/{day:03d}_VMerdional.mat") - vzonal_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Vzonal.mat") + rf"{base_path}/{year}/{day:03d}_VMerdional.mat") + vzonal_data = loadmat(rf"{base_path}/{year}/{day:03d}_Vzonal.mat") # 将数据转换为DataFrame height_df = pd.DataFrame(height_data['Height']) @@ -380,12 +383,14 @@ def process_vzonal_day(day, year=2015): def process_vmeridional_day(day, year=2015): try: # 读取数据 - height_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Height.mat") - lat_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Lat.mat") - lon_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Lon.mat") + base_path = DATA_BASEPATH.tidi + + height_data = loadmat(rf"{base_path}/{year}/{day:03d}_Height.mat") + lat_data = loadmat(rf"{base_path}/{year}/{day:03d}_Lat.mat") + lon_data = loadmat(rf"{base_path}/{year}/{day:03d}_Lon.mat") vmeridional_data = loadmat( - rf"./tidi/data/{year}/{day:03d}_VMerdional.mat") - vzonal_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Vzonal.mat") + rf"{base_path}/{year}/{day:03d}_VMerdional.mat") + vzonal_data = loadmat(rf"{base_path}/{year}/{day:03d}_Vzonal.mat") # 将数据转换为DataFrame height_df = pd.DataFrame(height_data['Height']) @@ -755,11 +760,11 @@ def day_to_month(day): class TidiPlotv2: def __init__(self, year): self.year = year - - if os.path.exists(f"./tidi/cache/{year}/all_days_vzonal_results.parquet") \ - and os.path.exists(f"./tidi/cache/{year}/all_days_vmeridional_results.parquet"): - all_days_vzonal_results = pd.read_parquet(f"./tidi/cache/{year}/all_days_vzonal_results.parquet") - all_days_vmeridional_results = pd.read_parquet(f"./tidi/cache/{year}/all_days_vmeridional_results.parquet") + cache_path = f"{DATA_BASEPATH.tidi}/cache" + if os.path.exists(f"{cache_path}/{year}/all_days_vzonal_results.parquet") \ + and os.path.exists(f"{cache_path}/{year}/all_days_vmeridional_results.parquet"): + all_days_vzonal_results = pd.read_parquet(f"{cache_path}/{year}/all_days_vzonal_results.parquet") + all_days_vmeridional_results = pd.read_parquet(f"{cache_path}/{year}/all_days_vmeridional_results.parquet") else: all_days_vzonal_results = pd.DataFrame() @@ -784,11 +789,11 @@ class TidiPlotv2: # cache the results # if dir not exists, create it - if not os.path.exists(f"./tidi/cache/{year}"): - os.makedirs(f"./tidi/cache/{year}") + if not os.path.exists(f"{cache_path}/{year}"): + os.makedirs(f"{cache_path}/{year}") - all_days_vzonal_results.to_parquet(f"./tidi/cache/{year}/all_days_vzonal_results.parquet") - all_days_vmeridional_results.to_parquet(f"./tidi/cache/{year}/all_days_vmeridional_results.parquet") + all_days_vzonal_results.to_parquet(f"{cache_path}/{year}/all_days_vzonal_results.parquet") + all_days_vmeridional_results.to_parquet(f"{cache_path}/{year}/all_days_vmeridional_results.parquet") self.all_days_vzonal_results = all_days_vzonal_results self.all_days_vmeridional_results = all_days_vmeridional_results @@ -837,7 +842,7 @@ class TidiPlotv2: months = self.months '''绘图''' plt.clf() - plt.rcParams['font.family'] = 'SimSun' # 宋体 + plt.rcParams['font.family'] = 'SimHei' # 宋体 plt.rcParams['font.size'] = 12 # 中文字号 plt.rcParams['axes.unicode_minus'] = False # 正确显示负号 plt.rcParams['font.sans-serif'] = 'Times New Roman' # 新罗马 diff --git a/tidi/staged/plot.py b/tidi/staged/plot.py index 71d610d..f557c01 100644 --- a/tidi/staged/plot.py +++ b/tidi/staged/plot.py @@ -10,6 +10,8 @@ import matplotlib.pyplot as plt # 解决绘图中中文不能显示的问题 import matplotlib + +from CONSTANT import DATA_BASEPATH # 设置中文显示和负号正常显示 matplotlib.rcParams['font.sans-serif'] = ['SimHei'] # 显示中文 matplotlib.rcParams['axes.unicode_minus'] = False # 正常显示负号 @@ -38,7 +40,7 @@ def tidi_render(mode, year, k, T=10): raise ValueError('mode must be V_Zonal or V_Meridional') if k not in list(range(-4, 5)): raise ValueError('k must be in range(-4, 5)') - df = pd.read_csv(f'./tidi/data/{year}/TIDI_{mode}_data.txt', sep='\s+') + df = pd.read_csv(f'{DATA_BASEPATH.tidi}/{year}/TIDI_{mode}_data.txt', sep='\s+') # 删除有 NaN 的行 # V_Meridional df = df.dropna(subset=[mode]) diff --git a/tidi/staged/plot_old.py b/tidi/staged/plot_old.py index 32d9c3e..96320aa 100644 --- a/tidi/staged/plot_old.py +++ b/tidi/staged/plot_old.py @@ -806,7 +806,7 @@ months = [day_to_month(day) for day in dates] '''绘图''' -plt.rcParams['font.family'] = 'SimSun' # 宋体 +plt.rcParams['font.family'] = 'SimHei' # 宋体 plt.rcParams['font.size'] = 12 # 中文字号 plt.rcParams['axes.unicode_minus'] = False # 正确显示负号 plt.rcParams['font.sans-serif'] = 'Times New Roman' # 新罗马 diff --git a/tidi/staged/process.py b/tidi/staged/process.py index 5334182..0a10ef4 100644 --- a/tidi/staged/process.py +++ b/tidi/staged/process.py @@ -3,6 +3,8 @@ import numpy as np from scipy.io import loadmat import pandas as pd +from CONSTANT import DATA_BASEPATH + #第156行代码,纬度可以自己选择,但是都是每5°做一个选择 #70km-120km每2.5km一个,共21个,第六个就是82.5km高度的情况,高度也可以自己选择 @@ -15,7 +17,7 @@ vmeridional_list = [] vzonal_list = [] # 文件路径 -base_path = "./tidi/data/2022/" +base_path = f"{DATA_BASEPATH.tidi}/2022/" # 循环读取从第1天到第365天的数据 for day in range(1, 366): # 365天数据,确保循环次数为365