zephyr/zephyr-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

migrate: new data dir

Browse Source
This commit is contained in:
Dustella 2025-02-13 20:03:12 +08:00
parent 7237b0ec19
commit df29a6229e
Signed by: Dustella
GPG Key ID: 35AA0AA3DC402D5C
25 changed files with 93 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -8,4 +8,5 @@
!*/
dist
build
build
!.gitkeep

10
CONSTANT.py Normal file
View File

@ -0,0 +1,10 @@
from dataclasses import dataclass
@dataclass
class DATA_BASEPATH:
balloon = "./data/balloon"
cosmic= "./data/cosmic"
radar= "./data/radar"
saber= "./data/saber"
tidi= "./data/tidi"

1
backend.py
View File

@ -1,7 +1,6 @@
from matplotlib import pyplot as plt
import cosmic
import tidi
from utils import *
import saber
import radar
import balloon

9
balloon/__init__.py
View File

@ -1,4 +1,5 @@
from quart import Blueprint
from CONSTANT import DATA_BASEPATH
import balloon.extract_wave
import balloon.read_data
import balloon.plot_once
@ -6,11 +7,13 @@ import balloon.plot_year
from quart import jsonify, request, send_file
from balloon.plot_once_backend import render_by_mode_single
from balloon.plot_year_backend import get_all_modes, render_based_on_mode
from utils import *
from balloon.utils import *
all_year_data = pd.read_parquet("./balloon/ballon_data_lin.parquet")
BASE_PATH_BALLOON = DATA_BASEPATH.balloon
all_ballon_files = glob.glob("./balloon/data/**/**.nc", recursive=True)
all_year_data = pd.read_parquet(f"{BASE_PATH_BALLOON}/ballon_data_lin.parquet")
all_ballon_files = glob.glob(f"{BASE_PATH_BALLOON}/**/**.nc", recursive=True)
def get_dataframe_between_year(start_year, end_year):
res = all_year_data

0
utils.py → balloon/utils.py
View File

5
cosmic/single.py
View File

@ -7,6 +7,8 @@ from scipy.optimize import curve_fit
import netCDF4 as nc
import matplotlib.pyplot as plt
from CONSTANT import DATA_BASEPATH
# 模块1: 数据读取与预处理
@ -377,8 +379,9 @@ def plot_results_mean_ktemp_Ptz(mean_ktemp_Ptz, heights):
class SingleCosmicWavePlot:
def __init__(self, year, day):
BASE_PATH_COSMIC = DATA_BASEPATH.cosmic
base_folder_path = f"./cosmic/data/{year}"
base_folder_path = f"./{BASE_PATH_COSMIC}/{year}"
day_num = day
# 模块1调用
final_df = read_and_preprocess_data(base_folder_path, day_num)

4
cosmic/temp_render.py
View File

@ -8,13 +8,15 @@ import matplotlib.pyplot as plt
# 定义拟合函数
# 解决绘图中中文不能显示的问题
import matplotlib
from CONSTANT import DATA_BASEPATH
# 设置中文显示和负号正常显示
matplotlib.rcParams['font.sans-serif'] = ['SimHei'] # 显示中文
matplotlib.rcParams['axes.unicode_minus'] = False # 正常显示负号
# 读取一年的数据文件
def temp_render(path:str = "./cosmic/cosmic.txt", T = 16):
def temp_render(path:str = f"{DATA_BASEPATH.cosmic}/cosmic.txt", T = 16):
def u_func(x, *params):
a1, b1, a2, b2, a3, b3, a4, b4, a5, b5, a6, b6, a7, b7, a8, b8, a9, b9 = params
return (

0
data/.gitkeep Normal file
View File

0
data/balloon/.gitkeep Normal file
View File

0
data/cosmic/.gitkeep Normal file
View File

0
data/radar/.gitkeep Normal file
View File

0
data/saber/.gitkeep Normal file
View File

0
data/tidi/.gitkeep Normal file
View File

11
file_stuff.py
View File

@ -1,11 +0,0 @@
import pandas as pd
all_year_data = pd.read_parquet("./cache/ballon_data_lin.parquet")
def get_dataframe_between_year(start_year, end_year):
res = all_year_data
filtered_res = res[(res['file_name'].str.extract(r'LIN-(\d{4})')[0].astype(int) >= start_year) &
(res['file_name'].str.extract(r'LIN-(\d{4})')[0].astype(int) <= end_year)]
return filtered_res

10
radar/__init__.py
View File

@ -3,11 +3,13 @@ import glob
from io import BytesIO
from quart import Blueprint, request, send_file
from matplotlib import pyplot as plt
from CONSTANT import DATA_BASEPATH
from radar.plot_original import final_render_v2
from radar.plot_prod import final_plot_v2
from quart.utils import run_sync
globed_all_files = glob.glob("./radar/data/**/**.txt", recursive=True)
BASE_PATH_RADAR = DATA_BASEPATH.radar
globed_all_files = glob.glob(f"{BASE_PATH_RADAR}/**/**.txt", recursive=True)
radar_module = Blueprint("Radar", __name__)
@ -39,7 +41,7 @@ async def render_v1():
model_name = request.args.get('model_name')
mode = request.args.get('mode')
renderer = final_render_v2(int(H), int(year), station, wind_type)
renderer = await run_sync(final_render_v2)(int(H), int(year), station, wind_type)
if mode=="day":
day = request.args.get('day')
renderer.render_day(day,model_name)
@ -78,7 +80,7 @@ async def render_v2():
month_range = (1, 12)
buffer = final_plot_v2(int(year), station, model_name, month_range)
buffer = await run_sync(final_plot_v2)(int(year), station, model_name, month_range)
buffer.seek(0)
return await send_file(buffer, mimetype='image/png')

8
radar/plot_original.py
View File

@ -17,6 +17,8 @@ from datetime import datetime
# 解决绘图中中文不能显示的问题
import matplotlib
from CONSTANT import DATA_BASEPATH
# 设置中文显示和负号正常显示
matplotlib.rcParams['font.sans-serif'] = ['SimHei'] # 显示中文
matplotlib.rcParams['axes.unicode_minus'] = False # 正常显示负号
@ -377,7 +379,8 @@ def get_final_df(H, year, station):
# # station = '黑龙江漠河站'
# 文件路径设置
file_dir = rf'./radar/data/{station}/{year}' # 数据文件路径
file_dir = rf'{DATA_BASEPATH.radar}/{station}/{year}' # 数据文件路径
# output_dir = rf'./out\{station}\{year}' # 参数txt、图片输出路径
# os.makedirs(output_dir, exist_ok=True) # 确保输出路径存在
@ -461,7 +464,8 @@ class final_render_v2():
@staticmethod
def get_all_pathes():
result = glob.glob(f"./radar/data/**/*.txt", recursive=True)
#
result = glob.glob(f"{DATA_BASEPATH.radar}/**/*.txt", recursive=True)
# normalize path
result = [os.path.normpath(path).replace("\\", "/") for path in result]
return result

4
radar/plot_prod.py
View File

@ -10,6 +10,8 @@ import seaborn as sns
# 解决绘图中中文不能显示的问题
import matplotlib
from CONSTANT import DATA_BASEPATH
matplotlib.rcParams['font.sans-serif'] = ['SimHei']
matplotlib.rcParams['axes.unicode_minus'] = False
@ -170,7 +172,7 @@ def final_plot_v2(year, station, model_name, month_range=(1,12)):
- model_name: str，模型名称（如 '潮汐波', '2日行星波', '5日行星波', '10日行星波', '16日行星波'）
"""
# 配置文件路径
file_dir = rf'./radar/data/{station}/{year}'
file_dir = rf'{DATA_BASEPATH.radar}/{station}/{year}'
# output_dir = rf'./radar/tmp\{station}\{year}\时空'
# os.makedirs(output_dir, exist_ok=True)

27
saber/__init__.py
View File

@ -2,6 +2,7 @@ import glob
from io import BytesIO
from quart import Blueprint, request, send_file
from matplotlib import pyplot as plt
from CONSTANT import DATA_BASEPATH
from saber.process import DataProcessor
from saber.render import Renderer
from saber.utils import *
@ -48,7 +49,7 @@ async def extract_payload():
return await send_file(buffer, mimetype="image/png")
_all_saber_files = glob.glob("./saber/data/**/**.nc", recursive=True)
_all_saber_files = glob.glob(f"{DATA_BASEPATH.saber}/**/**.nc", recursive=True)
@saber_module.route("/metadata")
@ -59,10 +60,10 @@ def get_files():
@saber_module.route("/metadata/list_days")
def get_days():
async def get_days():
path = request.args.get("path")
ncfile = data_nc_load(path)
ncfile = await run_sync(data_nc_load)(path)
return ncfile.date_time.tolist()
@ -73,8 +74,8 @@ async def do_plot_wave_day_fitting():
day = request.args.get("day")
height = request.args.get("height_no")
ncfile = data_nc_load(path)
processor = get_processer()
ncfile = await run_sync(data_nc_load)(path)
processor = await run_sync(get_processer)()
data = await run_sync(processor.process_day)(ncfile, int(day))
await run_sync(renderer.plot_wave_fitting)(data, int(height))
return await extract_payload()
@ -86,8 +87,8 @@ async def do_day_fft_ifft_plot():
day = request.args.get("day")
cycle_no = request.args.get("cycle_no")
ncfile = data_nc_load(path)
processor = get_processer()
ncfile = await run_sync(data_nc_load)(path)
processor = await run_sync(get_processer)()
data = await run_sync(processor.process_day)(ncfile, int(day))
# 高度滤波处理
await run_sync(renderer.day_fft_ifft_plot)(wave_data=data, cycle_no=int(cycle_no))
@ -100,8 +101,8 @@ async def do_day_cycle_power_wave_plot():
day = request.args.get("day")
cycle_no = request.args.get("cycle_no")
ncfile = data_nc_load(path)
processor = get_processer()
ncfile = await run_sync(data_nc_load)(path)
processor = await run_sync(get_processer)()
data = await run_sync(processor.process_day)(ncfile, int(day))
await run_sync(renderer.day_cycle_power_wave_plot)(wave_data=data, cycle_no=int(cycle_no))
return await extract_payload()
@ -112,8 +113,8 @@ async def do_month_power_wave_plot():
path = request.args.get("path")
month = request.args.get("month")
ncfile = data_nc_load(path)
processor = get_processer()
ncfile = await run_sync(data_nc_load)(path)
processor = await run_sync(get_processer)()
data =await run_sync(processor.process_month)(ncfile)
await run_sync(renderer.month_power_wave_plot)(wave_data=data, date_time=ncfile.date_time)
return await extract_payload()
@ -122,9 +123,9 @@ async def do_month_power_wave_plot():
@saber_module.route("/render/year_power_wave_plot")
async def do_year_power_wave_plot():
year = request.args.get("year")
processor = get_processer()
processor = await run_sync(get_processer)()
data = await run_sync(processor.process_year)([
data_nc_load(path) for path in glob.glob(f"./saber/data/{year}/*.nc")
data_nc_load(path) for path in glob.glob(f"{DATA_BASEPATH.saber}/{year}/*.nc")
])
await run_sync(renderer.year_power_wave_plot)(year_wave=data)
return await extract_payload()

5
saber/archive/saber_render.py
View File

@ -2,6 +2,7 @@ import numpy as np
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from CONSTANT import DATA_BASEPATH
from saber.utils import *
# from matplotlib.colors import LinearSegmentedColormap
# 设置字体为支持中文的字体
@ -694,7 +695,7 @@ def year_power_wave_plot(year, path, latitude_min, latitude_max, altitude_min, a
# 7 主程序，运行数据,并输出主要统计分析图 -
# -----------------------------------------------------------------------------------------------------------------------------
# 7-1 挑选某一天进行运行主要程序
file_path = "./SABER/data\\2016\\SABER_Temp_O3_April2016_v2.0.nc"
file_path = f"{DATA_BASEPATH.saber}\\2016\\SABER_Temp_O3_April2016_v2.0.nc"
# day_read=2018113,
day_read = 2016094
# 初始化某一天、某个纬度、高度范围等参数
@ -776,7 +777,7 @@ def filename_read(year, month_num, path):
year = 2018
path = "./saber/data"
path = f"{DATA_BASEPATH.saber}"
# 初始化某一天、某个纬度、高度范围等参数
latitude_min = 30.0
latitude_max = 40.0

3
saber/process.py
View File

@ -2,6 +2,7 @@ from dataclasses import dataclass
from datetime import datetime
from typing import Dict, List, Optional, Tuple
import numpy as np
from CONSTANT import DATA_BASEPATH
from saber.utils import *
# lat_range=(latitude_min, latitude_max),
@ -224,7 +225,7 @@ class DataProcessor:
if __name__ == "__main__":
year = 2018
path = "./saber/data/2012/SABER_Temp_O3_April2012_v2.0.nc"
path = f"{DATA_BASEPATH.saber}/2012/SABER_Temp_O3_April2012_v2.0.nc"
# 初始化某一天、某个纬度、高度范围等参数
latitude_min = 30.0
latitude_max = 40.0

3
tidi/__init__.py
View File

@ -4,6 +4,7 @@ from quart import Blueprint, request, send_file
from quart.utils import run_sync
from matplotlib import pyplot as plt
from CONSTANT import DATA_BASEPATH
from tidi.plot import TidiPlotv2
from tidi.staged.plot import tidi_render
@ -12,7 +13,7 @@ tidi_module = Blueprint("Tidi", __name__)
@tidi_module.route('/metadata')
def get_all_years():
res = glob.glob("./tidi/data/**/**.txt", recursive=True)
res = glob.glob(f"{DATA_BASEPATH.tidi}/**/**.txt", recursive=True)
# search for the folder name that is year
return {

45
tidi/plot.py
View File

@ -5,6 +5,8 @@ from scipy.io import loadmat
from scipy.optimize import curve_fit
import matplotlib.pyplot as plt
import seaborn as sns
from CONSTANT import DATA_BASEPATH
# ---------------------------------------------------------------------------------------
# -----vzonal----------------------------------------------------------------------------
@ -12,12 +14,13 @@ import seaborn as sns
def process_vzonal_day(day, year=2015):
try:
# 读取数 据
height_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Height.mat")
lat_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Lat.mat")
lon_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Lon.mat")
base_path = DATA_BASEPATH.tidi
height_data = loadmat(rf"{base_path}/{year}/{day:03d}_Height.mat")
lat_data = loadmat(rf"{base_path}/{year}/{day:03d}_Lat.mat")
lon_data = loadmat(rf"{base_path}/{year}/{day:03d}_Lon.mat")
vmeridional_data = loadmat(
rf"./tidi/data/{year}/{day:03d}_VMerdional.mat")
vzonal_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Vzonal.mat")
rf"{base_path}/{year}/{day:03d}_VMerdional.mat")
vzonal_data = loadmat(rf"{base_path}/{year}/{day:03d}_Vzonal.mat")
# 将数据转换为DataFrame
height_df = pd.DataFrame(height_data['Height'])
@ -380,12 +383,14 @@ def process_vzonal_day(day, year=2015):
def process_vmeridional_day(day, year=2015):
try:
# 读取数据
height_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Height.mat")
lat_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Lat.mat")
lon_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Lon.mat")
base_path = DATA_BASEPATH.tidi
height_data = loadmat(rf"{base_path}/{year}/{day:03d}_Height.mat")
lat_data = loadmat(rf"{base_path}/{year}/{day:03d}_Lat.mat")
lon_data = loadmat(rf"{base_path}/{year}/{day:03d}_Lon.mat")
vmeridional_data = loadmat(
rf"./tidi/data/{year}/{day:03d}_VMerdional.mat")
vzonal_data = loadmat(rf"./tidi/data/{year}/{day:03d}_Vzonal.mat")
rf"{base_path}/{year}/{day:03d}_VMerdional.mat")
vzonal_data = loadmat(rf"{base_path}/{year}/{day:03d}_Vzonal.mat")
# 将数据转换为DataFrame
height_df = pd.DataFrame(height_data['Height'])
@ -755,11 +760,11 @@ def day_to_month(day):
class TidiPlotv2:
def __init__(self, year):
self.year = year
if os.path.exists(f"./tidi/cache/{year}/all_days_vzonal_results.parquet") \
and os.path.exists(f"./tidi/cache/{year}/all_days_vmeridional_results.parquet"):
all_days_vzonal_results = pd.read_parquet(f"./tidi/cache/{year}/all_days_vzonal_results.parquet")
all_days_vmeridional_results = pd.read_parquet(f"./tidi/cache/{year}/all_days_vmeridional_results.parquet")
cache_path = f"{DATA_BASEPATH.tidi}/cache"
if os.path.exists(f"{cache_path}/{year}/all_days_vzonal_results.parquet") \
and os.path.exists(f"{cache_path}/{year}/all_days_vmeridional_results.parquet"):
all_days_vzonal_results = pd.read_parquet(f"{cache_path}/{year}/all_days_vzonal_results.parquet")
all_days_vmeridional_results = pd.read_parquet(f"{cache_path}/{year}/all_days_vmeridional_results.parquet")
else:
all_days_vzonal_results = pd.DataFrame()
@ -784,11 +789,11 @@ class TidiPlotv2:
# cache the results
# if dir not exists, create it
if not os.path.exists(f"./tidi/cache/{year}"):
os.makedirs(f"./tidi/cache/{year}")
if not os.path.exists(f"{cache_path}/{year}"):
os.makedirs(f"{cache_path}/{year}")
all_days_vzonal_results.to_parquet(f"./tidi/cache/{year}/all_days_vzonal_results.parquet")
all_days_vmeridional_results.to_parquet(f"./tidi/cache/{year}/all_days_vmeridional_results.parquet")
all_days_vzonal_results.to_parquet(f"{cache_path}/{year}/all_days_vzonal_results.parquet")
all_days_vmeridional_results.to_parquet(f"{cache_path}/{year}/all_days_vmeridional_results.parquet")
self.all_days_vzonal_results = all_days_vzonal_results
self.all_days_vmeridional_results = all_days_vmeridional_results
@ -837,7 +842,7 @@ class TidiPlotv2:
months = self.months
'''绘图'''
plt.clf()
plt.rcParams['font.family'] = 'SimSun' # 宋体
plt.rcParams['font.family'] = 'SimHei' # 宋体
plt.rcParams['font.size'] = 12 # 中文字号
plt.rcParams['axes.unicode_minus'] = False # 正确显示负号
plt.rcParams['font.sans-serif'] = 'Times New Roman' # 新罗马

4
tidi/staged/plot.py
View File

@ -10,6 +10,8 @@ import matplotlib.pyplot as plt
# 解决绘图中中文不能显示的问题
import matplotlib
from CONSTANT import DATA_BASEPATH
# 设置中文显示和负号正常显示
matplotlib.rcParams['font.sans-serif'] = ['SimHei'] # 显示中文
matplotlib.rcParams['axes.unicode_minus'] = False # 正常显示负号
@ -38,7 +40,7 @@ def tidi_render(mode, year, k, T=10):
raise ValueError('mode must be V_Zonal or V_Meridional')
if k not in list(range(-4, 5)):
raise ValueError('k must be in range(-4, 5)')
df = pd.read_csv(f'./tidi/data/{year}/TIDI_{mode}_data.txt', sep='\s+')
df = pd.read_csv(f'{DATA_BASEPATH.tidi}/{year}/TIDI_{mode}_data.txt', sep='\s+')
# 删除有 NaN 的行
# V_Meridional
df = df.dropna(subset=[mode])

2
tidi/staged/plot_old.py
View File

@ -806,7 +806,7 @@ months = [day_to_month(day) for day in dates]
'''绘图'''
plt.rcParams['font.family'] = 'SimSun' # 宋体
plt.rcParams['font.family'] = 'SimHei' # 宋体
plt.rcParams['font.size'] = 12 # 中文字号
plt.rcParams['axes.unicode_minus'] = False # 正确显示负号
plt.rcParams['font.sans-serif'] = 'Times New Roman' # 新罗马

4
tidi/staged/process.py
View File

@ -3,6 +3,8 @@ import numpy as np
from scipy.io import loadmat
import pandas as pd
from CONSTANT import DATA_BASEPATH
#第156行代码，纬度可以自己选择，但是都是每5°做一个选择
#70km-120km每2.5km一个，共21个，第六个就是82.5km高度的情况，高度也可以自己选择
@ -15,7 +17,7 @@ vmeridional_list = []
vzonal_list = []
# 文件路径
base_path = "./tidi/data/2022/"
base_path = f"{DATA_BASEPATH.tidi}/2022/"
# 循环读取从第1天到第365天的数据
for day in range(1, 366): # 365天数据，确保循环次数为365