deal server data

desktop
esea_info 3 years ago
parent ec21261957
commit d0008a65da
  1. 1
      .gitignore
  2. 102
      awrams.py
  3. 12
      yiwinframe.py

1
.gitignore vendored

@ -4,3 +4,4 @@
/*/__pycache__ /*/__pycache__
*.log *.log
/data/output/ /data/output/
/server/

@ -16,7 +16,7 @@ import struct
import numpy as np import numpy as np
from pubsub import pub from pubsub import pub
from pathlib import PurePath,Path from pathlib import PurePath,Path
from myconfig import CURRENT_DIR,DATA_DIR,OUTPUT_DIR,NEWLINE,ROWFACTOR,SAVE_EXT_NAME,TOKEN from myconfig import CURRENT_DIR,DATA_DIR,OUTPUT_DIR,SERVER_DIR,NEWLINE,ROWFACTOR,SAVE_EXT_NAME,TOKEN
from myconfig import DeviceType,RamsesSURFACE,RamsesAWRAMS,RamsesPROFILE from myconfig import DeviceType,RamsesSURFACE,RamsesAWRAMS,RamsesPROFILE
from tools.mylogger import log from tools.mylogger import log
from tools.myexception import MyException from tools.myexception import MyException
@ -53,7 +53,7 @@ class HandHeldBuf:
return ret return ret
def write_buf(self, buf: bytes) -> None: def write_buf(self, buf: bytes) -> None:
id = self.id # id = self.id
len = buf.__len__() len = buf.__len__()
# logging.info(f'Received ID:{id} Size:{len}') # logging.info(f'Received ID:{id} Size:{len}')
self.__buf = self.__buf+buf self.__buf = self.__buf+buf
@ -294,9 +294,11 @@ class HandHeldPath(object):
获得成对的info sensor 文件 获得成对的info sensor 文件
[目录名文件名年月日时间] [目录名文件名年月日时间]
''' '''
log.info(f" === ", self.server_path, "getDataFileList_SERVER")
self.filelist = [] self.filelist = []
fs = self.server_path.glob( "*/*/*/*/info.bin" ) fs = self.server_path.glob( "*/*/*/*/*/info.bin" )
for f in fs: for f in fs:
# print(f)
error_file = {} error_file = {}
if f.stat().st_size==0: if f.stat().st_size==0:
error_file.update( {"path": f } ) error_file.update( {"path": f } )
@ -307,59 +309,20 @@ class HandHeldPath(object):
## 获得全部bin文件 (info.bin (num).bin),不含 pic.bin ## 获得全部bin文件 (info.bin (num).bin),不含 pic.bin
current_path = f.parent current_path = f.parent
bin_files = current_path.glob('*.bin') bin_files = current_path.glob('*.bin')
bin_len = len(bin_files)
res_files = [] res_files = []
res_files.append( f)
# 检查(num).bin 是否有遗漏 # 检查(num).bin 是否有遗漏
for i in range(bin_len-1): for bl in bin_files:
tmp_fpath = current_path.joinpath(str(i)+".bin") if bl.stem.isdigit():
if tmp_fpath.exists(): res_files.append(bl)
res_files.append( tmp_fpath ) pass
else:
res_files =[]
break
if res_files != []: if res_files != []:
res_files.append(f)
self.filelist.append(res_files) self.filelist.append(res_files)
pass pass
# def getDataFileList_SD(self, mode:int = 0 ):
# '''
# 获得成对的info sensor 文件
# [目录名,文件名,年月日,时间]
# '''
# # ret = []
# fs = self.data_path.glob( "*/*/info/*" )
# for f in fs:
# error_file = {}
# if f.stat().st_size==0:
# error_file.update( {"path": f.name } )
# error_file.update( {"error":"info file size is zero"} )
# self.error_result.append(error_file)
# continue
# self.info_path_fname = f
# sensor_path = self.getSensorPathFromInfoPath( f )
# # sensor_path = Path(sensor_purepath)
# # sensor 文件不存在
# if not sensor_path.exists():
# error_file.update( { "path" : f } )
# error_file.update( {"error":"cannot find the sensor file "} )
# self.error_result.append(error_file)
# continue
# # sensor文件大小为0
# if sensor_path.stat().st_size==0:
# error_file.update( { "path" : f } )
# error_file.update( {"error" : "the file size of the sensor is zero' "} )
# self.error_result.append(error_file)
# continue
# self.setFilelist(f,sensor_path,mode=mode)
# # print(self.filelist)
# pass
def setFilelist(self, info_path:Path, sensor_path:Path): def setFilelist(self, info_path:Path, sensor_path:Path):
temp = {} temp = {}
temp.update( {"info_path" : info_path } ) temp.update( {"info_path" : info_path } )
@ -476,6 +439,7 @@ class AWRAMS(object):
''' '''
log.info(f" 服务器数据下载,二次处理", __name__, "getDataFiles_server", ) log.info(f" 服务器数据下载,二次处理", __name__, "getDataFiles_server", )
self.hhp.setMode( 1 ) self.hhp.setMode( 1 )
self.hhp.setServerPath( SERVER_DIR )
self.hhp.filelist = [] self.hhp.filelist = []
self.hhp.getDataFileList_SERVER( ) self.hhp.getDataFileList_SERVER( )
self.filelist = self.hhp.getFilelist() self.filelist = self.hhp.getFilelist()
@ -536,18 +500,44 @@ class AWRAMS(object):
raise MyException( "处理文件"+ self.info_path_fname + "出现错误") raise MyException( "处理文件"+ self.info_path_fname + "出现错误")
pass pass
# 0.bin 1.bin ... 多个文件
for i in range(len(df)-1): for i in range(len(df)-1):
fpath:Path = df[i]
stem = fpath.stem
if stem.isdigit( ) :
tmp_buf = fpath.read_bytes()
self.hhb.write_buf( tmp_buf )
pass pass
try: # awrams每次只有一组数据
self.dealOneMeasurement( self.sensor_path_fname )
# self.measurement_interval = int(self.info_dict['Measure_Interval'])
# self.measurement_repeat = int(self.info_dict['Measure_Repeat'])
# self.dealOneHandheldMeasurement(self.sensor_path_fname )
except Exception as e:
log.error( "处理传感器文件" + self.sensor_path_fname.name + " 出现错误",__name__,"dealAllMeasurements")
raise MyException( "处理传感器文件" + self.sensor_path_fname.name + " 出现错误" )
pass pass
# 将buf 处理成光强值 self.intensity_before_avg
self.decode_sensor_buf()
# 获得真实波长 self.real_wavelength
self.getRealWavelenthDict()
# 多组数据平均 self.intensity_after_avg
self.getAvg( self.intensity_before_avg )
# 波长插值 self.res esky lsky lw
self.__do_sensor_dict_interpo()
# 获得输出目录路径 self.ymdhms
self.get_ymdhms()
# get Lw Rs
self.getLwRs()
# 保存 data info
self.saveOnefileForLskyEskyLwaterLwRS()
# 通知ui
self.__set_msg("data", {"tm":self.ymdhms, "res":self.res} )
pub.sendMessage("update" , msg=self.msg)
# try: # awrams每次只有一组数据
# self.dealOneMeasurement( self.sensor_path_fname )
# # self.measurement_interval = int(self.info_dict['Measure_Interval'])
# # self.measurement_repeat = int(self.info_dict['Measure_Repeat'])
# # self.dealOneHandheldMeasurement(self.sensor_path_fname )
# except Exception as e:
# log.error( "处理传感器文件" + self.sensor_path_fname.name + " 出现错误",__name__,"dealAllMeasurements")
# raise MyException( "处理传感器文件" + self.sensor_path_fname.name + " 出现错误" )
# pass
self.__set_msg('notice', '处理文件完成') self.__set_msg('notice', '处理文件完成')
pub.sendMessage('update' , msg=self.msg) pub.sendMessage('update' , msg=self.msg)
log.info(f"Finished !! ", __name__, "dealAllMeasurements") log.info(f"Finished !! ", __name__, "dealAllMeasurements")

@ -251,14 +251,6 @@ class YiwinFrame(wx.Frame):
pass pass
# if self.awrams.dealAllMeasurements( ):
# self.onNotify("正在处理数据" )
# self.statusBar.SetStatusText(u" 数据处理完成......", 0)
# self.onNotify("处理数据已经结束" )
pass
def OnDealServer(self, e)-> None: def OnDealServer(self, e)-> None:
'''对服务器数据进行再次处理,''' '''对服务器数据进行再次处理,'''
log.info(f"OnDealServer: 处理服务器数据", __name__, "", "") log.info(f"OnDealServer: 处理服务器数据", __name__, "", "")
@ -268,7 +260,7 @@ class YiwinFrame(wx.Frame):
self.statusBar.SetStatusText("Pls get the cal info, and try again",0) self.statusBar.SetStatusText("Pls get the cal info, and try again",0)
pass pass
self.awrams.getDataFileList() self.awrams.getDataFiles_server( )
self.awrams.setDeviceID(self.device_id) self.awrams.setDeviceID(self.device_id)
self.awrams.setRetrieve(self.retrieve) self.awrams.setRetrieve(self.retrieve)
if self.syscfg != {}: if self.syscfg != {}:
@ -281,7 +273,7 @@ class YiwinFrame(wx.Frame):
# 处理数据 # 处理数据
self.alterStatus_0(" 正在处理数据...." ) self.alterStatus_0(" 正在处理数据...." )
try: try:
self.mythread.set_task( self.awrams.dealAllMeasurements ) self.mythread.set_task( self.awrams.dealAllMeasurements_Server )
self.mythread.start() self.mythread.start()
except Exception as e: except Exception as e:
log.error(e) log.error(e)

Loading…
Cancel
Save