From 1a4066f8e18605289ac0fc343e320ccd0d050db2 Mon Sep 17 00:00:00 2001 From: esea_info Date: Wed, 12 Apr 2023 18:34:16 +0800 Subject: [PATCH] first commit --- .gitignore | 3 +- Ramses.py | 200 ++++++ app.py | 115 ++++ atest.py | 132 ++++ calfile/awrams/Back_SAM_852F.dat | 297 +++++++++ calfile/awrams/Back_SAM_859F.dat | 297 +++++++++ calfile/awrams/Back_SAM_85B5.dat | 297 +++++++++ calfile/awrams/CalAQ_SAM_852F.dat | 297 +++++++++ calfile/awrams/CalAQ_SAM_859F.dat | 297 +++++++++ calfile/awrams/CalAQ_SAM_85B5.dat | 297 +++++++++ calfile/awrams/Cal_SAM_852F.dat | 297 +++++++++ calfile/awrams/Cal_SAM_859F.dat | 297 +++++++++ calfile/awrams/Cal_SAM_85B5.dat | 297 +++++++++ calfile/awrams/SAMIP_50ED_ALL.ini | 81 +++ calfile/awrams/SAM_852F.ini | 32 + calfile/awrams/SAM_859F.ini | 30 + calfile/awrams/SAM_85B5.ini | 30 + config.yml | 20 + configuration.py | 237 +++++++ handheld.py | 991 ++++++++++++++++++++++++++++ myconfig.py | 322 +++++++++ readcal.py | 1008 +++++++++++++++++++++++++++++ receive.py | 466 +++++++++++++ retrieve.yml | 4 + tools/__init__.py | 0 tools/myexception.py | 11 + tools/mylogger.py | 285 ++++++++ tools/mypath.py | 265 ++++++++ tools/mytime.py | 66 ++ 29 files changed, 6970 insertions(+), 1 deletion(-) create mode 100644 Ramses.py create mode 100644 app.py create mode 100644 atest.py create mode 100644 calfile/awrams/Back_SAM_852F.dat create mode 100644 calfile/awrams/Back_SAM_859F.dat create mode 100644 calfile/awrams/Back_SAM_85B5.dat create mode 100644 calfile/awrams/CalAQ_SAM_852F.dat create mode 100644 calfile/awrams/CalAQ_SAM_859F.dat create mode 100644 calfile/awrams/CalAQ_SAM_85B5.dat create mode 100644 calfile/awrams/Cal_SAM_852F.dat create mode 100644 calfile/awrams/Cal_SAM_859F.dat create mode 100644 calfile/awrams/Cal_SAM_85B5.dat create mode 100644 calfile/awrams/SAMIP_50ED_ALL.ini create mode 100644 calfile/awrams/SAM_852F.ini create mode 100644 calfile/awrams/SAM_859F.ini create mode 100644 calfile/awrams/SAM_85B5.ini create mode 100644 config.yml create mode 100644 configuration.py create mode 100644 handheld.py create mode 100644 myconfig.py create mode 100644 readcal.py create mode 100644 receive.py create mode 100644 retrieve.yml create mode 100644 tools/__init__.py create mode 100644 tools/myexception.py create mode 100644 tools/mylogger.py create mode 100644 tools/mypath.py create mode 100644 tools/mytime.py diff --git a/.gitignore b/.gitignore index b315aa5..d536aaa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /__pycache__/ /dist/ /build/ -/*/__pycache__ +/*/__pycache__/ +/data/output/ *.log \ No newline at end of file diff --git a/Ramses.py b/Ramses.py new file mode 100644 index 0000000..a5429af --- /dev/null +++ b/Ramses.py @@ -0,0 +1,200 @@ +#! python3 +# -*- encoding: utf-8 -*- +''' +@File : Ramses.py +@Time : 2023/03/03 11:09:50 +@Author : Jim @ Yiwin +@Version : 1.0 +@Contact : jim@yi-win.com +@Desc : +@para : 23 ..07 .... 06 05 04 03 02 01 00 + ip信息不包含 +''' + +import struct +import numpy as np +from pathlib import Path +from tools.mylogger import log +from myconfig import RamsesAWRAMS, RamsesSURFACE, RamsesPROFILE, DeviceType + + +class Ramses(object): + + def __init__(self,): + """ + @description :处理Ramses的数据标定 Hex -- realWavelength Intensity + @param : 23 ..07 .... 06 05 04 03 02 01 00 + ip信息不包含 + @Returns : realWavelength Intensity + """ + self.buf = b'' + self.it = None + self.light_int = None # 未标定的整数值 + self.spectrum = None # 光谱强度 + # self.current_buf = "" + # self.current_buf_seq = 0 + # self.current_it_int = {"it": 0, "light_int": []} # 积分时间及换算的整数值 + # self.res = {"wavelength": [], "light": []} + self.cal_cfg = {} + # self.current_cal = {} # 当前传感器的序列号 + pass + + def setBuf(self, buf: bytes): + self.buf = buf + pass + + def setCalCfg(self, d: dict): + self.cal_cfg = d + pass + + def getRealWavelength(self, d: dict): + self.cal_cfg = d + pass + + def getSpectrum(self): + return self.spectrum + + def resetPara(self, ): + self.buf = b'' + self.it = None + self.light_int = None + self.spectrum = None # 光谱强度 + self.cal_cfg = {} + pass + + def resetItSpectrum(self, ): + self.it = None + self.spectrum = None # 光谱强度 + pass + + def printPara(self, ): + print(f"**************Ramses printPara*******************") + print(f"{self.buf}") + print(f"{self.cal_cfg}") + print(f"{self.it}") + print(f"{self.light_int}") + print(f"{self.spectrum}") + print(f"**************Ramses printPara*******************") + pass + + def dealBuf(self, ip_included:bool=False): + """多个传感器的数据处理， 头部是否包含Ip帧的信息""" + log.info(f" dealBuf ", __name__) + + res = {} + len_ = len(self.buf) + if len_ < 576: + return + if ip_included: + self.buf = self.buf[26:] + len_ = len_ - 26 + if len_ % 576 != 0: + return + + for i in range(int(len_/576)): + res.update({i+1: {}}) + temp_buf = self.buf[7:71] + self.buf[79:143] + \ + self.buf[151:215] + self.buf[223:287] + \ + self.buf[295:359] + self.buf[367:431] + \ + self.buf[439:503] + self.buf[511:575] + self.ConvertAndCalibrate( temp_buf ) + # print(len(temp_buf)) + temp = self.__ConvertBytesToInt(temp_buf) + res.update( { i+1: temp } ) + # print(res) + pass + + def ConvertAndCalibrate(self,) -> None: + log.debug(f" ConvertAndCalibrate ", __name__) + temp = self.__ConvertBytesToInt( ) + self.__CalibrateSpectrumData( ) + pass + + # 转换一个传感器的部分 + def __ConvertBytesToInt(self ) -> None: + res = {} + d = [] # List [ Tuple[ it:int, sing_set:tuple[int] ] ] + self.it = 2 << int(self.buf[1]) # integrated time + self.light_int = struct.unpack( + " None: + log.info(f"ConfigAWRAMS init: ", __name__, "", "") + self.device_type = None + self.configuration =None + self.cal_configuration = {} + pass + + + + def setDeviceType(self, device_type:str) -> None: + self.device_type = device_type.lower() + pass + + def setSystemCfgDict(self, cfg:dict) -> None: + self.configuration = cfg + log.info(f"self.configuration : {self.configuration} ", __name__, "", "") + pass + + def getCalConfiguration(self) -> None: + if self.device_type == None: + self.cal_configuration = None + if self.configuration == None: + self.cal_configuration =None + + for k,v in self.configuration.items(): + + if v["SN"] == "" or v['FUNC']=="": + pass + else: + self.cal_configuration.update( {v["FUNC"]:{}} ) + self.cal_configuration[v["FUNC"]].update( {"SN":v['SN']} ) + self.cal_configuration[v["FUNC"]].update( {"FUNC":v['FUNC']} ) + + self.__init_configuration_basic() + self.__init_configuration_cal() + self.__init_configuration_IP_SAM() + pass + + def __init_configuration_basic(self ) -> None: + # self.cfgtool = Config() + for k in self.cal_configuration.keys(): + sn = self.cal_configuration[k]["SN"] + if self.__isSamIniExisted(sn): + self.cal_configuration[k].update({ "TYPE" : "SAM" }) + self.cal_configuration[k].update({ "samsn" : sn }) + self.cal_configuration[k].update({ "inifile" : "SAM_"+sn+".ini" }) + self.cal_configuration[k].update({ "calfile" : "Cal_SAM_"+sn+".dat" }) + self.cal_configuration[k].update({ "calaqfile" : "CalAQ_SAM_"+sn+".dat" }) + self.cal_configuration[k].update({ "backfile" : "Back_SAM_"+sn+".dat" }) + if self.__isSamIPIniExisted(sn): + self.cal_configuration[k].update({ "TYPE" : "SAMIP" }) + samsn = self.__getSAMSN(sn) + if samsn== None: + log.warning(f"Cannot get samsn from Sensor: {sn}", __name__, "", "" ) + raise Exception(f"Cannot get samsn from Sensor: {sn}") + self.cal_configuration[k].update({ "samsn" : samsn }) + self.cal_configuration[k].update({ "inifile" : "SAMIP_"+sn+"_ALL.ini" }) + self.cal_configuration[k].update({ "calfile" : "Cal_SAM_"+samsn+".dat" }) + self.cal_configuration[k].update({ "calaqfile" : "CalAQ_SAM_"+samsn+".dat" }) + self.cal_configuration[k].update({ "backfile" : "Back_SAM_"+samsn+".dat" }) + if not self.__isSamIniExisted(sn) and not self.__isSamIPIniExisted(sn): + log.warning(f"Cannot find ini file for Sensor: {sn}", __name__, "", "" ) + raise Exception(f"Cannot find ini file for Sensor: {sn}") + + pass + + def __init_configuration_cal(self ) -> None: + # self.cfgtool = Config() + for k in self.cal_configuration.keys(): + sn = self.cal_configuration[k]["SN"] + # Device File + calpath = CAL_DIR.joinpath(self.device_type, self.cal_configuration[k]["calfile"]) + if calpath.exists( ): + res = ReadCal.read_columns_set_by_mark( calpath, FILE_MARK, 1 ) + self.cal_configuration[k].update({ "cal" : res[1][0] }) + calaqpath = CAL_DIR.joinpath(self.device_type, self.cal_configuration[k]["calaqfile"]) + if calaqpath.exists( ): + res = ReadCal.read_columns_set_by_mark( calaqpath, FILE_MARK, 1 ) + self.cal_configuration[k].update({ "calaq" : res[1][0] }) + backpath = CAL_DIR.joinpath(self.device_type, self.cal_configuration[k]["backfile"]) + if calaqpath.exists( ): + res = ReadCal.read_columns_set_by_mark( backpath, FILE_MARK, 1,2 ) + self.cal_configuration[k].update({ "b0" : res[1][0] }) + self.cal_configuration[k].update({ "b1" : res[1][1] }) + pass + + + + def __init_configuration_IP_SAM(self ) -> None: + # self.cfgtool = Config() + for j in self.cal_configuration.keys(): + # log.debug(f"__init_configuration_IP_SAM {j}", __name__, "", "" ) + inipath = CAL_DIR.joinpath(self.device_type, self.cal_configuration[j]["inifile"]) + # log.debug(f"__init_configuration_IP_SAM {inipath}", __name__, "", "" ) + sam = ReadCal.readSAMCalFromIni(inipath) + # log.debug(f"__init_configuration_IP_SAM {sam}", __name__, "", "" ) + for k,v in sam.items(): + self.cal_configuration[j].update({ k : v }) + if self.cal_configuration[j]["TYPE"] == "SAMIP": + ip = ReadCal.readIPCalFromIni(inipath) + for k,v in ip.items(): + self.cal_configuration[j].update({ k : v }) + + def __isSamIniExisted(self,sn) ->bool: + sn_0 = "SAM_"+str(sn)+".ini" + path_ = CAL_DIR.joinpath(self.device_type.lower(), sn_0) + if path_.exists(): + return True + return False + + + def __isSamIPIniExisted(self,sn) ->bool: + sn_0 = "SAMIP_"+str(sn)+"_ALL.ini" + path_ = CAL_DIR.joinpath(self.device_type.lower(), sn_0) + if path_.exists(): + return True + return False + + def __getSAMSN(self,sn) -> None: + sn_0 = "SAMIP_"+str(sn)+"_ALL.ini" + path_ = CAL_DIR.joinpath(self.device_type.lower(), sn_0) + # path_ = DATA_DIR.joinpath(self.device.lower(), CAL_DIR, sn_0) + samsn = ReadCal.readSamSNFromIni( path_ ) + if samsn == None: + return None + return samsn + pass + + # def __init2__(self, device:str, **kwargs) -> None: + # """ + # get cal parameter for every sensor + # para : {"1":{"SN":"85B5","FUNC","Lsky"},"2":{},"3":{}} + # """ + # # log.info(f"ProcessAWRAMS kwargs: {kwargs}", __name__, "", "") + # # log.info(f"len: { len(kwargs)}", __name__, "", "") + + # if len(kwargs) != 3: + # log.warning(f" pass a wrong para to ProcessAWRAMS {kwargs}", __name__, "", "") + # self.device = device.lower() # surface profile awrams + # self.ramses = {} + + # # 生成标定文件 { } + # for k,v in kwargs.items(): + # self.ramses.update( {v["FUNC"]:{}} ) + # self.ramses[v["FUNC"]].update( {"SN":v['SN']} ) + # self.ramses[v["FUNC"]].update( {"FUNC":v['FUNC']} ) + # pass + # log.debug(f" ===== {self.ramses}",__name__, "", "" ) + + + # # if kwargs.__contains__("1"): + # # self.ramses.append( self.cfgtool.getDictByAttr("ramses")) + # # self.cfgtool.set_attr(self.ramses[1],kwargs['1']"SN",kwargs['1']) + # # if kwargs.__contains__("2"): + # # self.ramses.append( self.cfgtool.getDictByAttr("ramses")) + # # self.cfgtool.set_attr(self.ramses[2],"SN",kwargs['1']) + # # if kwargs.__contains__("3"): + # # self.ramses.append( self.cfgtool.getDictByAttr("ramses")) + # # self.cfgtool.set_attr(self.ramses[3],"SN",kwargs['1']) + + # self.__init_configuration_basic() + # self.__init_configuration_cal() + # self.__init_configuration_IP_SAM() + + # # log.info(f"ProcessAWRAMS after initiate: {kwargs}", __name__, "", "") + + # def __init_configuration_basic2(self ) -> None: + # # self.cfgtool = Config() + # for k in self.ramses.keys(): + # sn = self.ramses[k]["SN"] + # if self.__isSamIniExisted(sn): + # self.ramses[k].update({ "TYPE" : "SAM" }) + # self.ramses[k].update({ "samsn" : sn }) + # self.ramses[k].update({ "inifile" : "SAM_"+sn+".ini" }) + # self.ramses[k].update({ "calfile" : "Cal_SAM_"+sn+".dat" }) + # self.ramses[k].update({ "calaqfile" : "CalAQ_SAM_"+sn+".dat" }) + # self.ramses[k].update({ "backfile" : "Back_SAM_"+sn+".dat" }) + # if self.__isSamIPIniExisted(sn): + # self.ramses[k].update({ "TYPE" : "SAMIP" }) + # samsn = self.__getSAMSN(sn) + # if samsn== None: + # log.warning(f"Cannot get samsn from Sensor: {sn}", __name__, "", "" ) + # raise Exception(f"Cannot get samsn from Sensor: {sn}") + # self.ramses[k].update({ "samsn" : samsn }) + # self.ramses[k].update({ "inifile" : "SAMIP_"+sn+"_ALL.ini" }) + # self.ramses[k].update({ "calfile" : "Cal_SAM_"+samsn+".dat" }) + # self.ramses[k].update({ "calaqfile" : "CalAQ_SAM_"+samsn+".dat" }) + # self.ramses[k].update({ "backfile" : "Back_SAM_"+samsn+".dat" }) + # if not self.__isSamIniExisted(sn) and not self.__isSamIPIniExisted(sn): + # log.warning(f"Cannot find ini file for Sensor: {sn}", __name__, "", "" ) + # raise Exception(f"Cannot find ini file for Sensor: {sn}") + + # pass + + # def __init_configuration_cal2(self ) -> None: + # # self.cfgtool = Config() + # for k in self.ramses.keys(): + # sn = self.ramses[k]["SN"] + # # Device File + # calpath = CAL_DIR.joinpath(self.device, self.ramses[k]["calfile"]) + # if calpath.exists( ): + # res = Readfile.read_columns_set_by_mark( calpath, FILE_MARK, 1 ) + # self.ramses[k].update({ "cal" : res[1][0] }) + # calaqpath = CAL_DIR.joinpath(self.device, self.ramses[k]["calaqfile"]) + # if calaqpath.exists( ): + # res = Readfile.read_columns_set_by_mark( calaqpath, FILE_MARK, 1 ) + # self.ramses[k].update({ "calaq" : res[1][0] }) + # backpath = CAL_DIR.joinpath(self.device, self.ramses[k]["backfile"]) + # if calaqpath.exists( ): + # res = Readfile.read_columns_set_by_mark( backpath, FILE_MARK, 1,2 ) + # self.ramses[k].update({ "b0" : res[1][0] }) + # self.ramses[k].update({ "b1" : res[1][1] }) + # pass + + # def __init_configuration_IP_SAM2(self ) -> None: + # # self.cfgtool = Config() + # for j in self.ramses.keys(): + # # log.debug(f"__init_configuration_IP_SAM {j}", __name__, "", "" ) + # inipath = CAL_DIR.joinpath(self.device, self.ramses[j]["inifile"]) + # # log.debug(f"__init_configuration_IP_SAM {inipath}", __name__, "", "" ) + # sam = Readfile.readSAMCalFromIni(inipath) + # # log.debug(f"__init_configuration_IP_SAM {sam}", __name__, "", "" ) + # for k,v in sam.items(): + # self.ramses[j].update({ k : v }) + # if self.ramses[j]["TYPE"] == "SAMIP": + # ip = Readfile.readIPCalFromIni(inipath) + # for k,v in ip.items(): + # self.ramses[j].update({ k : v }) \ No newline at end of file diff --git a/handheld.py b/handheld.py new file mode 100644 index 0000000..88b154d --- /dev/null +++ b/handheld.py @@ -0,0 +1,991 @@ +#! python3 +# -*- encoding: utf-8 -*- +''' +@File : handheld.py +@Time : 2023/02/24 17:20:59 +@Author : Jim @ Yiwin +@Version : 1.0 +@Contact : jim@yi-win.com +''' + +CUR_TIME_STR_FMT = "%Y-%m-%d %H:%M:%S" + +import time +import locale +import struct +import numpy as np +from pathlib import PurePath,Path +from myconfig import CURRENT_DIR,DATA_DIR,OUTPUT_DIR,NEWLINE,ROWFACTOR,SAVE_EXT_NAME,TOKEN +from myconfig import DeviceType,RamsesSURFACE,RamsesAWRAMS,RamsesPROFILE +from tools.mylogger import log +from tools.myexception import MyException +from tools.mytime import MyTime +from tools.mypath import MyDir +from Ramses import Ramses + +class HandHeldBuf: + def __init__(self,) -> None: + self.__buf = b'' + self.__head = {} + self.__begin_sign = b'\x23' + self.__end_sign = b'\x0D' + self.data_ip = b'' + self.measure_group = { + "Lsky": b'', + "Esky": b'', + "Lwater": b'', + } + self.one_group_data= b'' + self.state = 0 + + def readFile2Buf(self, fpath) -> None: + with open(fpath,"rb") as f: + self.__buf = f.read() + pass + pass + + def read_buf(self, size: int) -> bytes: + if size > self.__buf.__len__(): + return b'' + ret = self.__buf[0:size] + self.__buf = self.__buf[size:] + return ret + + def write_buf(self, buf: bytes) -> None: + len = buf.__len__() + # logging.info(f'Received ID:{id} Size:{len}') + self.__buf = self.__buf+buf + + def get_buf_size(self) -> int: + return self.__buf.__len__() + + def back_bytes(self, buf: bytes) -> None: + self.__buf = buf+self.__buf + + def reset_head(self) -> None: + self.__head = {} + + def reset_buf(self) -> None: + self.__buf = b'' + + def getResult(self) -> str: + return self.res + + def resetMeasureGroup(self) -> None: + self.measure_group['Lsky'] = b'' + self.measure_group['Esky'] = b'' + self.measure_group['Lwater'] = b'' + + def getMeasureGroup(self) -> dict: + return self.measure_group + + def decode_handheld(self) -> bool: + '''以26个字节开始,一般26个00 ,然后23... 07.... ''' + if self.get_buf_size() < 1754: + self.__buf = b'' + return False + + self.data_ip = self.read_buf(26) + Lsky = self.read_buf(576) + Esky = self.read_buf(576) + Lwater = self.read_buf(576) + + self.measure_group['Lsky'] = self.deal_576_to_512(Lsky) + self.measure_group['Esky'] = self.deal_576_to_512(Esky) + self.measure_group['Lwater'] = self.deal_576_to_512(Lwater) + + if self.measure_group['Lsky'] == b'' \ + or self.measure_group['Esky'] == b'' \ + or self.measure_group['Lwater'] == b'' : + return False + + return True + + def decode_one_group_handheld(self) -> bool: + '''以26个字节开始,一般26个00 ,然后23... 07.... ''' + if self.get_buf_size() < 1754: + self.__buf = b'' + return False + + self.data_ip = self.read_buf(26) + self.one_group_data = self.read_buf(1728) + + return True + + def deal_576_to_512(self,data:bytes) -> bytes: + ''' 576字节校验,拆分成字典 23... 07.... ,然后254*2''' + ret = {} + index = 72 + for i in range(8): + temp = data[i*index : i*index+index] + # print( temp.hex()) + if temp[0] != 35 and temp[0]<8 and temp>0 : + return b'' + pass + ret.update( { temp[4]: temp} ) + if len(ret) != 8: + return b'' + ret_byte = ret[7][7:71]+ ret[6][7:71]+ ret[5][7:71]+ ret[4][7:71] \ + + ret[3][7:71]+ ret[2][7:71]+ ret[1][7:71]+ ret[0][7:71] + return ret_byte + + + def decode(self) -> str: + ret = '' + temp_buf = b'' + token = ";" + if TOKEN: + token = TOKEN + + if self.state == 0: + while self.get_buf_size() >= 1: + if self.read_buf(1) != self.__end_sign: + continue + self.state = 1 + break + + if self.state == 1: + while self.get_buf_size() >= 1: + buf = self.read_buf(1) + if buf != self.__end_sign: + temp_buf += buf + if buf == self.__end_sign: + ret = temp_buf[0:20].decode( + 'utf-8').strip(" ") + token + temp_buf[-9:-1].decode('utf-8').strip(" ") + log.info(f"decode : {ret}") + temp_buf = b'' + self.back_bytes(temp_buf) # 写回临时buf到 + self.res = ret + return ret + pass + + +class HandHeldPath(object): + '''处理一次测量''' + def __init__(self, ): + self.mode = 0 + self.cfg ={} + self.mydir = MyDir() + self.base_path:Path = Path() + self.output_path:Path = Path() + self.data_path:Path = Path() + self.filelist = [] + self.error_result = [] + pass + + def setMode( self, mode:int = 0 ): + self.mode = mode + pass + + def setBasePath( self, fpath:Path ): + self.base_path = fpath + pass + + def setDataPath( self, fpath:Path ): + self.data_path = fpath + pass + + def setOutputPath( self, fpath:Path ): + self.output_path = fpath + pass + + def getSensorPathFromInfoPath( self, info_path:Path, ) -> Path: + # 服务器上转移后的模式 + if self.mode == 1: + sensor_path = self.base_path.joinpath( + info_path.parent, + "sensor.bin" + ) + elif self.mode == 0: + sensor_path = self.base_path.joinpath( + "data", + info_path.parts[1][:4] + "_" + info_path.parts[1][-2:], + info_path.parts[2], + "sensor", + info_path.name + ) + else: + sensor_path = None + return sensor_path + pass + + def getOutputPathFromSensorPath(self,sensor_path:Path ) -> Path: + if self.mode == 1: + output_path = self.output_path.joinpath( + self.info_dict['year'] + "_" + self.info_dict['month'] + + "_" + self.info_dict['day'] + "_" + self.info_dict['hour'] + + "_" + self.info_dict['minute'] + "_" + self.info_dict['second'] + ) + elif self.mode == 0: + output_path = self.output_path.joinpath( + sensor_path.parts[1][:4] + "_" + sensor_path.parts[1][-2:] + + "_" +sensor_path.parts[2] + "_" +sensor_path.name + ) + else: + output_path = None + return output_path + pass + + def getCurrentMeasureTimeFromPath(self,fpath:Path ) -> str: + ret = '' + if self.mode == 1: # 读信息txt获得时间 + txt_path = fpath.parent.glob("*.txt") + txt_stem = txt_path[0].stem + ret = fpath.parts[1][:4]+"-"+fpath.parts[1][-2:]+"-"+fpath.parts[2]+" " \ + + txt_stem[-9:-7] + ":" + txt_stem[-6:-4] + ":" + txt_stem[-3:-1] + pass + elif self.mode == 0: + ret = fpath.parts[1][:4]+"-"+fpath.parts[1][-2:]+"-"+fpath.parts[2]+" " \ + + fpath.name[0:2] + ":" + fpath.name[3:5] + ":" + fpath.name[6:8] + else: + pass + return ret + pass + + + + def getDataFileList(self, ): + ''' + 获得成对的info sensor 文件 + [目录名，文件名，年月日，时间, measure_id] + ''' + # ret = [] + fs = None + self.filelist = [] + if self.mode == 1: + fs = self.data_path.glob( "*/*/*/*/info.bin" ) + else: + fs = self.data_path.glob( "*/*/info/*" ) + + for f in fs: + error_file = {} + if f.stat().st_size==0: + error_file.update( {"path": f } ) + error_file.update( {"error":"info file size is zero"} ) + self.error_result.append(error_file) + continue + + # self.info_path_fname = f + sensor_path = self.getSensorPathFromInfoPath(f) + # sensor_path = Path(sensor_purepath) + # sensor 文件不存在 + if not sensor_path.exists(): + error_file.update( {"path":f} ) + error_file.update( {"error":"cannot find the sensor file "} ) + self.error_result.append(error_file) + continue + + # sensor文件大小为0 + if sensor_path.stat().st_size==0: + error_file.update( {"path":sensor_path} ) + error_file.update( {"error":"sensor file size of the sensor is zero' "} ) + self.error_result.append(error_file) + continue + self.setFilelist(f,sensor_path ) + pass + + + def setFilelist(self, info_path:Path, sensor_path:Path): + temp = {} + temp.update( {"info_path" : info_path } ) + temp.update( {"name" : info_path.name} ) + temp.update( {"parent" : info_path.parent} ) + + if self.mode==1: # 服务器转移后目录 + temp.update( {"year" : info_path.parts[-5] } ) + temp.update( {"month" :info_path.parts[-4] } ) + temp.update( {"day" :info_path.parts[-3] } ) + + temp.update( {"year" : info_path.parts[1][:4] } ) + temp.update( {"month" :info_path.parts[1][-2:] } ) + temp.update( {"day" :info_path.parts[2] } ) + temp.update( { "sensor_path" :sensor_path } ) + self.filelist.append(temp) + + def getFilelist(self, ): + return self.filelist + + def printTest(self,d,sleep_time:int=5): + log.info( f"***** : I am testing from HandheldPath ********", __name__ ) + print(d) + log.info( f"***** : Ending testing from HandheldPath ********", __name__ ) + time.sleep(sleep_time) + pass + +class Handheld(object): + def __init__(self, ): + """ + @description : 手持数据初始化 + 入口： dealOneMeasurement_Handheld() + """ + self.device_type = DeviceType.AWRAMS.name + self.device_enum = None + self.device_id = {} + self.syscfg = {} + self.retrieve ={} + self.error_result=[] + self.mode = 0 # 0:默认SD数据， 1:服务器数据 + self.base_path = CURRENT_DIR + self.current_path = CURRENT_DIR + self.data_path = DATA_DIR + self.output_path = OUTPUT_DIR + self.info_path_fname:Path = self.base_path + self.sensor_path_fname:Path = self.base_path + self.filelist = [] # 包含了全部路径信息 + self.intensity_before_avg = None + self.intensity_after_avg = {} # 最终结果{lsky: esky: lwater: } + self.intensity_after_interpo = {} # 最终结果{lsky: esky: lwater: } + + self.one_group_result = {} # 手持式 多次间隔 + self.wavelength = np.array([]) + self.real_wavelength = {} # 最终结果{lsky: esky: lwater: } + self.current_filepath = '' # 不含后缀 + self.current_measure_time = None # 当前测量时间 + self.current_group_num = 0 + self.measurement_interval = 0 + self.measurement_repeat = 1 + # self.sl2f = SaveList2File() + self.mydir = MyDir() + self.my_time = MyTime() + self.hhb = HandHeldBuf() + self.hhp = HandHeldPath() + self.ramses = Ramses() + self.info_dict = {} + self.res = {} # 最终结果{lsky: esky: lwater: Lw: Rs:} + + # 方法 + self.setDeviceEnum() + self.setMode() + self.set_hhp_path() + pass + + def setMode(self,mode=0 ): + self.hhp.setMode (mode) # 0 ：读SD卡 1：读服务器数据 + + def setMeasureID(self, mid:int): + self.measure_id = mid + + def setOldFolder(self, tuple_path:tuple): + self.old_folder = self.base_path.joinpath( *tuple_path ) + + def transferFromOldFolder(self, ): + log.info( f"transferFromOldFolder: ", __name__ ) + self.getNewFolderFromOldFolder() + bin_files = self.old_folder.glob('*.bin') + for bf in bin_files: + if bf.name == "pic.bin": + bf.replace( self.new_folder.joinpath( "pic.jpg" ) ) + else: + bf.replace( self.new_folder.joinpath(bf.name) ) + pass + + def deleteOldFolder(self, ): + log.info( f"deleteOldFolder: ", __name__ ) + self.mydir.setBaseDir(self.old_folder) + self.mydir.deleteDir() + self.old_folder = DATA_DIR + + def getNewFolderFromOldFolder( self, ) -> Path: + # 服务器上转移后的模式 + tmp_folder = self.old_folder.parent.parent + path_tuple = ("20"+str(self.info_dict['year']) + ,str(self.info_dict['month']) + ,str(self.info_dict['day']) + ,self.old_folder.parts[-1] ) + self.new_folder =tmp_folder.joinpath(*path_tuple) + self.mydir.setBaseDir(self.new_folder) + self.mydir.newDirIfNot() + pass + + def getInfoDict(self, ): + info_path = self.old_folder.joinpath("info.bin") + hexbin = self.read_bin(info_path) + self.info_dict = self.decode_info(hexbin) + self.new_folder = self.getNewFolderFromOldFolder() + + def getCurrentMeasureTimeFromInfoDict(self, ) -> str: + ret = "20"+ str(self.info_dict['year']) + '-' \ + + str(self.info_dict['month']) + '-' \ + + str(self.info_dict['day']) + ' ' \ + + str(self.info_dict['hour']) + ':' \ + + str(self.info_dict['minute']) + ':' \ + + str(self.info_dict['second']) + return ret + pass + + def set_hhp_path(self, ): + self.hhp.setBasePath (self.base_path) + self.hhp.setDataPath (self.data_path) + self.hhp.setOutputPath (self.output_path) + + def setDeviceEnum(self, ): + if self.device_type == DeviceType.SURFACE.name: + self.device_enum = RamsesSURFACE + if self.device_type == DeviceType.AWRAMS.name: + self.device_enum = RamsesAWRAMS + if self.device_type == DeviceType.PROFILE.name: + self.device_enum = RamsesPROFILE + + def getDataFileList(self, ): + self.hhp.getDataFileList() + self.filelist = self.hhp.getFilelist() + # self.printTest(self.filelist) + pass + + def dealAllMeasurements(self, ): + log.info(f" 所有测量文件", __name__, "dealAllMeasurements", ) + if len(self.filelist)<1: + pass + # 前面已经考虑各种文件错误 + for df in self.filelist: + # 处理信息帧 + self.info_dict = {} + self.info_path_fname:Path = df["info_path"] + self.sensor_path_fname:Path = df["sensor_path"] + hex_info = self.read_bin( self.info_path_fname ) + try: + self.info_dict = self.decode_info( hex_info ) + except: + log.error( f"处理信息文件" + + "/" +self.info_path_fname.stem + + "出现错误", __name__, "dealAllMeasurements" ) + raise MyException( "处理文件"+ self.info_path_fname + "出现错误") + pass + + try: # awrams每次只有一组数据 + self.dealOneMeasurement(self.sensor_path_fname ) + # self.measurement_interval = int(self.info_dict['Measure_Interval']) + # self.measurement_repeat = int(self.info_dict['Measure_Repeat']) + # self.dealOneHandheldMeasurement(self.sensor_path_fname ) + except Exception as e: + log.error( "处理传感器文件" + self.sensor_path_fname.name + " 出现错误",__name__,"dealAllMeasurements") + raise MyException( "处理传感器文件" + self.sensor_path_fname.name + " 出现错误" ) + pass + log.info(f"Finished !! ", __name__, "dealAllMeasurements") + return True,self.error_result + pass + + def dealOneMeasurement_Handheld(self, fpath:Path): + '''handheld一次测量包含多组数据''' + # 调用handheldbuf 处理，将一组数据提交出来 + log.info(f" 手持测量数据", __name__, "dealOneMeasurement_Handheld") + + info_path = self.new_folder.joinpath("info.bin") + sensor_path = self.new_folder.joinpath("sensor.bin") + + # 信息帧解析 + hex_info = self.read_bin( info_path ) + try: + self.info_dict = self.decode_info( hex_info ) + except: + log.error( f"处理信息文件" + + "/" +self.info_path_fname.stem + + "出现错误", __name__, "dealAllMeasurements" ) + raise MyException( "处理文件"+ self.info_path_fname + "出现错误") + pass + + # 当前文件名 + self.output_path = self.new_folder() + self.hhp.setMode(1) + self.current_measure_time = self.getCurrentMeasureTimeFromInfoDict() + ymdhms = "20"+ str(self.info_dict['year']) + '_' \ + + str(self.info_dict['month']) + '_' \ + + str(self.info_dict['day']) + '_' \ + + str(self.info_dict['hour']) + '_' \ + + str(self.info_dict['minute']) + '_' \ + + str(self.info_dict['second']) + + log.debug(f"current_measure_time: {self.current_measure_time}", __name__, "dealOneMeasurement_Handheld") + + self.hhb.readFile2Buf(sensor_path) + log.debug(f"buf: {self.hhb.get_buf_size()}", __name__, "dealOneMeasurement_Handheld") + self.decode_sensor_buf() + + # 解析Buf, 对buf分组，获得[{lsky: esky : lwater} .... ] + len_total = len(self.intensity_before_avg) + if len_total%self.measurement_repeat != 0: + self.res = {} + return # 返回退出 + group_num = int(len_total/self.measurement_repeat) + + log.debug(f"group_num...: {group_num}", __name__, "dealOneMeasurement_Handheld") + self.real_wavelength = self.getWavelenthDict() + if group_num == 1: + self.dealOneGroup() # self.intensity_before_avg + return + + self.dealMultiGroup(group_num) + + def dealMultiGroup(self, group_num:int ): + log.info(f"group_num: {group_num}", __name__, "dealMultiGroup") + # 分组进行处理 + for i in range(group_num): + self.current_group_num = i + # 重设当前测量时间 + self.current_measure_time = "" + self.res = {} + tmp_before_avg = [] + for j in range( self.measurement_repeat ): + tmp_before_avg.append( self.intensity_before_avg[j+i*self.measurement_repeat] ) + pass + self.getAvg(tmp_before_avg) + self.__do_sensor_dict_interpo() + self.appendSave( ) + + + def dealOneGroup(self, ): + # 分组，并获得平均值， 255个未插值结果 (依据 measurement_interval measurement_repeat) + self.getAvg( self.intensity_before_avg ) + + # 插值 + self.real_wavelength = self.getWavelenthDict() + self.__do_sensor_dict_interpo() + + self.ymdhms = "20"+ str(self.info_dict['year']) + '_' \ + + str(self.info_dict['month']) + '_' \ + + str(self.info_dict['day']) + '_' \ + + str(self.info_dict['hour']) + '_' \ + + str(self.info_dict['minute']) + '_' \ + + str(self.info_dict['second']) + + self.output_path = self.output_path.joinpath( self.ymdhms ) + self.appendSave() + + def dealOneMeasurement(self, fpath:Path): + '''handheld一次测量包含多组数据''' + # 调用handheldbuf 处理，将一组数据提交出来 + log.info(f"dealOneMeasurement: 一组测量数据", __name__, "dealOneMeasurement") + if len(self.filelist)<1: + pass + + # 当前文件名 + self.output_path = OUTPUT_DIR + self.current_filepath = fpath + self.current_measure_time = self.hhp.getCurrentMeasureTimeFromPath(fpath) + self.ymdhms = "20"+ str(self.info_dict['year']) + '_' \ + + str(self.info_dict['month']) + '_' \ + + str(self.info_dict['day']) + '_' \ + + str(self.info_dict['hour']) + '_' \ + + str(self.info_dict['minute']) + '_' \ + + str(self.info_dict['second']) + self.output_path = self.output_path.joinpath( self.ymdhms ) + log.debug(f"current_measure_time: {self.current_measure_time}", __name__, "dealOneHandheldMeasurement") + + self.hhb.readFile2Buf(fpath) + self.decode_sensor_buf() + self.real_wavelength = self.getWavelenthDict() + self.dealOneGroup() + + def dealOneMeasurement_Online(self, ): + '''handheld一次测量包含多组数据''' + # 调用handheldbuf 处理，将一组数据提交出来 + log.info(f"dealOneMeasurement_Online: 一组测量数据", __name__, "dealOneMeasurement_Online") + + # 当前文件名 + self.output_path = self.new_folder + # self.current_measure_time = self.hhp.getCurrentMeasureTimeFromPath(fpath) + self.ymdhms = self.getTimeFnameFromInfoDict() + + # getBuf + self.get_sensor_buf() + self.decode_sensor_buf() + self.real_wavelength = self.getWavelenthDict() + + self.getAvg( self.intensity_before_avg ) + self.real_wavelength = self.getWavelenthDict() + self.__do_sensor_dict_interpo() + + self.saveOnefileForLskyEskyLwaterLwRS() + + def get_sensor_buf(self,) : + buf = b"" + self.mydir.setBaseDir(self.new_folder) + my_bin_list = self.mydir.get_files_from_currentdir("*.bin") + my_bin_list = self.mydir.sort_filepath_and_check(my_bin_list) + for mbl in my_bin_list: + buf = buf + self.read_bin(mbl) + self.hhb.write_buf(buf) + pass + + def getTimeFnameFromInfoDict(self,) : + ymdhms = "20"+ str(self.info_dict['year']) + '_' \ + + str(self.info_dict['month']) + '_' \ + + str(self.info_dict['day']) + '_' \ + + str(self.info_dict['hour']) + '_' \ + + str(self.info_dict['minute']) + '_' \ + + str(self.info_dict['second']) + return ymdhms + + def decode_sensor_buf(self,) : + # 处理Buf，对多组数据取平均 + self.intensity_before_avg = [] + # res_before_avg = [] + self.clearRes() # 清空数据 + while True: + if not self.hhb.decode_one_group_handheld() : + break # 清空数据 + + res = {} + buf = self.hhb.one_group_data[26:] + for i in range(1,4,1): + temp_buf = buf[7:71] + buf[79:143] + \ + buf[151:215] + buf[223:287] + \ + buf[295:359] + buf[367:431] + \ + buf[439:503] + buf[511:575] + + # Ramses类 标定处理，设置buf 标定文件 + self.ramses.setBuf(temp_buf) + func = self.getFuncBySeq(i) + cfg = self.getCfgByFunc( func) + self.ramses.setCalCfg(cfg) + self.ramses.resetItSpectrum() + self.ramses.ConvertAndCalibrate() + res.update({ func : self.ramses.spectrum }) + self.intensity_before_avg.append( res ) + pass + + def getAvg( self, d:list) : + log.info(f"getAvg: 平均多组数据", __name__, "getAvg") + data = d + ret = {} + len_result = len(data) + if len_result == 0: + self.intensity_after_avg ={} + return None + if len_result == 1: + self.intensity_after_avg = data[0] + return None + ret = data[0] + + res_dict = self.getRetDict() + + for k in res_dict.keys(): + for i in range(1,len_result,1): + data[0][k] = data[0][k] + data[i][k] + ret = data[0][k]/len_result + self.intensity_after_avg.update( { k : ret } ) + log.debug(f"getAvg: {self.intensity_after_avg}", __name__, "getAvg") + pass + + def getRetDict(self,) : + ret_dict = { } + ret_dict.update( {self.device_enum(1).name:np.array([])} ) + ret_dict.update( {self.device_enum(2).name:np.array([])} ) + ret_dict.update( {self.device_enum(3).name:np.array([])} ) + # self.one_group_result = ret_dict + return ret_dict + + def __do_sensor_dict_interpo(self,) : + log.info( f"同步处理多个个插值 ", __name__, "__do_sensor_dict_interpo" ) + self.clearRes() + for k in self.intensity_after_avg.keys(): + tmp = np.interp( self.wavelength, self.real_wavelength[k], self.intensity_after_avg[k] ) + self.res.update( { k : tmp } ) + + def getCfgByDid(self, func:str) : + cfg_id:dict = self.cfg.get(int(self.device_id)) + cfg_sensor = cfg_id.get( func) + return cfg_sensor + pass + + def getFuncBySeq(self, seq:int) : + func = "" + if self.device_type == DeviceType.AWRAMS.name: + func = RamsesAWRAMS(seq).name + if self.device_type == DeviceType.SURFACE.name: + func = RamsesAWRAMS(seq).name + if self.device_type == DeviceType.PROFILE.name: + func = RamsesAWRAMS(seq).name + return func + + def getCfgByFunc(self, func:str) : + cfg_id:dict = self.syscfg.get(int(self.device_id)) + cfg_sensor = cfg_id.get( func) + return cfg_sensor + pass + + def getCfgBySeq(self, seq:int) : + func = "" + if self.device_type == DeviceType.AWRAMS.name: + func = RamsesAWRAMS(seq).name + if self.device_type == DeviceType.SURFACE.name: + func = RamsesAWRAMS(seq).name + if self.device_type == DeviceType.PROFILE.name: + func = RamsesAWRAMS(seq).name + cfg_id:dict = self.syscfg.get(int(self.device_id)) + cfg_sensor = cfg_id.get( func) + return cfg_sensor + pass + + def saveOnefileForLskyEskyLwaterLwRS(self, ) -> bool: + log.info(f" ", __name__, "saveOnefileForLskyEskyLwaterLwRS") + self.mydir.setBaseDir(self.output_path) #基路径 + + time_name = self.getTimeFnameFromInfoDict() + Lw = self.res["Lwater"] - ROWFACTOR * self.res["Lsky"] + self.res.update({ self.device_enum(4).name : Lw }) + Rs = self.res[self.device_enum(4).name] / self.res["Esky"] + self.res.update({ self.device_enum(5).name : Rs }) + + self.mydir.setHeader( self.wavelength.tolist(), TOKEN, "device_id_"+str(self.device_id) ) + header = self.mydir.header_str + + save_path_csv = self.new_folder.joinpath(time_name + SAVE_EXT_NAME) + data_str = "" + for k in self.res.keys(): + self.mydir.setContent(self.res[k].tolist(),TOKEN,k ) + data_str = data_str + NEWLINE + self.mydir.content_str + save_path_csv.write_text(header+data_str) + + path_info_txt = self.new_folder.joinpath( time_name + "_info.txt" ) + self.save_dict_to_file( self.info_dict, path_info_txt ) + + def appendSave(self, ) -> bool: + '''self.output_path''' + self.checkAndSaveData( ) + self.getLwRsAndSave() + path_info_txt = self.output_path.joinpath( "info.txt" ) + self.save_dict_to_file( self.info_dict, path_info_txt ) + + def checkAndSaveData(self, ) -> bool: + """ + check self.Lsky Esky Lwater and Save + 处理self.res 的数据 + """ + log.info(f"checkAndSaveData: {self.output_path.parts}", __name__) + self.mydir.setBaseDir(self.output_path) #基路径 + self.mydir.setHeader( self.wavelength.tolist(), TOKEN, "device_id_"+str(self.device_id) ) + # print(f"header_str : {self.mydir.header_str[-1:]}") + + if self.current_group_num == 0: + self.newFileByFunc( self.device_enum(1).name ) + self.newFileByFunc( self.device_enum(2).name ) + self.newFileByFunc( self.device_enum(3).name ) + pass + + self.appendFileByFunc( self.device_enum(1).name ) + self.appendFileByFunc( self.device_enum(2).name ) + self.appendFileByFunc( self.device_enum(3).name ) + + def newFileByFunc(self, func:str) -> None: + self.mydir.newFileIfNot( func+SAVE_EXT_NAME) + if self.mydir.checkHeader() == -1: + log.error(f"请备份文件:{self.mydir.current_filepath.parent} {self.mydir.current_filepath.name}, 并删除文件后再试!", __name__) + raise MyException( f"请备份文件:{self.mydir.current_filepath.parent} {self.mydir.current_filepath.name}, 并删除文件后再试!" ) + return False + if self.mydir.checkHeader() == 1: + pass + if self.mydir.checkHeader() == 0: + self.mydir.writeHeader() + pass + + # 写入数据content + self.mydir.setContent(self.res[func].tolist(), TOKEN, self.current_measure_time) + self.mydir.writeContent() + + def appendFileByFunc(self, func:str) -> None: + # 追加写入数据content + self.mydir.setContent(self.res[func].tolist(), TOKEN, self.current_measure_time) + self.mydir.writeContent() + + def clearRes(self, ) -> None: + self.res = { } + + def getLwRsAndSave(self, ) -> bool: + """ + 并计算Lw Rs并保存 + """ + Lw = self.res["Lwater"] - ROWFACTOR * self.res["Lsky"] + self.res.update({ "Lw" : Lw }) + Rs = self.res["Lw"] / self.res["Esky"] + self.res.update({ "Rs" : Rs }) + self.mydir.setBaseDir(self.output_path) #基路径 + + # 保存 + + if self.current_group_num == 0: + self.newFileByFunc( "Lw" ) + self.newFileByFunc( "Rs" ) + pass + + self.appendFileByFunc( "Lw" ) + self.appendFileByFunc( "Rs" ) + + return True + + def read_bin(self,fpath: Path): + log.debug(f" readbin: ", __name__, "", "" ) + ret = None + if not fpath.exists() : + log.info(f"not find file: {fpath} ") + return ret + with open(fpath, 'rb') as file: + ret = file.read() + return ret + log.debug(f" readbin: {ret} ", __name__, "", "" ) + return ret + pass + + def decode_info( self,info: bytes ) -> dict: + ret = {} + # 剖面型加了1311+24个字节共26个字节 + # 保留字节有所变化，改为序列号，每个序列号两个字节 + try: + temp = struct.unpack(" None: + temp_str = "" + for key, value in info_dict.items(): + temp_str = temp_str + key + " : " + str(value) + "\n" + with open(fpath, "w+") as f: + f.write(temp_str) + + def save_error_to_file(self, errlist: list, fpath: Path) -> None: + temp_str = "" + if len(errlist) <1: + return None + for errdict in errlist: + temp_str = temp_str + errdict["path"] +" : "+ errdict["error"] + "\n" + pass + with open(fpath, "w+") as f: + f.write(temp_str) + return None + pass + + def getWavelenthDict( self, ) -> dict: + ret_dict = self.getRetDict() + cfg_id:dict = self.syscfg.get( int(self.device_id)) + for k in ret_dict.keys(): + tmp = self.getWavelength( cfg_id.get(k) ) + ret_dict.update({ k : tmp }) + return ret_dict + + def getWavelength(self,ramsesdict:dict) -> np.ndarray: + ret = [] + for i in range(1,256): + tmp = float(ramsesdict['c0s']) + float(ramsesdict['c1s'])*i \ + + float(ramsesdict['c2s'])*i*i + float(ramsesdict['c3s'])*i*i*i + ret.append(tmp) + pass + return np.array(ret) + + def setSyscfg(self, syscfg:dict): + self.syscfg = syscfg + + def setRetrieve(self, retrieve:dict): + log.debug( f"setRetrieve : {retrieve}......", __name__ ) + self.retrieve = retrieve + self.setNewWavelength() + # self.wavelength:np.ndarray = np.arange ( self.retrieve['beginWL'], self.retrieve['endWL'], self.retrieve['interval'] ) + + def setNewWavelength(self, ): + self.wavelength:np.ndarray = np.arange ( + self.retrieve['beginWL'], self.retrieve['endWL'], self.retrieve['interval'] ) + + def setDeviceID(self, did:int): + self.device_id = did + + def setDeviceType(self, device_type): + self.device_type = device_type + + def getErrorInfoDict(self, ): + return self.error_result + + def printResult(self,): + log.info( f"***** : Print Lsky Esky Lwater Lw Rs......", __name__ ) + print(self.res[self.device_enum(1)]) + print(self.res[self.device_enum(2)]) + print(self.res[self.device_enum(3)]) + print(self.res[self.device_enum(4)]) + print(self.res[self.device_enum(5)]) + pass + + def printTest(self,d,sleep_time:int=5): + log.info( f"***** : I am testing ********", __name__ ) + print(d) + log.info( f"***** : Ending testing ********", __name__ ) + time.sleep(sleep_time) + pass + +if __name__ == "__main__": + # hh = HandHeld() + # hh.getDataFileList() + # hh.dealAllMeasurements() + + # data/2023_02/07/sensor/14_02_46 + # hhb= HandHeldBuf() + # hhb.readFile2Buf("data/2023_02/07/sensor/14_02_46") + # hhb.decode_handheld() + # print(hh.error_result) + # print("======") + # print(hh.filelist) + + # print( hh.getDataFname() ) + + # t = b'\x07' + # print( int(t[0])) + # print( 2 << int(t[0])) + + + pass + diff --git a/myconfig.py b/myconfig.py new file mode 100644 index 0000000..db93dec --- /dev/null +++ b/myconfig.py @@ -0,0 +1,322 @@ +#! python3 +# -*- encoding: utf-8 -*- +''' +@File : myconfig.py +@Time : 2023/03/01 15:28:20 +@Author : Jim @ Yiwin +@Version : 1.0 +@Contact : jim@yi-win.com +@Descrip : SysConfig +''' + +import yaml +from enum import Enum +from pathlib import Path + +DEVICE_ID = [2] +CURRENT_DIR = Path() +DATA_DIR = Path("data") +CAL_DIR = Path("calfile") +OUTPUT_DIR = Path("data", "output") +YAML_FILE_NAME = "config.yml" +RETRIEVE_CFG_FILE = "retrieve.yml" +FILE_MARK = ['Spectrum', 'DATA'] +BEGIN_WAVELENGTH = 350 +END_WAVELENGTH = 950 +SAVE_EXT_NAME = ".csv" +INTERVAL = 1.0 +SEPARATOR = ";" +TOKEN = ";" +NEWLINE = "\n" +ROWFACTOR = 0.026 + +class DeviceType(Enum) : + AWRAMS = 1 + SURFACE = 2 + PROFILE = 3 + +class RamsesFunc(Enum): + Lsky = 1 + Esky = 2 + Lwater = 3 + Lw = 4 + Rs = 5 + +class RamsesAWRAMS(Enum): + Lsky = 1 + Esky = 2 + Lwater = 3 + Lw = 4 + Rs = 5 + +class RamsesSURFACE(Enum): + Lsky = 1 + Esky = 2 + Lwater = 3 + Lw = 4 + Rs = 5 + +class RamsesPROFILE(Enum): + Ed = 1 + Esky = 2 + Lu = 3 #upwelling + Lw = 4 + Rs = 5 + +IP_CAL = { + "Incl_Orientation": "up", + "Incl_Xgain": 1.0, + "Incl_Xoffset": 125, + "Incl_Ygain": 0.9375, + "Incl_Yoffset": 126, + "Incl_KBG": 1.2073, + "Incl_Kref": 0.1275, + "Press_Current_mA": 1.08, + "Press_Surface_bar": 5.57, + "Press_Gain": 2.7, + "WithIncl": 1, + "WithPress": 1, + "Press_Sens_mV_bar_4mA": 71.36, + "Press_Sens_mV_bar_1mA": 17.84, + "Press_Type": "PA-10/TAB/10bar", + "CalibrationDate": "08.06.2018", +} + +RAMSES_CAL = { + "SN": "", + "TYPE": "SAM", # SAMIP or SAM + "FUNC": "Lsky", + "inifile": "", + "calfile": "", + "calaqfile": "", + "backfile": "", + "samsn": "", + "b0": [], + "b1": [], + "cal": [], + "calaq": [], + "DarkPixelStart": 237, + "DarkPixelStop": 254, + "Firmware": 2.06, + "IDDataBack": "DLAB_2016-11-29_14-47-59_729_812", + "IDDataCal": "DLAB_2016-12-07_12-00-24_364_510", + "IDDataCalAQ": "DLAB_2016-12-07_12-02-43_591_545", + "IntegrationTime": 0, + "Reverse": 0, + "SerialNo_MMS": 103307, + "WavelengthRange": "310..1100", + "c0s": 299.895, + "c1s": 3.31161, + "c2s": 0.00031652, + "c3s": -1.73194e-06, + "c4s": +0.000000000E+00, + "cs": 102842, + "savefile": "" +} + +class MyConfig(object): + """ + 设置 ID对应的传感器 + """ + + def __init__(self) -> None: + self.device_id = [] + self.device_type = None + self.current_device_id = None + self.system_cfg = {} + self.cfg_path = Path() + self.yml_cfg_file = YAML_FILE_NAME + self.retrieve_cfg_file = Path(RETRIEVE_CFG_FILE) + self.system_cal_cfg = {} + self.validate = { } + + def addDeviceID(self, id:int) -> None: # + self.device_id.append(id) + pass + + def setDeviceID(self, id:int) -> bool: # + if id in self.device_id: + self.current_device_id = id + return True + else: + self.current_device_id = None + return False + pass + + def setDeviceType(self, device_type:DeviceType) -> None: + self.device_type = device_type + pass + + def setRetrieveCfg(self, rtv_yml:str="") -> None: + if rtv_yml =="": + return None + self.retrieve_cfg_file = Path(rtv_yml) + pass + + def getSystemCfg(self,)->None: + ''' + 不同系统，修改此函数，或添加函数 getSystemCfg***()供调用 + ''' + if self.current_device_id == None: + self.system_cfg = None + if self.device_type == None: + self.system_cfg = None + temp_cfg = {} + if self.device_type == DeviceType.AWRAMS: + temp_cfg = { + 1: {"SN": "85B5", "FUNC": RamsesAWRAMS(1).name}, + 2: {"SN": "50ED", "FUNC": RamsesAWRAMS(2).name}, + 3: {"SN": "852F", "FUNC": RamsesAWRAMS(3).name} + } + if self.device_type == DeviceType.SURFACE: + temp_cfg = { + 1: {"SN": "85B5", "FUNC": RamsesSURFACE(1).name}, + 2: {"SN": "50ED", "FUNC": RamsesSURFACE(2).name}, + 3: {"SN": "852F", "FUNC": RamsesSURFACE(3).name} + } + if self.device_type == DeviceType.PROFILE: + temp_cfg = { + 1: {"SN": "85B5", "FUNC": RamsesPROFILE(1).name}, + 2: {"SN": "50ED", "FUNC": RamsesPROFILE(2).name}, + 3: {"SN": "852F", "FUNC": RamsesPROFILE(3).name} + } + self.system_cfg.update( { self.current_device_id : temp_cfg } ) + pass + + def setCfgRamsesSN(self, sn_cfg: dict)->None: + if len(sn_cfg) == 0: + return None + + for k in self.system_cfg[self.current_device_id].keys(): + if str(k) in sn_cfg.keys() : + self.system_cfg[self.current_device_id][k]["SN"] = sn_cfg[str(k)] + else: + self.system_cfg[self.current_device_id][k]["SN"] = None + # if k in sn_cfg.keys() : + # self.system_cfg[self.current_device_id][k]["SN"] = sn_cfg[k] + pass + + def setSystemCalCfg(self, sn_cfg: dict)->None: + if len(sn_cfg) == 0: + pass + + def getDictByAttr(self, *args) -> dict: + ret = {} + if len(args) == 0: + return ret + if len(args) == 1: + if not hasattr(self, args[0]): + return ret + tmp = getattr(self, args[0]) + if isinstance(tmp, dict): + ret.update(tmp) + return ret + if len(args) == 2: + if not hasattr(self, args[0]): + return ret + if not isinstance(getattr(self, args[0]), dict): + return ret + tmp: dict = getattr(self, args[0]) + if not tmp.__contains__(args[1]): + # print(f"------------{args[1]}") + return ret + tmp2 = tmp[args[1]] + if isinstance(tmp2, dict): + ret.update(tmp2) + return ret + if len(args) > 2: + return ret + pass + + # 设置字典对应的键值 + def set_attr(self, d: dict, k, v) -> bool: + if d.__contains__(k): + d.update({k: v}) + return True + return False + + def write_yaml(self, d: dict): + with open(self.yml_cfg_file, "w", encoding="utf-8") as f: + yaml.dump(d, f) + + def read_yaml(self ) -> dict: + with open(self.yml_cfg_file, "r", encoding="utf-8") as f: + content = f.read() # conent 读出来是字符串 + d = yaml.load(content, Loader=yaml.FullLoader) # 用load方法转字典 + return d + + def write_rtv_yaml(self, d: dict): + with open(self.retrieve_cfg_file, "w", encoding="utf-8") as f: + yaml.dump(d, f) + + def read_rtv_yaml(self ) -> dict: + with open(self.retrieve_cfg_file, "r", encoding="utf-8") as f: + content = f.read() # conent 读出来是字符串 + d = yaml.load(content, Loader=yaml.FullLoader) # 用load方法转字典 + return d + + def get_retrieve(self) -> dict: + retrieve = {} + retrieve.update({"beginWL": BEGIN_WAVELENGTH}) + retrieve.update({"endWL": END_WAVELENGTH}) + retrieve.update({"interval": INTERVAL}) + retrieve.update({"rowFactor": ROWFACTOR}) + return retrieve + pass + +ramses_buf_str= "\ + 23a0000007fefe0a0781067d067d068e0693069c069c06b006b506cb06e40619076607e1076c081509cd09bb0a7d0bee0b1d0c6d0cca0ca40ddc0f30135b18b4224d320e43f852c8\ + 23a0000006fefe17639c71c97c9484bb89358be98e5d98b1a37eadccb66abd26be31b97db124aa18a3f29c0499349735968e93a48eea8a028bc28cec8d048f1c92c096de9ab99d43\ + 23a0000005fefee0a157a9ecb1b1b97dc034c507c741c7d2c65ec550c20dbde9b535ae56a60a9e2296cd8ee887278129798170c669b16503632e61605f3a5def5a8e5862561154be\ + 23a0000004fefe3c51f54dda4a0c48634595426a3f5a3cc539903767362b36cc351f356634b633fc32c4310630fb2ec32fe63199349037e03ac03dc03eae3c303a9639d13a413da7\ + 23a0000003fefe97412746bc49404caa4dc54d374ca646d63bde326b332739e43c503d2b3c363aca37ef342d32c22f782d1d2bd328ec250222101e691b2b1a7119d3184018fa174d\ + 23a0000002fefed3177a17e3160616f81423149313f11233126711d8103b10a40f160f8a0efb0d580d660c6f0bd80aa10a580a000ab4099b0994097109e2082c087a0736072c077d\ + 23a0000001fefe2a071e071607140718071e073c074d076a076e076b076007670763076c0760075c073d07350724071207fb06ef06d706cc06b706a506960684067b0672066306a2\ + 23a0000000fefe58065406500642064e064306470642063f064806410644064306430641064206460640063c063e063e063d064406430644063c063c063c063c06400640064906ed\ + 23a0000007fefe0a071b071c07260737073e0749075907650774078b07c0072c08e3082b0ada0be80d9c107b147318931ae51add1a271a77192f1bfb1d99207c265c30c738f03d53\ + 23a0000006fefe07438f489d4d8452d957945cff639171a5832a97deab7fbe09c86cc5acb9d5ab299e90915287de7f8b7ae275ed6f056aff664f662665bc63936335655767926879\ + 23a0000005fefec269206d3e724377fe7b7a8050831f85b68637881689e4884787008580823d7f887bee773d7496702a6cde654c5f895a1f570d5416514d4e9d4bde4897468c44c3\ + 23a0000004fefe71421640db3dc83bbb39533777346c31b92e912c152b5b2aff292129e727cb26d925fd249223b4214020da1fe91ff41f28209420dc201520071e671c221c8d1cfe\ + 23a0000003fefecd1dd31fe72164235924cb24b924f72331211e1ca018ef19221d121f621fe91e081ed51c761b311a2f193c18651788166015a013de11d1106e1041101810e80f2e\ + 23a0000002fefee50fd90fbc0f6d0ff70e610ee00d930d460de40c900c450c010cbd0b7c0b480bfc0aab0a2c0a9e094d092b09fb08d008a208990881087d083908e6078d07660747\ + 23a0000001fefe5b075a07500751074d074e0751075d076807730788077e077c077c07810777077a076c0765075c075607480740073607270723070d070d070b07ff06fd06f0062f\ + 23a0000000fefef606f106ed06f506ef06eb06f106ec06f006e906e806ed06e606ef06e506ea06ec06ee06eb06e906ee06f006ef06ea06e906e606e606e606ec06ea06ea06f506f2\ + 23a0000007fefe0605830479047d047e04810482048204850487048b048c04850487048c049304950498049c04a804a904b604b804c604d004e104f104190546059805ff059a06bb\ + 23a0000006fefe90073209ea0b7610c117cc22b2324a487b645e8798acf2c927d28dc1f2a1ac809664804e543d7a308327a321d81dab1b0f1bca1bb41dba2098243329382e773335\ + 23a0000005fefeca38213e544329486f4c1c500653335592564f577d573b57a556b4557e540a534451424f004d714a9b47994458410e3e9c3af8368c33b630982eca2c052b3429ae\ + 23a0000004fefe632763255a234821321f0e1d061b291983171e16d814a91379124b1138104b0f610e990dd80c240c880bfc0a7b0af8097709020990082508de07c507ba07a50762\ + 23a0000003fefe950781076407460727070007d306ab066e06270605060d0614061306fe05e805d105b405a00586057105630549053d0528051105fd04f304ee04e404dc04dd04c3\ + 23a0000002fefed804d904d004c604bf04b904b904b104a804a604a6049d04980496048f04970491048c04880486047e0484047a047d047b047c0478047a04760476047004700430\ + 23a0000001fefe6f0474046c046d0472046d047304750470046e04760472046f047604700473046f04730470047504700470046d046b047004710470046e046d046a046b046904e4\ + 23a0000000fefe690470046e046b046c04680468046e0466046a046a0469046b046d04640466046c046c046b046a04660466046b046c046a046a0468046f046804740471048d0417\ + " + + +if __name__ == "__main__": + cfg = MyConfig() + cfg.addDeviceID(2) + cfg.addDeviceID(3) + cfg.setDeviceType(DeviceType.AWRAMS) + cfg.setDeviceID(2) + cfg.getSystemCfg() + print(cfg.system_cfg) + + d = {"1":"8888","2":["7777"],"3":["9999"]} + cfg.setCfgRamsesSN(d) + print("修改后。。。。。\n") + print(cfg.system_cfg) + + # cfg.write_yaml( cfg.system_cfg) + + dd = cfg.read_yaml() + # dd 作为cfg.system_cfg 使用 + for k,v in dd.items(): + print(k) + print(type(k)) + + retrieve = { + "beginWL": 350, + "endWL": 950, + "interval": 1, + "rowFactor": 0.026 + } diff --git a/readcal.py b/readcal.py new file mode 100644 index 0000000..59f6fcc --- /dev/null +++ b/readcal.py @@ -0,0 +1,1008 @@ +from tools.mylogger import log +from tools.myexception import MyException +from typing import Tuple,List,Optional,Union,Callable +from pathlib import Path, PurePath +from myconfig import MyConfig,DeviceType,FILE_MARK,CURRENT_DIR,CAL_DIR,DATA_DIR,SAVE_EXT_NAME,\ + RamsesFunc,INTERVAL,SEPARATOR + +class ReadCal(): + def __init__(self) -> None: + pass + + # 数据加时间值 + @staticmethod + def read_columns_set_by_mark(fpath:Path, mark:list, *column ) -> Tuple[str,List] : + # def read_column_1set_by_mark(fpath:str, column:int, *args ) -> Tuple[ str, List ] : + ''' + return: time:str List + mark : ['Spectrum','DATA'] + default end with "[END]" + ''' + log.info(f"read_column_1set_by_mark : \ + {fpath} , mark:{mark} , column:{column}", __name__, "", "") + + # 处理mark args[0] = Spectrum + if len(mark) != 2: + log.error( f"read_column_1set_by_mark() wrong args ",__name__, "", "" ) + raise MyException( f"read_column_1set_by_mark() wrong args " ) + + if len(column) == 0: + log.error(f"read_column_1set_by_mark(), no column para, pls input column ",__name__, "", "" ) + raise MyException( f"read_column_1set_by_mark() , pls input column " ) + + if len( mark ) == 0: + mark_1 = '[Spectrum]' + mark_2 = '[DATA]' + + mark_1 = '['+ mark[0] +']' + mark_2 = '['+ mark[1] +']' + res = [] + sn = ReadCal.readFileSNbyIDDevice(fpath) + # 预备返回数据结构 + # res_time = [] + res_data = [] + columnLen = len(column) + for i in range(columnLen): + res_data.append( [] ) + + with fpath.open('r') as f_handle: + rflag = 0 + # res = [] + # res_time = [] + # res_data = [] + + for line in f_handle: + if mark_1 in line and rflag == 0: + log.debug(f" find {mark_1}", __name__, "", "" ) + rflag += 1 + continue + + if mark_2 in line and rflag > 3 : + log.debug(f" find {mark_1} end ", __name__, "", "" ) + rflag = 0 + continue + + # 获取时间 + if rflag == 1 : + data = line.strip('\n').strip(' ').split('=') + if data[0].strip() == "DateTime" : + log.debug(f" find {mark_1} -> DateTime {data[1]} ", __name__, "", "" ) + time_ = data[1].strip() + rflag += 1 + continue + + if rflag == 2 and ( mark_2 in line ) : + log.debug(f" find {mark_2} -> ", __name__, "", "" ) + rflag += 1 + continue + + if rflag == 3 : + if mark_2 not in line: + # log.debug(f" get data {line}", __name__, "", "" ) + data = line.strip('\n').strip(' ').split(' ') + # 忽略第一行，第一行0开头，保存积分时间的 + if data[0].strip() == "0" : + continue + else: + # if data[1] != "-NAN" and data[1] != "NAN" and data[1] != "+NAN" : + # 处理多列 + for i in range(columnLen): + # res_time.append( time_ ) + res_data[i].append( data[column[i]] ) + continue + + if rflag == 3 and ( mark_2 in line ) : + log.debug(f" find {mark_2} end , line: {line} ", __name__, "", "" ) + rflag += 1 + continue #需要继续让其找到 + + if rflag == 4: + log.debug(f" job done......", __name__, "", "" ) + res_time = time_ + # res_data.append( res ) + # 只读一组数据就 break + rflag += 1 + # res = [] + # rflag = 0 + + if rflag == 5: + log.debug(f" job done, break...", __name__, "", "" ) + break + return ( res_time, res_data) + pass + + # 数据加时间值 + @staticmethod + def read_columns_sets_by_mark(fpath:Path, mark:list, *column + ) -> Tuple[List[str],List[List]] : + # def read_column_1set_by_mark(fpath:str, column:int, *args ) -> Tuple[ str, List ] : + ''' + @desc: 获取多列数据，且为多套数据，非单套数据 + @return: time:List, List + @args : ['Spectrum','DATA'] + default end with "[END]" + ''' + log.info(f"read_column_1set_by_mark : \ + {fpath} , mark:{mark} , column:{column}", __name__, "", "") + + # 处理mark args[0] = Spectrum + if len(mark) != 2: + log.error( f"read_columns_sets_by_mark() wrong args ",__name__, "", "" ) + raise MyException( f"read_columns_sets_by_mark() wrong args " ) + + if len(column) == 0: + log.error(f"read_columns_sets_by_mark(), no column para, pls input column ",__name__, "", "" ) + raise MyException( f"read_columns_sets_by_mark() , pls input column " ) + + if len( mark ) == 0: + mark_1 = '[Spectrum]' + mark_2 = '[DATA]' + + mark_1 = '['+ mark[0] +']' + mark_2 = '['+ mark[1] +']' + res = [] + sn = ReadCal.readFileSNbyIDDevice(fpath) + # 预备返回数据结构 + res_data = [] + res_time = [] + res = [] + columnLen = len(column) + for i in range(columnLen): + # res_time.append( [] ) + res.append( [] ) + + with fpath.open('r') as f_handle: + rflag = 0 + + for line in f_handle: + if mark_1 in line and rflag == 0: + log.debug(f" find {mark_1}", __name__, "", "" ) + rflag += 1 + continue + + if mark_2 in line and rflag > 3 : + log.debug(f" find {mark_1} end ", __name__, "", "" ) + rflag = 0 + continue + + # 获取时间 + if rflag == 1 : + data = line.strip('\n').strip(' ').split('=') + if data[0].strip() == "DateTime" : + log.debug(f" find {mark_1} -> DateTime {data[1]} ", __name__, "", "" ) + time_ = data[1].strip() + rflag += 1 + continue + + if rflag == 2 and ( mark_2 in line ) : + log.debug(f" find {mark_2} -> ", __name__, "", "" ) + rflag += 1 + continue + + if rflag == 3 : + if mark_2 not in line: + # log.debug(f" get data {line}", __name__, "", "" ) + data = line.strip('\n').strip(' ').split(' ') + # 忽略第一行，第一行0开头，保存积分时间的 + if data[0].strip() == "0" : + continue + else: + # if data[1] != "-NAN" and data[1] != "NAN" and data[1] != "+NAN" : + # 处理多列 + for i in range(columnLen): + # res_time.append( time_ ) + res[i].append( data[column[i]] ) + continue + + if rflag == 3 and ( mark_2 in line ) : + log.debug(f" find {mark_2} end , line: {line} ", __name__, "", "" ) + rflag += 1 + continue #需要继续让其找到 + + if rflag == 4: + log.debug(f" job done......", __name__, "", "" ) + res_time.append( time_ ) + res_data.append( res ) + rflag += 1 + # res = [] + # rflag = 0 + + if rflag == 5: + log.debug(f" job done, break...", __name__, "", "" ) + rflag = 0 + continue + + log.debug(f" res.time len {len(res_time)} , \ + res.data len {len(res_data)} , ", __name__, "", "" ) + + return ( res_time, res_data) + + # 数据加时间值 + @staticmethod + def read_columns_sets_by_mark_callback(fpath:Path, mark:list, callback:Callable, *column ) : + ''' + @desc: 获取多列数据，且为多套数据，非单套数据 + @return: time:List, List + @args : ['Spectrum','DATA'] + default end with "[END]" + ''' + log.debug(f"read_column_1set_by_mark : \ + {fpath} , mark:{mark} , column:{column}", __name__, "", "") + + # 处理mark args[0] = Spectrum + if len(mark) != 2: + log.error( f"read_columns_sets_by_mark() wrong args ",__name__, "", "" ) + raise MyException( f"read_columns_sets_by_mark() wrong args " ) + + if len(column) == 0: + log.error(f"read_columns_sets_by_mark(), no column para, pls input column ",__name__, "", "" ) + raise MyException( f"read_columns_sets_by_mark() , pls input column " ) + + if len( mark ) == 0: + mark_1 = '[Spectrum]' + mark_2 = '[DATA]' + + mark_1 = '['+ mark[0] +']' + mark_2 = '['+ mark[1] +']' + + # 预备返回数据结构 + res_data = [] + res_time = [] + res = [] + sn = ReadCal.readFileSNbyIDDevice(fpath) + + columnLen = len(column) + for i in range(columnLen): + # res_time.append( [] ) + res.append( [] ) + + with fpath.open('r') as f_handle: + rflag = 0 + + for line in f_handle: + if mark_1 in line and rflag == 0: + log.debug(f" find {mark_1}", __name__, "", "" ) + rflag += 1 + continue + + if mark_2 in line and rflag > 3 : + log.debug(f" find {mark_1} end ", __name__, "", "" ) + rflag = 0 + continue + + # 获取时间 + if rflag == 1 : + data = line.strip('\n').strip(' ').split('=') + if data[0].strip() == "DateTime" : + log.debug(f" find {mark_1} -> DateTime {data[1]} ", __name__, "", "" ) + time_ = data[1].strip() + rflag += 1 + continue + + if rflag == 2 and ( mark_2 in line ) : + log.debug(f" find {mark_2} -> ", __name__, "", "" ) + rflag += 1 + continue + + if rflag == 3 : + if mark_2 not in line: + # log.debug(f" get data {line}", __name__, "", "" ) + data = line.strip('\n').strip(' ').split(' ') + # 忽略第一行，第一行0开头，保存积分时间的 + if data[0].strip() == "0" : + continue + else: + # if data[1] != "-NAN" and data[1] != "NAN" and data[1] != "+NAN" : + # 处理多列 + for i in range(columnLen): + res[i].append( data[column[i]] ) + continue + + if rflag == 3 and ( mark_2 in line ) : + log.debug(f" find {mark_2} end , line: {line} ", __name__, "", "" ) + rflag += 1 + continue #需要继续让其找到 + + if rflag == 4: + log.debug(f" job done......", __name__, "", "" ) + # callable , not return。 每读出一组就回调，不建议。频繁读写硬盘 + # callback(time_, res) + res_time.append( time_ ) + res_data.append( res ) + rflag += 1 + # res = [] + # rflag = 0 + + if rflag == 5: + log.debug(f" job done, break...", __name__, "", "" ) + rflag = 0 + continue + + # callable , not return。 一个文件读完直接处理 + + callback(sn, res_time, res_data) + + log.debug(f" res.time len {len(res_time)} , \ + res.data len {len(res_data)} , ", __name__, "", "" ) + + # return ( res_time, res_data) + + @staticmethod + def readDataIPinfo(fpath:Path, mark:list, callback:Callable ): + ''' + SAMIP sensor ,Inclination Pressure, InclX InclY Pressure + ''' + log.debug(f"readDataIPinfo : {fpath} , mark:{mark} , ", __name__, "", "") + + # 处理mark args[0] = Spectrum + if len(mark) != 2: + log.error( f"readDataIPinfo() wrong args ",__name__, "", "" ) + raise MyException( f"readDataIPinfo() wrong args " ) + + if len( mark ) == 0: + mark_1 = '[Spectrum]' + mark_2 = '[DATA]' + + mark_1 = '['+ mark[0] +']' + + # 预备返回数据结构 + res_data = [] + res_time = [] + res = [] + tags = ['InclX' , 'InclY', 'Pressure'] + + with fpath.open('r') as f_handle: + rflag = 0 + for line in f_handle: + if mark_1 in line and rflag == 0: + log.debug(f" find {mark_1}", __name__, "", "" ) + rflag += 1 + continue + if mark_1 in line and rflag > 5: + log.debug(f" find {mark_1} end", __name__, "", "" ) + rflag = 0 + continue + # 获取时间 + if rflag == 1 : + data = line.strip('\n').strip(' ').split('=') + if data[0].strip() == "DateTime" : + log.debug(f" find {mark_1} -> DateTime {data[1]} ", __name__, "", "" ) + time_ = data[1].strip() + rflag += 1 + continue + if rflag == 2: + if "[" not in line : + data = line.strip('\n').strip(' ').split('=') + # RAMSES 数据通过MethodName 获得传感器序列号 + if data[0].strip() in tags : + res.append( data[1].strip(' ') ) + rflag += 1 + if rflag == 5: + res_time.append(time_) + res_data.append(res) + rflag = 0 + pass + callback(res_time,res_data) + return (res_time,res_data) + + @staticmethod + def readSamSNFromIni( fpath:Path ): + """ + @description : 依据文件路径获得samsn, 兼容 SAM SAMIP传感器 + """ + + with fpath.open('r') as f_handle: + for line in f_handle: + # IDDevice = SAM_85AF + ln = line.strip('\n').strip(' ').split('=') + if ln[0].strip() == "IDDevice": + if ln[1].strip().split('_')[0] == "SAM": + return ln[1].strip().split('_')[1] + return None + pass + + @staticmethod + def readIPCalFromIni( fpath:Path ): + """ + @description : 依据文件路径获得IPCal IP标定信息 + """ + ipcal = {} + with fpath.open('r') as f_handle: + flag = 0 + for line in f_handle: + # IDDevice = SAM_85AF + ln = line.strip('\n').strip(' ').split('=') + + if flag == 0 and ln[0].strip() == "IDDevice": + if ln[1].strip().split('_')[0] == "IP": + ipcal["IPSN"] = ln[1].strip().split('_')[1] + flag += 1 + continue + + if flag == 1 and "[" in line: + flag += 1 + continue + + if flag == 2: + if "[" in line: + flag = flag + 1 + + if not "[" in line: + line_ = line.strip('\n').strip(' ').split('=') + ipcal.update({line_[0].strip():line_[1].strip()}) + + if flag == 3: + break + + return ipcal + + @staticmethod + def readSAMCalFromIni( fpath:Path ): + """ + @description : 依据文件路径获得SAMCal SAM标定信息 + """ + samcal = {} + with fpath.open('r') as f_handle: + flag = 0 + for line in f_handle: + # IDDevice = SAM_85AF + line_ = line.strip('\n').strip(' ').split('=') + if flag == 0 and line_[0].strip() == "IDDevice": + if line_[1].strip().split('_')[0] == "SAM": + samcal["SAMSN"] = line_[1].strip().split('_')[1] + flag += 1 + continue + + if flag == 1 and "[" in line: + flag += 1 + continue + + if flag == 2: + if "[" in line: + flag = flag + 1 + + if not "[" in line: + line_ = line.strip('\n').strip(' ').split('=') + samcal.update({line_[0].strip():line_[1].strip()}) + + if flag == 3: + break + return samcal + + @staticmethod + def readFileSNbyIDDevice(fpath:Path): + sn_from_file = '' + with fpath.open('r') as f_handle: + rflag = 0 + for line in f_handle: + if '[Spectrum]' in line and '[END]' not in line and rflag == 0: + rflag += 1 + pass + if rflag == 1: + data = line.strip('\n').strip(' ').split('=') + # RAMSES 数据通过MethodName 获得传感器序列号 + if data[0].strip() == "IDDevice": + sn_from_file = data[1].strip(' ').split('_')[1] + break + return sn_from_file + + @staticmethod + def readDatafileSNbyMethodName(fpath:Path): + ''' + SAMIP SAM的数据文件中, MethodName对应的值都是 SAM_{SAMSN} + 读SAMIP的数据,建议用 {SAMSN}的处理 + ''' + sn_from_file = '' + with fpath.open('r') as f_handle: + rflag = 0 + for line in f_handle: + if '[Spectrum]' in line and '[END]' not in line and rflag == 0: + rflag += 1 + pass + if rflag == 1: + data = line.strip('\n').strip(' ').split('=') + # RAMSES 数据通过MethodName 获得传感器序列号 + if data[0].strip() == "MethodName": + sn_from_file = data[1].strip(' ').split('_')[1] + break + return sn_from_file + + +if __name__ == "__main__": + log.info(f"******** main read *********", __name__, "", "") + + # path = PurePath() + # print(path) + + pass + + # def read_data_sn(self,fpath): + # with open(fpath, 'r') as f_handle: + # rflag = 0 + # for line in f_handle: + # if '[Spectrum]' in line and '[END]' not in line and rflag == 0: + # rflag += 1 + # pass + # if rflag == 1 : + # data = line.strip('\n').strip(' ').split('=') + # if data[0].strip() == "IDDevice" : + # return data[1].strip(' ').split('_')[1] + # pass + + # def read_data1(self,fpath): + # with open(fpath, 'r') as f_handle: + # rflag = 0 + # res = [] + # res_time = [] + # res_data = [] + + # for line in f_handle: + # if '[Spectrum]' in line and '[END]' not in line and rflag == 0: + # rflag += 1 + # pass + + # if '[Spectrum]' in line and '[END]' in line : + # rflag = 0 + # pass + + # # 获取时间 + # if rflag == 1 : + # data = line.strip('\n').strip(' ').split('=') + # if data[0].strip() == "DateTime" : + # time_ = data[1].strip() + # rflag += 1 + + # if rflag == 2 and ( '[DATA]' in line or '[Data]' in line ) : + # rflag += 1 + + # if rflag == 3 : + # data = line.strip('\n').strip(' ').split(' ') + + # if int(data[0].strip()) == 0 : + # pass + # else: + # # if data[1] != "-NAN" and data[1] != "NAN" and data[1] != "+NAN" : + + # res.append( float(data[1] ) ) + # if rflag == 3 and '[' in line and ']' in line: + # rflag += 1 + # if rflag == 4: + # # 这个地方 + # # self. __save_absorbance_data( res_time, res, sn ) + # res_time.append( time_ ) + # res_data.append( res ) + # # ?? 可以调用函数直接处理 + # # DealResult.deal_one_data(time_,res) + # res = [] + # rflag = 0 + # # return [['data']] + # pass + + + # def read_data_column( self,fpath, column=1 ): + # column_ = 1 + # if int(column_) : + # column_ = int(column_) + # pass + # else: + # pass + + # with open(fpath, 'r') as f_handle: + # rflag = 0 + # res = [] + # res_time = [] + # res_data = [] + + # for line in f_handle: + # if '[Spectrum]' in line and '[END]' not in line and rflag == 0: + # rflag += 1 + # pass + + # if '[Spectrum]' in line and '[END]' in line : + # rflag = 0 + # pass + + # # 获取时间 + # if rflag == 1 : + # data = line.strip('\n').strip(' ').split('=') + # if data[0].strip() == "DateTime" : + # time_ = data[1].strip() + # rflag += 1 + + # if rflag == 2 and ( '[DATA]' in line or '[Data]' in line ) : + # rflag += 1 + + # if rflag == 3 : + # data = line.strip('\n').strip(' ').split(' ') + + # if int(data[0].strip()) == 0 : + # pass + # else: + # res.append( float(data[column_] ) ) + # if rflag == 3 and '[' in line and ']' in line: + # rflag += 1 + # if rflag == 4: + # # 这个地方 + # # self. __save_absorbance_data( res_time, res, sn ) + # # res_time.append( time_ ) + # # res_data.append( res ) + # # ?? 可以调用函数直接处理 + # return time_, res + + # # return [['data']] + # pass + + # def read_data_wl( self, fpath ): + # wl = self.read_data_column( fpath, column=0 ) + # pass + + # def get_cal_file(self,sn): + # sn_ = sn + # file_ = {} + # if open( os.path.join( CAL_PATH, "SAM_"+sn_+"ini" ),"r"): + # file_ = { + # "type": "SAM", + # "back":"Back_SAM_", + # "cal":"Cal_SAM_", + # "calaq":"CalAQ_SAM_", + # "ini":"SAM_", #ini + # "immersion":"immersion_factors_Lu.DAT" + # } + # return file_ + # if open( os.path.join( CAL_PATH, "SAMIP_"+ sn_ +"_ALL.ini" ),"r"): + # file_ = { + # "type": "SAMIP", + # # "back":"Back_SAM_" + sn_ + CAL_EXT_NAME, + # # "cal":"Cal_SAM_" + sn_ + CAL_EXT_NAME, + # # "calaq":"CalAQ_SAM_" + sn_ + CAL_EXT_NAME, + # "ini":"SAMIP_"+sn+"_ALL.ini", #ini + # "immersion":"immersion_factors_Lu.DAT" + # } + # if file_['ini'] : + # samsn = self.read_ini(file_['ini'], "SAMSN") + # file_.update("cal", "Cal_SAM_" + samsn + CAL_EXT_NAME) + # file_.update("calaq", "CalAQ_SAM_" + samsn + CAL_EXT_NAME) + # file_.update("back", "Back_SAM_" + samsn + CAL_EXT_NAME) + # return file_ + # return + + # def read_cal(self,sn): + # # SAM SAMIP 不同 + # self.cal_data= {} + # # 1. 依据序列号读 ini, 判读是否有文件 SAM_8578.ini SAMIP_50BB_ALL.ini + # self.calfile = self.get_cal_file(sn) # 其中sam的是错误的 + + # # 2 读ini 文件错误 + + # cal_data = {} + # for key in self.calfile: + # if key == "immersion" : + # path_ = os.path.join( CAL_PATH, self.calfile[key] ) + # content_ = [self.read_data_column( path_, column=0 )[1]] + # content_.append( self.read_data_column( path_, column=0 )[1] ) + # cal_data.update({key:content_}) + + # if key == "cal" : + + # path_ = os.path.join( CAL_PATH, self.calfile[key] ) + # content_ = self.read_data_column( path_, column=0 ) + # cal_data.update({key:content_}) + + # if key == "back" : + # path_ = os.path.join( CAL_PATH, self.calfile[key] ) + # content_ = self.read_data_column( path_, column=0 ) + # cal_data.update({key:content_}) + + # if key == "calaq" : + # path_ = os.path.join( CAL_PATH, self.calfile[key] ) + # content_ = self.read_data_column( path_, column=0 ) + # cal_data.update({key:content_}) + + # if key == "ini" : + # cal_data.update({ "samcal" : self.read_ini( os.path.join( CAL_PATH, self.calfile[key] ), "SAMCAL" ) }) + # cal_data.update({ "ipcal" : self.read_ini( os.path.join( CAL_PATH, self.calfile[key] ), "IPCAL" ) }) + + # pass + + # DealResult.deal_cal(cal_data) + + # pass + + # def read_data(self,fname,lst): + # """ + # @description : 线程守护读取数据文件 ，循环读取 + # 考虑一次读取多组,只读一列的情况 + # sn ,begin,end, 指定读的列 + # 返回：[ [[第一组时间],[第二组时间] .... ], [ [第一组data],[第二组data] ...] ] + # RAMSES 要考虑 : InclX = -6.42, InclY = -6.5625 , Pressure = 6.61394049205538 + # 需要读一组波长出来 + # """ + # log.info( ":::::::::::: Class : %s -> Function :__read_data " % ( __name__, ) ) + # log.info( " __read_data : lst %s" % ( lst, ) ) + # sn = lst[0] + # begin = lst[1] + # end = lst[2] + # column = lst[3] + # samip_sn= lst[4] + + # # 检查 sn--- 不检查sn ，读数据就 成csv + + # # 读取一组数据的0列作为波长？？？ + + # # if is_sn_ok : + + # log.info( "正在处理文件 .... " + fname ) + # with open(fname, 'r') as f_handle: + # time_ = "" + # rflag = 0 + # intlx = 0.0 + # intly = 0.0 + # pressure = 0.0 + # res = [] + # res_time = [] + # res_data = [] + + # for line in f_handle: + # if rflag == 0 and '[Spectrum]' in line and '[END]' not in line : + # rflag += 1 + # pass + + # if '[Spectrum]' in line and '[END]' in line : + # rflag = 0 + # pass + + # # 获取时间, InclX, InclY ,Pressure + # if rflag == 1 : + # data = line.strip('\n').strip(' ').split('=') + # if data[0].strip() == "DateTime" : + # time_ = data[1].strip() + # rflag += 1 + # # InclX, InclY ,Pressure + + # if rflag == 2: + # data = line.strip('\n').strip(' ').split('=') + + # if data[0].strip() == "InclX" : + # intlx = data[1].strip() + + # if data[0].strip() == "InclY" : + # intly = data[1].strip() + + # if data[0].strip() == "Pressure" : + # pressure = data[1].strip() + + + # # DATA set 结束添加数据 + # if rflag == 4 and '[DATA]' in line and '[END]' in line: + # # time.sleep(10) + # res_time.append( [time_,intlx,intly,pressure] ) + # res_data.append( res ) + # res = [] + # rflag += 1 + + # if rflag == 3 : + # data = line.strip('\n').strip(' ').split(' ') + # # print("rflag %s +++++++++++++++++++++++++++++++++ %s" %(rflag,data) ) + # if data[0] != "0" and data[1] != "-NAN" and data[1] != "NAN" and data[1] != "+NAN" : #去掉第一行 0， 对应的积分时间 2^（n+1） + # # print("append %s -- %s" %(data[0],data[1]) ) + # res.append( float( data[column-1] ) ) + # if data[0] != "0" and float(data[0]) > 950 : + # # print("> 950... %s %s" %(data[0],data[1]) ) + # rflag += 1 + + # # 处理data 部分 + # if rflag == 2 and '[DATA]' in line and '[END]' not in line: + # # print("rflag %s +++++++++++++++++++++++++++++++++ data+1 " %(rflag,) ) + # rflag += 1 + + + # if rflag == 5: + # DealResult.deal_ramses_data(time_, [[intlx,intly,pressure], res]) + # # 这个地方 + # # self. __save_absorbance_data( res_time, res, sn ) + # # print("rflag 4 ......................") + # # res_time.append( [time_,intlx,intly,pressure] ) + # # print(" append time %s " %res_time) + # # res_data.append( res ) + # res = [] + # rflag = 0 + # intlx = 0.0 + # intly =0.0 + # pressure =0.0 + # # else: + # # return [] + + # log.info( "已经转换完文件，正在保存... " + fname ) + + # # 分析处理数据，还是返回原始数据 或空 ？？ + # log.info( " Function :__read_data %s - %s " % ( res_time[0] ,res_data[0] ) ) + + + # # # 光强保存文件路径 + # # fpath = os.path.join(INTENSITY_PATH, samip_sn + SAVE_EXT_NAME ) + # # fpath_IP = os.path.join(INTENSITY_PATH, samip_sn + "_IP" + SAVE_EXT_NAME ) + + # # for i in range ( len(res_time) ) : + # # # print("1111111111..............%s %s " %(res_time[i],res_time[i][0]) ) + # # # time.sleep(2) + # # ProcessFile.save_time_list( str(res_time[i][0]) , res_data[i], TOKEN, fpath ) + # # # 不是samip 不保存 IP 文件 + # # if sn != samip_sn: + # # ProcessFile.save_time_list( str(res_time[i][0]) , res_time[i][1:], TOKEN, fpath_IP ) + + + # # log.info( "保存结束 ... " + fname ) + + # # 返回 ok 表示文件处理结束 + # return + # pass + + # def read_ini(self,fname ,type_): + # """ + # @description : 三个不同任务， SAM　SN， SAM ATTR, IP ATTR + # 分别记为 SAMSN SAMCAL IPCAL + # """ + # log.info( " RAMSES __read_ini -> lst -> %s " % type_) + # typ = type_ + # data = [] + + # if typ == "SAMSN" : + + # with open(fname, 'r') as f_handle: + # for line in f_handle: + # # IDDevice = SAM_85AF + # ln = line.strip('\n').strip(' ').split('=') + # if ln[0].strip() == "IDDevice" : + # if ln[1].strip().split('_')[0] =="SAM" : + # return ln[1].strip().split('_')[1] + # pass + + # if typ == "SAMCAL" : + # samcal= {} + # with open(fname, 'r') as f_handle: + # flag = 0 + # for line in f_handle: + # # IDDevice = SAM_85AF + # line_ = line.strip('\n').strip(' ').split('=') + # if flag == 0 and line_[0].strip() == "IDDevice" : + # if line_[1].strip().split('_')[0] =="SAM" : + # samcal["SAMSN"] = line_[1].strip().split('_')[1] + # flag += 1 + + # if flag == 2 and "[END]" in line: + # flag += 1 + + # if flag == 2: + # # print(" 1 %s %s %s" % (fname,line_[0],line_[1])) + # samcal[line_[0]] = line_[1] + + # if flag == 1 and line_[0].strip() == "[ATTRIBUTES]" : + # flag += 1 + + # if flag == 2 and "[END]" in line: + # flag += 1 + + # return samcal + + # pass + + # if typ == "IPCAL" : + # ipcal= {} + # with open(fname, 'r') as f_handle: + # flag = 0 + # for line in f_handle: + # # IDDevice = SAM_85AF + # ln = line.strip('\n').strip(' ').split('=') + + # if flag == 0 and ln[0].strip() == "IDDevice" : + # if ln[1].strip().split('_')[0] =="IP" : + # ipcal["IPSN"] = ln[1].strip().split('_')[1] + # flag += 1 + + # if flag == 2 and "[END]" in line: + # flag += 1 + + # if flag == 2: + # ipcal[ln[0]] = ln[1] + + # if flag == 1 and ln[0].strip() == "[ATTRIBUTES]" : + # flag += 1 + + # return ipcal + + # return + # pass + + # def read_raw( self, fname , lst, uiraw): + # """ + # @description : 三个不同任务， SAM　SN， SAM ATTR, IP ATTR + # 对波长进行标定处理 + # 并存入文件 + # λ(N) = C0s + C1s·N + C2s· N2 + C3s·N3 + # """ + + # sep = uiraw["1"] + # air_water = int(uiraw["2"]) + + # with open(fname, 'r') as f_handle: + # sn = "" + # wl = "" + # is_samip = 0 + # # data = data[2:] + # # cal_data = lst[sn] + # wl_cal = [] + # cal_data = [] + # intg_time = 128 + # ip_cal = [] + # cal = [] + # calaq = [] + # b0 = [] + # b1 = [] + + # for line in f_handle: + # data = line.split(",") + # sn_d = data[0] + + # if sn_d != sn : + # sn = sn_d + # intg_time = int( data[1] ) + # cal_data = lst[sn] + # # 判断sam SAMIP + # if cal_data[0][0] != cal_data[0][1] : + # is_samip = 1 + + # wl_cal = cal_data[1] + # ip_cal = cal_data[2] + + # if air_water == 1 : + # cal = cal_data[3] + # else: + # cal = cal_data[2] + + # b0 = cal_data[4] + # b1 = cal_data[5] + + # for i in range(1,256): + # temp = float(wl_cal["C0s"]) + \ + # float(wl_cal["C1s"]) * i + \ + # float(wl_cal["C2s"]) * i * i + \ + # float(wl_cal["C3s"]) * i *i *i + # wl.append(temp) + + + + # # 处理数据 + # data = data[2:] + # intensity = [] + # Cn = [] + + # for i in range( 255 ): + # # NAN 为 0 + # # if cal[i] = "+NAN": + # # intensity.append(0) + # # else: + # Mn = data[i] / 65535 + # Bn = float( b0[i] ) + float( b1[i] ) * intg_time / 8192 #积分时间比 + # Cn.append(Bn - Mn) + + # offset = 0 # DarkPixelStart = 237 DarkPixelStop = 254的平均值. 数组中236-254 + # for i in range( int( wl_cal['DarkPixelStart']) -1 , int( wl_cal['DarkPixelStop']) ): + # offset = offset + float( Cn[i] ) + # offset = offset/ ( int( wl_cal['DarkPixelStop']) - int( wl_cal['DarkPixelStart']) +1 ) + + # for i in range( 255 ): + # Dn = Cn[i] - offset + # En = 8192 * Dn / intg_time + + # # ???? Sn 来自 cal calaq 文件的部分 + # Sn = cal[i] + # intensity.append( En / Sn ) + + # # 还需要处理传感器的 IP + # if is_samip : + # # ip 数据过来后的格式 + # pass + + # return intensity + + # # 数据分别写到文件 diff --git a/receive.py b/receive.py new file mode 100644 index 0000000..7b11f98 --- /dev/null +++ b/receive.py @@ -0,0 +1,466 @@ +#!/usr/bin/env python +# coding:utf-8 + +''' +# 因为图片帧不是必须，必须按帧处理数据。 可能摄像头坏掉，没有传感器数据？？ +# 按帧处理数据，必须在每帧接收完毕判断，数据是否完整， 完整则进一步处理 ！！！！！！！！！！！！ +# 时间作为目录 +''' +import socket +import socketserver +from socketserver import TCPServer,ThreadingMixIn +import threading +# import datetime +import time +# import os +import struct +from tools.mypath import MyDir +from tools.mylogger import log +from pathlib import Path,PurePath +from myconfig import DATA_DIR,DeviceType +from handheld import Handheld + + +IP = "" +PORT = 7887 +ADDRESS = (IP, PORT) # 绑定地址 + +# LOGGING_LEVEL = logging.DEBUG +# LOGGING_LEVEL = logging.INFO +# LOGGING_LEVEL = logging.WARNING + +DATA_FRAME_HEAD = b'\x11\x13\x55\xaa' +DATA_FRAME_TAIL = b'\xff\xd9' +PIC_BEGIN_BYTES = b'\xff\xd8' + +# 连接超时 +TIMEOUT_SECOND = 8 * 3600 + +# 连接线程池 +conn_pool = [] + +# save_path = Path + +class MyTCPServer(TCPServer): + def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True, cfg=None, retrieve=None): + self.cfg = cfg + self.retrieve = retrieve + TCPServer.__init__(self, server_address, RequestHandlerClass, bind_and_activate=True ) + +class MyThreadingTCPServer(ThreadingMixIn, MyTCPServer): pass + +class MyException(Exception): + def __init__(self, message="自定义异常"): + self.message = message + + +class illumination_sensor: + def __init__(self, socket: socket.socket) -> None: + self.__buf = b'' + self.__head = { + # "id" : -1, # 设备id + # 'type' : -1, # 类型 信息 传感器 图片 + # 'num' : -1, # 光学传感器的第几次测量 + # 'con' : -1, # 总的测量序号 + # 'size' : -1, # 字节大小 + # 'packet_con' : -1, # 第几帧 + # 'packet_all' : -1, # 总帧数 + # 'head' : b'', # 帧头 + # 'payload' : b'' # 具体内容 + } + self.state = 0 + self.socket = socket + self.id = 0 + # self.is_data_complete = { + # "info_frame": False, + # 'sensor_frame': False, + # 'pic_frame': False + # } + # self.data = { + # "info_frame": {}, + # 'sensor_frame': {}, + # 'pic_frame': {} + # } + self.timeout_base = int(time.time()) + + def set_id(self, id) -> None: # 对应设备ID + self.id = id + + # def set_socket(self,socket:socket.socket) -> socket.socket: + # tmp = self.socket + # self.socket=socket + # return tmp + + def read_buf(self, size: int) -> bytes: + if size > self.__buf.__len__(): + return b'' + ret = self.__buf[0:size] + self.__buf = self.__buf[size:] + return ret + + def write_buf(self, buf: bytes) -> None: + id = self.id + len = buf.__len__() + # logging.info(f'Received ID:{id} Size:{len}') + self.__buf = self.__buf+buf + + def get_buf_size(self) -> int: + return self.__buf.__len__() + + def back_bytes(self, buf: bytes) -> None: + self.__buf = buf+self.__buf + + def reset_head(self) -> None: + self.__head = {} + # self.__head['id'] = -1 + # self.__head['type'] = -1 + # self.__head['num'] = -1 + # self.__head['con'] = -1 + # self.__head['size'] = -1 + # self.__head['packet_con'] = -1 + # self.__head['packet_all'] = -1 + # self.__head['head'] = b'' + # self.__head['payload'] = b'' + + def reset_data(self) -> None: + self.data['info_frame']: dict = {} + self.data['sensor_frame']: dict = {} + self.data['pic_frame']: dict = {} + + def decode(self) -> dict: + if self.__head == {}: + while self.get_buf_size() >= 15: + if self.read_buf(1) != b'\x11': + continue + c = self.read_buf(1) + if c != b'\x13': + self.back_bytes(c) + continue + c = self.read_buf(1) + if c != b'\x55': + self.back_bytes(c) + continue + c = self.read_buf(1) + if c != b'\xaa': + self.back_bytes(c) + continue + head = self.read_buf(11) + head_list = struct.unpack('> 4 + self.__head['num'] = head_list[1] & 0x0f + self.__head['con'] = head_list[2] + self.__head['size'] = head_list[3] + self.__head['packet_con'] = head_list[4] + self.__head['packet_all'] = head_list[5] + self.__head['head'] = b'\x11\x13\x55\xaa'+head + break + + if self.__head != {}: + payload = self.read_buf(self.__head['size']) + if payload != b'': + self.__head['payload'] = payload + data = self.__head.copy() + self.__head = {} + self.id = data['id'] + return data + + return {} + +class DealData: + """ + @description : 调用AWRAMS类处理数据 + @param : + @Returns : + """ + + def __init__(self) -> None: + self.device_id = None + self.devie_type = DeviceType.SURFACE.name + self.measure_id = None + self.cfg = {} + self.awrams = None + pass + + def deal(self, id: int, con: int, cfg:dict , retrieve) -> None: # 取字典中的 payload + log.info(f" 接收到数据开始处理数据 device_id {id} ") + self.device_id = id + self.measure_id = con + if self.device_id is None: + self.device_id = id + if self.cfg == {}: + self.cfg = cfg + # self.cfg = cfg.get(self.device_id) + if self.awrams is None: + self.awrams = Handheld() ##处理数据 + self.awrams.setSyscfg(self.cfg) + self.awrams.setRetrieve(retrieve) + self.awrams.setDeviceID(self.device_id) + self.awrams.setMeasureID(self.measure_id) + path_tuple = ( "data", str(id), str(con) ) + self.awrams.setOldFolder( path_tuple ) + self.awrams.getInfoDict( ) + self.awrams.transferFromOldFolder() + self.awrams.deleteOldFolder() + self.awrams.dealOneMeasurement_Handheld() + log.info(f" Complete Dealing one group.") + # self.awrams.readOneFolder( ) + + + @staticmethod + def read_bin(fpath:Path): + ret = None + if fpath.exists() == False: + log.warning(f"not find file: {fpath} ") + return ret + ret = fpath.read_bytes() + # with open( fpath, 'rb') as file: + # ret = file.read() + # return ret + return ret + pass + + @staticmethod + def decode_info(info: bytes) -> dict: + ret = {} + try: + temp = struct.unpack( "None: + temp_str = "" + for key, value in info_dict.items(): + temp_str = temp_str + key + " : " + str(value) + "\n" + with open(fpath, "w+") as f: + f.write(temp_str) + + ret = None + if fpath.exists() == False: + log.info(f"not find file: {fpath} ") + return ret + with open(fpath, 'rb') as file: + ret = file.read() + return ret + return ret + pass + + @staticmethod + def check_spectrum_data(dst_dir:Path): + # 判断目录下是否有 0.bin ...15.bin 文件 + sensor_file_list = dst_dir.glob( '*[0-9].bin' ) + fname_without_ext = [] + for fl in sensor_file_list: + temp = fl.stem + if not temp.isdigit: + log.warning( f" {dst_dir} 目录光谱文件的文件名 {temp} 不为数字,type:{type(temp)}，请检查异常" ) + return False + fname_without_ext.append( int(temp) ) + + if len(fname_without_ext) ==0: + log.warning( f" {dst_dir} 目录没有发现光谱文件，请检查异常" ) + return False + + # 排序，然后检查是否有遗漏项 + fname_without_ext.sort() + for i in fname_without_ext: + if fname_without_ext[i] !=i: + log.warning( f" {dst_dir} 目录,序号{i}光谱文件的文件名没有发现，请检查异常" ) + return False + + return False + pass + + @staticmethod + def calibrate_spectrum_data(dst_dir): + ''' + 用ini back 等文件获得标定后数据 + ''' + log.info("calibrate_spectrum_data.... ") + pass + + @staticmethod + def retrieve_data(dst_dir): + ''' + 反演遥感反射率等参数 + ''' + log.info(" retrieve_data.... ") + pass + + def save(self,data: dict) -> None: + log.info(f"save .....to first dir {str(data['con'])} - type:{data['type']} - num {data['num']}") + # 路径 传感器id/测量序号（唯一） -- 处理数据时候改时间 + saveDir = DATA_DIR.joinpath(str(data['id']), str(data['con']) ) + if saveDir.exists() == False: + saveDir.mkdir(parents=True) + + if data['type'] == 0: + log.debug( f" {data['type']} - {data['num']}") + fpath = saveDir.joinpath("info.bin") + fpath.write_bytes( data['payload'] ) + elif data['type'] == 1: + log.debug( f" {data['type']} - {data['num']}") + senor_path = saveDir.joinpath( "sensor.bin" ) + with open( senor_path, "ab+") as f: + f.write(data['payload']) + pass + elif data['type'] == 2: + pass + else: + pass + + +# class MyServer(socketserver.BaseRequestHandler): +class MyServer(socketserver.BaseRequestHandler): + + def setup(self) -> None: + log.debug(f"retrieve {self.server.retrieve}",__name__, "", "" ) + self.cfg =self.server.cfg + self.retrieve =self.server.retrieve + self.sk: socket.socket = self.request + self.sensor = illumination_sensor(self.request) + self.dealData = DealData() + self.begin_time = time.time() + conn_pool.append(self.client_address) + pass + + def handle(self) -> None: + + log.info('... connected from {}'.format(self.client_address),'__name__') + while True: + # self.request.recv 方法接收客户端发来的消息 + try: + data_byte = self.sk.recv(1000) + except ConnectionResetError as e: + log.warning( + f"recv ConnectionResetError, client {self.client_address} had close .....") + break + except: + log.warning(" sk.recv(1000) exception .....") + pass + + log.debug(len(data_byte)) + + # 客户端主动关闭连接后，会不断接收b'', 所以跳出循环，运行到程序结尾，自动关闭线程 + if data_byte == b'': + log.info( + " b'' is received , maybe client {self.client_address} had close. ") + self.sk.close() + if hasattr(self, "sensor"): + del self.sensor # 销毁对象 + break + else: + break + continue + else: + self.sensor.write_buf(data_byte) + + data_byte = b'' # 客户端掉线后以前数据不长居内存 + + try: + data_ = self.sensor.decode() + except MyException as e: + log.warning(e) + break + except Exception as e: + log.warning("decode data 出现异常 ") + log.warning(e) + break + + if data_ != {}: + id = data_['id'] + data_code = data_['type'] + log.info(f'Received From ID:{id} DATA CODE:{data_code}') + + # 保存当前数据 + self.dealData.save(data_) + head = data_['head'] + log.info(f'Head :{head}') + + # 返回head给服务器 + self.sk.send(data_['head']) + + # 判断是否是最后一帧, 修改目录为时间格式并 + if data_["packet_con"] == data_["packet_all"]: + log.info(f'最后一帧数据已经收到并保存') + # id 为传感器测量id ,con 测量序号 + self.dealData.deal(data_['id'], data_["con"], self.cfg, self.retrieve) + pass + + if time.time() - self.begin_time > TIMEOUT_SECOND: + log.info(f'Received data timeout') + break + + def finish(self) -> None: + # 什么时候执行 finish + # 执行handle 出现意外错误时执行 finish，或handle break时执行 finish + # 关闭连接 + if self.sk: + self.sk.close() + + # conn_pool 连接池销毁 + if self.client_address in conn_pool: + conn_pool.remove(self.client_address) + + log.info( + f"finish(): stop one socket from client {self.client_address} ") + + + +if __name__ == '__main__': + server_ = socketserver.ThreadingTCPServer(ADDRESS, MyServer) + log.info('listening...........') + + try: + server_.serve_forever() + except KeyboardInterrupt: + log.info(" Ctrl+C 退出主程序 ") + server_.server_close() + except Exception as e: + log.info(" 系统异常， 如下: \n ") + log.info(e) + + # 有闪退，可以用线程将server_.serve_forever 包起来 + # threading.Thread(target=server_.serve_forever).start() diff --git a/retrieve.yml b/retrieve.yml new file mode 100644 index 0000000..ba0f0ee --- /dev/null +++ b/retrieve.yml @@ -0,0 +1,4 @@ +beginWL: 350 +endWL: 950 +interval: 1 +rowFactor: 0.026 diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/myexception.py b/tools/myexception.py new file mode 100644 index 0000000..1b526c0 --- /dev/null +++ b/tools/myexception.py @@ -0,0 +1,11 @@ +class TCPIPException(Exception): + def __init__(self, message="TCPIP 自定义异常"): + self.message = "TCPIP exception : " + message + +class SerialException(Exception): + def __init__(self, message="Serial 自定义异常"): + self.message = "serial exception : " + message + +class MyException(Exception): + def __init__(self, message=" 自定义异常"): + self.message = "自定义异常: " + message \ No newline at end of file diff --git a/tools/mylogger.py b/tools/mylogger.py new file mode 100644 index 0000000..9c85ec9 --- /dev/null +++ b/tools/mylogger.py @@ -0,0 +1,285 @@ +# coding=utf-8 +''' +单例模式日志 -- 使用一次关闭 handler +这种方法优缺点： +缺点： 输出的format 需要自己定义 ，并过滤 + 过滤要看是否以什么开头，或包含什么 +优点： 不占用文件资源，占用系统资源小 +调用 log.info( ) log.error() +''' + +import logging +import logging.handlers +import os +import time +import threading +# from config import LOG_PATH,LOG_LEVEL,LOG_ENABLED,LOG_FORMAT, \ +# LOG_TO_CONSOLE,LOG_TO_FILE + +MY_LOGGER_NAME = "DefaultLogger" +LOG_ENABLED = True # 是否启用日志 +LOG_TO_CONSOLE = True # 是否启用控制台输出日志 +LOG_TO_FILE = False # 是否启用文件输出 +LOG_COLOR_ENABLE = True # 是否启用颜色日志 + +LOGGER_DIR = "logs" +LOGGER_PATH = os.path.join( os.path.dirname(__file__), LOGGER_DIR ) +LOGGER_FILENAME = os.path.join( LOGGER_PATH, 'logs.log' ) + +""" +logging.INFO , logging.DEBUG , logging.WARNING , logging.ERROR , +""" +LOG_LEVEL = logging.INFO # 日志等级DEBUG INFO WARNIG ERROR +# LOG_LEVEL = logging.DEBUG +# LOG_LEVEL = logging.WARNING + +""" +# LOG_FORMAT = " %(name)s - %(module)s - %(filename)s - %(lineno)d | %(levelname)s : %(message)s" +# LOG_FORMAT = "%(levelname)s - %(asctime)s - process: %(process)d - threadname: %(thread)s " \ +# "- %(filename)s - %(funcName)s - %(lineno)d - %(module)s " \ +# "- %(message)s " +# LOG_FORMAT = "%(asctime)s - %(thread)s " \ +# "- %(levelname)s ::: %(message)s " +# '[%(asctime)s] |%(thread)s| %(levelname)-6s: %(message)s' +# fm = '%(levelname)：%(levelno)s：%(name)s：%(funcName)s：%(asctime)：%(pathname)：%(filename)：%(module)：%(thread)：%(threadName)' +# 此处日志颜色，修改日志颜色是通过 Filter实现的 +""" +LOG_FORMAT = '%(levelname)s\t[%(asctime)s] %(package)s:%(classname)s:%(funcname)s \t>> %(message)s' + +""" +# 此处日志颜色，修改日志颜色是通过 Filter实现的 +""" +LOG_FORMAT_COLOR_DICT = { + 'ERROR' : "\033[31mERROR\033[0m", + 'INFO' : "\033[36mINFO\033[0m", + 'DEBUG' : "\033[1mDEBUG\033[0m", + 'WARN' : "\033[33mWARN\033[0m", + 'WARNING' : "\033[33mWARNING\033[0m", + 'CRITICAL': "\033[35mCRITICAL\033[0m", +} + +""" +# Filter 用法， 以package class function 过滤 __package__ __class__ +# log.error( f"{__package__}::{__class__.__name__}::{sys._getframe().f_code.co_name} >> ") +# log.error( f"PacakgeName::ClassName::FunctionName:: ") +# LOGGER_FILTER_PACKAGE=[] 为空, 则Filter不起作用 +# 不为空，则只显示定义的报名 +# LOGGER_FILTER_CLASS=[] 为空, 则Filter不起作用 +# 不为空，则只显示定义的类或 +# LOGGER_FILTER_FUNCNAME=[] 为空, 则Filter不起作用 +# 不为空，则只显示定义的函数 +""" +# LOGGER_FILTER_PACKAGE = [ "test_logger" ] # 包名，文件名去 .py?? +LOGGER_FILTER_PACKAGE = [ ] +LOGGER_FILTER_CLASS = [ ] # 类名，文件名去 .py?? +# LOGGER_FILTER_CLASS = [ "LogTest" ] +# LOGGER_FILTER_FUNCNAME = [ "test1","test" ] # 函数名 +LOGGER_FILTER_FUNCNAME = [ ] +LOGGER_FILTER_LEVELNAME = [ ] # INFO DEBUG WARNING + +class PackageFilter(logging.Filter): + def __init__(self, filter_word:list = []): + self.filter_word = filter_word + pass + def filter(self, record: logging.LogRecord) -> bool: + if self.filter_word is not None: + return record.package in self.filter_word + +class ClassFilter(logging.Filter): + def __init__(self, filter_word:list = []): + self.filter_word = filter_word + pass + def filter(self, record: logging.LogRecord) -> bool: + if self.filter_word is not None: + return record.classname in self.filter_word + + pass + +class FunctionFilter(logging.Filter): + def __init__(self, filter_word:list = []): + self.filter_word = filter_word + pass + def filter(self, record:logging.LogRecord) -> bool: + if self.filter_word is not None: + return record.funcname in self.filter_word + +class LevelNameFilter(logging.Filter): + def __init__(self, filter_word:list = []): + self.filter_word = filter_word + pass + def filter(self, record:logging.LogRecord) -> bool: + if self.filter_word is not None: + return record.levelname in self.filter_word + +class ColorFilter(logging.Filter): + def __init__(self,): + pass + def filter(self, record: logging.LogRecord) -> bool: + record.levelname = LOG_FORMAT_COLOR_DICT.get(record.levelname) + return True + +class Log(object): + _instance_lock = threading.Lock() + + def __new__(cls, *args, **kwargs): + if not hasattr(Log, "_instance"): + with Log._instance_lock: + if not hasattr(Log, "_instance"): + Log._instance = object.__new__(cls) + return Log._instance + + def __init__(self, loggername = "DefaultLog" ): + # 文件命名 os.path.join()： 将多个路径组合后返回 + self.logger_filepath = LOGGER_FILENAME + self.loggername = loggername + self.level = LOG_LEVEL + + # 日志输出格式 + fm = LOG_FORMAT + self.formatter = logging.Formatter( fm ) + + # 生成记录器对象 + self.logger = logging.getLogger( self.loggername ) + self.logger.setLevel(LOG_LEVEL) + + # 日志过滤 + self.__add_filter() + + def __console(self, level, message, extra={} ): + # 添加 handler + self.__add_handler() + + # 判断日志级别 + if level == logging.INFO: + self.logger.info( message, extra=extra) + elif level == logging.DEBUG: + self.logger.debug(message,extra=extra) + elif level == logging.WARNING: + self.logger.warning(message,extra=extra) + elif level == logging.ERROR: + self.logger.error(message,extra=extra) + + # removeHandler在记录日志之后移除句柄，避免日志输出重复问题 + self.__remove_handler() + # if LOG_TO_FILE and self.file_handler: + # self.logger.removeHandler(self.file_handler) + # # 关闭打开的文件 + # self.file_handler.close() + # if LOG_TO_CONSOLE and self.stream_handler: + # self.logger.removeHandler(self.stream_handler) + # # 关闭打开的文件 + # self.stream_handler.close() + pass + + # debug < info< warning< error< critical + # debug模式 + def debug(self, message, package="Unknown", classname="Unknown", funcname="Unknown" ): + self.__console(logging.DEBUG, message, extra={"package":package, "classname":classname, "funcname":funcname} ) + # self.__remove_handler() + # info模式 + def info(self, message, package="Unknown", classname="Unknown", funcname="Unknown" ): + self.__console(logging.INFO, message, extra={"package":package, "classname":classname, "funcname":funcname} ) + # self.__remove_handler() + # warning模式 + def warning(self, message, package="Unknown", classname="Unknown", funcname="Unknown"): + self.__console(logging.WARNING, message, extra={"package":package, "classname":classname, "funcname":funcname} ) + # self.__remove_handler() + + # error模式 + def error(self, message, package="Unknown", classname="Unknown", funcname="Unknown"): + self.__console(logging.ERROR, message, extra={"package":package, "classname":classname, "funcname":funcname} ) + # self.__remove_handler() + + def __add_filter(self ): + if len( LOGGER_FILTER_PACKAGE ) > 0 : + self.logger.addFilter( PackageFilter( filter_word=LOGGER_FILTER_PACKAGE ) ) + if len( LOGGER_FILTER_CLASS ) > 0 : + self.logger.addFilter( ClassFilter( filter_word=LOGGER_FILTER_CLASS ) ) + if len( LOGGER_FILTER_FUNCNAME ) > 0 : + self.logger.addFilter( FunctionFilter( filter_word=LOGGER_FILTER_FUNCNAME ) ) + if len(LOGGER_FILTER_LEVELNAME) > 0 : + self.logger.addFilter( LevelNameFilter( filter_word=LOGGER_FILTER_LEVELNAME ) ) + + def __add_handler(self ): + if LOG_ENABLED and LOG_TO_FILE: + # 考虑使用 RotatingFileHandler TimedRotatingFileHandler防止日志过大 + # RotatingFileHandler("test", "a", 4096, 2, "utf-8") + # TimedRotatingFileHandler(filename=LOG_PATH+"thread_", when="D", interval=1, backupCount=7) + self.file_handler = logging.handlers.TimedRotatingFileHandler(filename=self.logger_filepath, when='D', interval=1, backupCount=30, encoding='utf-8') + # self.file_handler = logging.FileHandler( self.logger_filepath, encoding='utf-8' ) + self.file_handler.setFormatter( self.formatter ) + # self.file_handler.setLevel( LOG_LEVEL ) + # if LOG_COLOR_ENABLE: # 文件日志无需加彩色 + # self.file_handler.addFilter( ColorFilter( ) ) + self.logger.addHandler(self.file_handler) + + if LOG_ENABLED and LOG_TO_CONSOLE: + # 创建一个StreamHandler,用于输出到控制台 + + self.stream_handler = logging.StreamHandler() + self.stream_handler.setFormatter(self.formatter) + # self.stream_handler.setLevel( LOG_LEVEL ) + if LOG_COLOR_ENABLE: + self.stream_handler.addFilter( ColorFilter( ) ) + self.logger.addHandler(self.stream_handler) + + def __remove_handler(self ): + if LOG_TO_FILE and self.file_handler: + self.logger.removeHandler(self.file_handler) + if len(self.logger.handlers)>0: + self.logger.handlers.pop() + # 关闭打开的文件 + self.file_handler.close() + if LOG_TO_CONSOLE and self.stream_handler: + self.logger.removeHandler(self.stream_handler) + if len(self.logger.handlers)>0: + self.logger.handlers.pop() + # 关闭控制台 + self.stream_handler.close() + + def __remove_handler2(self ): + if LOG_ENABLED and LOG_TO_CONSOLE: + self.logger.removeHandler(self.stream_handler) + self.logger.handlers.pop() + # 关闭控制台 + self.stream_handler.close() + if LOG_ENABLED and LOG_TO_FILE: + self.logger.removeHandler(self.file_handler) + self.logger.handlers.pop() + # 关闭打开的文件 + self.file_handler.close() + + +log = Log( loggername = "DefaultLog") + +""" +filename: 指定日志文件名 +filemode: 和file函数意义相同，指定日志文件的打开模式，’w’或’a’ +format: 指定输出的格式和内容，format可以输出很多有用信息。显示的条目可以是以下内容： +%(levelname)： 日志级别的名字格式 +%(levelno)s： 日志级别的数字表示 +%(name)s： 日志名字 loggername +%(funcName)s： 函数名字 +%(asctime)： 日志时间，可以使用datefmt去定义时间格式，如上图。 +%(pathname)： 脚本的绝对路径 +%(filename)： 脚本的名字 +%(module)： 模块的名字 +%(thread): thread id +%(threadName)： 线程的名字 +""" + +""" +文件名行号 函数名, 要在调用的时候想办法了 +# 绝对路径 +print( __file__ ) +print( sys.argv[0] ) + +# 文件名 +print( os.path.basename(__file__) ) +print( os.path.basename(sys.argv[0]) ) + +self.__class__.__name__ +self.__class__.__name__, get_current_function_name() + +logger名 __name__ +""" \ No newline at end of file diff --git a/tools/mypath.py b/tools/mypath.py new file mode 100644 index 0000000..b8f1064 --- /dev/null +++ b/tools/mypath.py @@ -0,0 +1,265 @@ +from pathlib import PurePath, Path + +# from myconfig import NEWLINE,TOKEN,SEPARATOR +""" +""" +class MyDir(object): + """ + 操作方法:设置base tuple_dir header + 设置的是路径, 文件名要 ifNotNewFile 传入 + """ + + def __init__(self) -> None: + self.base_dir = Path() + self.dir_tuple = () + self.header = [] + self.header_str = "" + self.content = [] + self.content_str = "" + self.current_dir = None + self.current_filepath = None + pass + + def getDir(self,): + return self.current_dir + pass + + def setBaseDir(self, dir: Path): + self.base_dir = dir + self.current_dir = self.base_dir + pass + + def setDir(self, t:tuple=()): + self.dir_tuple = t + if len(self.dir_tuple) == 0 : + self.current_dir = self.base_dir + else: + self.current_dir = self.base_dir.joinpath( *t ) + pass + + def getDirFromBaseAndTuple(self, base_dir:Path, dir_tuple: tuple): + '''外部调用返回路径''' + ret_path = base_dir + t = dir_tuple + if len(t) == 0 : + ret_path = ret_path + else: + ret_path = ret_path.joinpath( *t ) + return ret_path + pass + + def setHeader(self, headerlist:list, headerSeperator: str = ";", headerinfo: str = None): + header_str = "" + if len(headerlist) == 0: + return + if headerinfo != None: + header_str = headerinfo + headerSeperator + for hl in headerlist: + header_str = header_str + str(hl) + headerSeperator + self.header_str = header_str[:-1] + pass + + def setContent(self, contentlist: list, contentSeperator: str = ";", contentinfo: str = None): + content_str = "" + if len(contentlist) == 0: + return + if contentinfo != None: + content_str = contentinfo + contentSeperator + tmp_str = "" + for cl in contentlist: + tmp_str = tmp_str + str(cl) + contentSeperator + self.content_str = content_str + tmp_str[:-1] + pass + + def newDirIfNot(self,) -> None: + # self.current_path = self.base_path.joinpath(self.path_tuple) + self.current_dir.mkdir(parents=True, exist_ok=True) + pass + + def newFileIfNot(self, fname: str) -> None: + self.newDirIfNot() + self.current_filepath = self.current_dir.joinpath(fname) + if not self.current_filepath.exists(): + with open(self.current_filepath, 'a') as f: + pass + return + pass + + def getCurrentFileSize(self,): + return self.current_filepath.stat().st_size + + def getFirstline(self,): + first_line = "" + with open(self.current_filepath, 'r') as f: # 打开文件 + first_line = f.readline() # 取第一行 + return first_line.strip('\n').strip('\r') + + def checkHeader(self,) -> int: + ''' + 返回: + 0 : 文件为空,可以直接写header + 1 : header对应上 无需处理 + -1: 需要提醒用户保存数据后,删除文件后再处理 + ''' + if self.getCurrentFileSize() == 0: + return 0 + first_line = self.getFirstline() + # print(f"firstline: {first_line}" ) + # print(f"header_str: {self.header_str}" ) + if first_line == self.header_str: + return 1 + return -1 + pass + + def writeHeader(self,) -> None: + with open(self.current_filepath, "a") as f: + f.write(self.header_str) + return None + pass + + def writeContent(self,new_line="\n") -> None: + with open(self.current_filepath, "a") as f: + f.write(new_line+self.content_str) + return None + pass + + def is_dir_empty(self, ): + '''文件夹是否为空''' + has_next = next(self.current_dir.iterdir(), None) + if has_next is None: + return True + return False + + def is_file_empty(self,): + '''文件是否为空''' + if self.current_dir.stat().st_size ==0: + return True + return False + + def deleteDir(self,): + '''文件夹是否为空''' + try: + if self.current_dir.exists(): + self.current_dir.rmdir() + except OSError as e: + raise Exception(e) + return True + + ## 其他需求 + def get_child_dir(self,) -> list: + ret = [] + tmp_dir = self.current_dir.glob("**/") + for td in tmp_dir: + if td.is_dir(): + ret.append(td.relative_to(self.current_dir)) + return ret + pass + + def get_child_dir_only(self,) -> list: + ret = [] + for d in self.current_dir.iterdir(): + if d.is_dir(): + ret.append(d.relative_to(self.current_dir)) + return ret + pass + + def get_files_from_currentdir(self, fmt:str="*/*" ) -> list: + '''fmt: * */* */*/*''' + ret = [] + tmp_dir = self.current_dir.glob(fmt) + print(tmp_dir) + for td in tmp_dir: + if td.is_file(): + ret.append(td) + return ret + pass + + def sort_dir_and_check( self, dirs:list ): + '''相对目录排序,目录最后一级''' + ret = [] + if len(dirs) == 0: + return ret + tmp = {} + tmp_lst = [] + for d in dirs: + last:str = d.parts[-1] + if last.isdigit() : + tmp.update( {last:d} ) + tmp_lst.append(int(last)) + pass + + tmp_lst.sort() + for t in tmp: + ret.append(tmp.get(str(t))) + pass + return ret + + + def sort_filepath_and_check(self, path_files:list): + '''相对目录排序,目录最后一级''' + ret = [] + if len(path_files) == 0: + return ret + tmp = {} + tmp_lst = [] + for d in path_files: + last:str = d.stem + if last.isdigit() : + tmp.update( {last:d} ) + tmp_lst.append(int(last)) + pass + + tmp_lst.sort() + for t in tmp: + ret.append(tmp.get(str(t))) + pass + return ret + + def group_and_sort_filepath(self,path_files:list): + ret = {} + # dirs_lst = [] + # len_files = len(path_files) + # if len_files == 0: + # return False + # for pf in path_files: + # pf_dir:str = pf.parts[-2] + # if pf_dir.isdigit() and int(pf_dir) not in dirs_lst: + # dirs_lst.append( int(pf_dir) ) + # dirs_lst.sort() + + + + + def check_dirs(self, dirs:list, begin:int =0, step:int=1): + '''检查目录是否从begin开始递增''' + len_dirs = len(dirs) + if len_dirs == 0: + return False + for i in range(begin,len_dirs,step) : + if dirs[i].parts[-1] != str(i) : + return False + return True + + def check_path_files(self,path_files:list,begin:int =0, step:int=1): + '''检查文件名从begin开始递增''' + len_files = len(path_files) + if len_files == 0: + return False + for i in range(begin,len_files,step) : + if path_files[i].stem != str(i) : + return False + return True + +if __name__ == "__main__": + mp = MyDir() + mp.setBaseDir(Path()) + print(mp.current_dir) + # t = ("test_dir","1","11") + t = ("test_dir", ) + mp.setDir( t ) + print(mp.current_dir) + + cd = mp.get_child_dir_only() + c = mp.sort_dir_and_check(cd) + print(cd ) + print(c ) diff --git a/tools/mytime.py b/tools/mytime.py new file mode 100644 index 0000000..3481cf8 --- /dev/null +++ b/tools/mytime.py @@ -0,0 +1,66 @@ +from datetime import datetime, timedelta +# import locale +# locale.setlocale(locale.LC_ALL, '') + +TIME_STR = "2022-06-10 16:16:16" +STD_TIME_STR_FMT = "%Y-%m-%d %H:%M:%S" # 小写y是两位年份 +CUR_TIME_STR_FMT = "%Y-%m-%d %H:%M:%S" + + +class MyTime(object): + """ + 操作方法:设置base tuple_path header + """ + + def __init__(self) -> None: + self.cur_time_str_fmt = "%Y-%m-%d %H:%M:%S" + self.std_time_str_fmt = "%Y-%m-%d %H:%M:%S" + self.cur_time_str = "" + self.std_time_str = "" + self.cur_datetime = "" + pass + + def setCurrentTimeStrFmt(self, s): + self.cur_time_str_fmt = s + pass + + def setStdTimeStrFmt(self, s): + self.std_time_str_fmt = s + pass + + def setCurrentTimeStr(self, s): + self.cur_time_str = s + self.cur_datetime = datetime.strptime(self.cur_time_str, self.cur_time_str_fmt) + pass + + def setStdTimeStr(self, s): + self.std_time_str = s + self.cur_datetime = datetime.strptime( + self.std_time_str, STD_TIME_STR_FMT) + pass + + def Current2STD(self): + # self.cur_datetime = datetime.strptime(self.cur_time_str , CUR_TIME_STR_FMT) + self.std_time_str = datetime.strftime( + self.cur_datetime, STD_TIME_STR_FMT) + pass + + def STD2Current(self, format: str): + # self.cur_datetime = datetime.strptime(self.std_time_str , STD_TIME_STR_FMT) + self.cur_time_str = datetime.strftime( + self.cur_datetime, CUR_TIME_STR_FMT) + pass + + def timeDelta(self, days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0): + self.cur_datetime = self.cur_datetime + \ + timedelta(days=0, seconds=0, microseconds=0, + milliseconds=0, minutes=0, hours=0, weeks=0) + pass + + +if __name__ == "__main__": + s = "2023-02-07 14:02:46" + mt = MyTime() + mt.setCurrentTimeStr(s) + print(mt.cur_datetime) + pass