diff --git a/.gitignore b/.gitignore index d536aaa..b996a7f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /__pycache__/ /dist/ /build/ +/tools/__pycache__/ /*/__pycache__/ /data/output/ *.log \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..33f5436 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +HANDHELD 接收及数据处理程序 + +分支：server desktop \ No newline at end of file diff --git a/Ramses.py b/Ramses.py new file mode 100644 index 0000000..a5429af --- /dev/null +++ b/Ramses.py @@ -0,0 +1,200 @@ +#! python3 +# -*- encoding: utf-8 -*- +''' +@File : Ramses.py +@Time : 2023/03/03 11:09:50 +@Author : Jim @ Yiwin +@Version : 1.0 +@Contact : jim@yi-win.com +@Desc : +@para : 23 ..07 .... 06 05 04 03 02 01 00 + ip信息不包含 +''' + +import struct +import numpy as np +from pathlib import Path +from tools.mylogger import log +from myconfig import RamsesAWRAMS, RamsesSURFACE, RamsesPROFILE, DeviceType + + +class Ramses(object): + + def __init__(self,): + """ + @description :处理Ramses的数据标定 Hex -- realWavelength Intensity + @param : 23 ..07 .... 06 05 04 03 02 01 00 + ip信息不包含 + @Returns : realWavelength Intensity + """ + self.buf = b'' + self.it = None + self.light_int = None # 未标定的整数值 + self.spectrum = None # 光谱强度 + # self.current_buf = "" + # self.current_buf_seq = 0 + # self.current_it_int = {"it": 0, "light_int": []} # 积分时间及换算的整数值 + # self.res = {"wavelength": [], "light": []} + self.cal_cfg = {} + # self.current_cal = {} # 当前传感器的序列号 + pass + + def setBuf(self, buf: bytes): + self.buf = buf + pass + + def setCalCfg(self, d: dict): + self.cal_cfg = d + pass + + def getRealWavelength(self, d: dict): + self.cal_cfg = d + pass + + def getSpectrum(self): + return self.spectrum + + def resetPara(self, ): + self.buf = b'' + self.it = None + self.light_int = None + self.spectrum = None # 光谱强度 + self.cal_cfg = {} + pass + + def resetItSpectrum(self, ): + self.it = None + self.spectrum = None # 光谱强度 + pass + + def printPara(self, ): + print(f"**************Ramses printPara*******************") + print(f"{self.buf}") + print(f"{self.cal_cfg}") + print(f"{self.it}") + print(f"{self.light_int}") + print(f"{self.spectrum}") + print(f"**************Ramses printPara*******************") + pass + + def dealBuf(self, ip_included:bool=False): + """多个传感器的数据处理， 头部是否包含Ip帧的信息""" + log.info(f" dealBuf ", __name__) + + res = {} + len_ = len(self.buf) + if len_ < 576: + return + if ip_included: + self.buf = self.buf[26:] + len_ = len_ - 26 + if len_ % 576 != 0: + return + + for i in range(int(len_/576)): + res.update({i+1: {}}) + temp_buf = self.buf[7:71] + self.buf[79:143] + \ + self.buf[151:215] + self.buf[223:287] + \ + self.buf[295:359] + self.buf[367:431] + \ + self.buf[439:503] + self.buf[511:575] + self.ConvertAndCalibrate( temp_buf ) + # print(len(temp_buf)) + temp = self.__ConvertBytesToInt(temp_buf) + res.update( { i+1: temp } ) + # print(res) + pass + + def ConvertAndCalibrate(self,) -> None: + log.debug(f" ConvertAndCalibrate ", __name__) + temp = self.__ConvertBytesToInt( ) + self.__CalibrateSpectrumData( ) + pass + + # 转换一个传感器的部分 + def __ConvertBytesToInt(self ) -> None: + res = {} + d = [] # List [ Tuple[ it:int, sing_set:tuple[int] ] ] + self.it = 2 << int(self.buf[1]) # integrated time + self.light_int = struct.unpack( + " None: + log.info(f"ConfigAWRAMS init: ", __name__, "", "") + self.device_type = None + self.configuration =None + self.cal_configuration = {} + pass + + + + def setDeviceType(self, device_type:str) -> None: + self.device_type = device_type.lower() + pass + + def setSystemCfgDict(self, cfg:dict) -> None: + self.configuration = cfg + log.info(f"self.configuration : {self.configuration} ", __name__, "", "") + pass + + def getCalConfiguration(self) -> None: + if self.device_type == None: + self.cal_configuration = None + if self.configuration == None: + self.cal_configuration =None + + for k,v in self.configuration.items(): + + if v["SN"] == "" or v['FUNC']=="": + pass + else: + self.cal_configuration.update( {v["FUNC"]:{}} ) + self.cal_configuration[v["FUNC"]].update( {"SN":v['SN']} ) + self.cal_configuration[v["FUNC"]].update( {"FUNC":v['FUNC']} ) + + self.__init_configuration_basic() + self.__init_configuration_cal() + self.__init_configuration_IP_SAM() + pass + + def __init_configuration_basic(self ) -> None: + # self.cfgtool = Config() + for k in self.cal_configuration.keys(): + sn = self.cal_configuration[k]["SN"] + if self.__isSamIniExisted(sn): + self.cal_configuration[k].update({ "TYPE" : "SAM" }) + self.cal_configuration[k].update({ "samsn" : sn }) + self.cal_configuration[k].update({ "inifile" : "SAM_"+sn+".ini" }) + self.cal_configuration[k].update({ "calfile" : "Cal_SAM_"+sn+".dat" }) + self.cal_configuration[k].update({ "calaqfile" : "CalAQ_SAM_"+sn+".dat" }) + self.cal_configuration[k].update({ "backfile" : "Back_SAM_"+sn+".dat" }) + if self.__isSamIPIniExisted(sn): + self.cal_configuration[k].update({ "TYPE" : "SAMIP" }) + samsn = self.__getSAMSN(sn) + if samsn== None: + log.warning(f"Cannot get samsn from Sensor: {sn}", __name__, "", "" ) + raise Exception(f"Cannot get samsn from Sensor: {sn}") + self.cal_configuration[k].update({ "samsn" : samsn }) + self.cal_configuration[k].update({ "inifile" : "SAMIP_"+sn+"_ALL.ini" }) + self.cal_configuration[k].update({ "calfile" : "Cal_SAM_"+samsn+".dat" }) + self.cal_configuration[k].update({ "calaqfile" : "CalAQ_SAM_"+samsn+".dat" }) + self.cal_configuration[k].update({ "backfile" : "Back_SAM_"+samsn+".dat" }) + if not self.__isSamIniExisted(sn) and not self.__isSamIPIniExisted(sn): + log.warning(f"Cannot find ini file for Sensor: {sn}", __name__, "", "" ) + raise Exception(f"Cannot find ini file for Sensor: {sn}") + + pass + + def __init_configuration_cal(self ) -> None: + # self.cfgtool = Config() + for k in self.cal_configuration.keys(): + sn = self.cal_configuration[k]["SN"] + # Device File + calpath = CAL_DIR.joinpath(self.device_type, self.cal_configuration[k]["calfile"]) + if calpath.exists( ): + res = ReadCal.read_columns_set_by_mark( calpath, FILE_MARK, 1 ) + self.cal_configuration[k].update({ "cal" : res[1][0] }) + calaqpath = CAL_DIR.joinpath(self.device_type, self.cal_configuration[k]["calaqfile"]) + if calaqpath.exists( ): + res = ReadCal.read_columns_set_by_mark( calaqpath, FILE_MARK, 1 ) + self.cal_configuration[k].update({ "calaq" : res[1][0] }) + backpath = CAL_DIR.joinpath(self.device_type, self.cal_configuration[k]["backfile"]) + if calaqpath.exists( ): + res = ReadCal.read_columns_set_by_mark( backpath, FILE_MARK, 1,2 ) + self.cal_configuration[k].update({ "b0" : res[1][0] }) + self.cal_configuration[k].update({ "b1" : res[1][1] }) + pass + + + + def __init_configuration_IP_SAM(self ) -> None: + # self.cfgtool = Config() + for j in self.cal_configuration.keys(): + # log.debug(f"__init_configuration_IP_SAM {j}", __name__, "", "" ) + inipath = CAL_DIR.joinpath(self.device_type, self.cal_configuration[j]["inifile"]) + # log.debug(f"__init_configuration_IP_SAM {inipath}", __name__, "", "" ) + sam = ReadCal.readSAMCalFromIni(inipath) + # log.debug(f"__init_configuration_IP_SAM {sam}", __name__, "", "" ) + for k,v in sam.items(): + self.cal_configuration[j].update({ k : v }) + if self.cal_configuration[j]["TYPE"] == "SAMIP": + ip = ReadCal.readIPCalFromIni(inipath) + for k,v in ip.items(): + self.cal_configuration[j].update({ k : v }) + + def __isSamIniExisted(self,sn) ->bool: + sn_0 = "SAM_"+str(sn)+".ini" + path_ = CAL_DIR.joinpath(self.device_type.lower(), sn_0) + if path_.exists(): + return True + return False + + + def __isSamIPIniExisted(self,sn) ->bool: + sn_0 = "SAMIP_"+str(sn)+"_ALL.ini" + path_ = CAL_DIR.joinpath(self.device_type.lower(), sn_0) + if path_.exists(): + return True + return False + + def __getSAMSN(self,sn) -> None: + sn_0 = "SAMIP_"+str(sn)+"_ALL.ini" + path_ = CAL_DIR.joinpath(self.device_type.lower(), sn_0) + # path_ = DATA_DIR.joinpath(self.device.lower(), CAL_DIR, sn_0) + samsn = ReadCal.readSamSNFromIni( path_ ) + if samsn == None: + return None + return samsn + pass + + # def __init2__(self, device:str, **kwargs) -> None: + # """ + # get cal parameter for every sensor + # para : {"1":{"SN":"85B5","FUNC","Lsky"},"2":{},"3":{}} + # """ + # # log.info(f"ProcessAWRAMS kwargs: {kwargs}", __name__, "", "") + # # log.info(f"len: { len(kwargs)}", __name__, "", "") + + # if len(kwargs) != 3: + # log.warning(f" pass a wrong para to ProcessAWRAMS {kwargs}", __name__, "", "") + # self.device = device.lower() # surface profile awrams + # self.ramses = {} + + # # 生成标定文件 { } + # for k,v in kwargs.items(): + # self.ramses.update( {v["FUNC"]:{}} ) + # self.ramses[v["FUNC"]].update( {"SN":v['SN']} ) + # self.ramses[v["FUNC"]].update( {"FUNC":v['FUNC']} ) + # pass + # log.debug(f" ===== {self.ramses}",__name__, "", "" ) + + + # # if kwargs.__contains__("1"): + # # self.ramses.append( self.cfgtool.getDictByAttr("ramses")) + # # self.cfgtool.set_attr(self.ramses[1],kwargs['1']"SN",kwargs['1']) + # # if kwargs.__contains__("2"): + # # self.ramses.append( self.cfgtool.getDictByAttr("ramses")) + # # self.cfgtool.set_attr(self.ramses[2],"SN",kwargs['1']) + # # if kwargs.__contains__("3"): + # # self.ramses.append( self.cfgtool.getDictByAttr("ramses")) + # # self.cfgtool.set_attr(self.ramses[3],"SN",kwargs['1']) + + # self.__init_configuration_basic() + # self.__init_configuration_cal() + # self.__init_configuration_IP_SAM() + + # # log.info(f"ProcessAWRAMS after initiate: {kwargs}", __name__, "", "") + + # def __init_configuration_basic2(self ) -> None: + # # self.cfgtool = Config() + # for k in self.ramses.keys(): + # sn = self.ramses[k]["SN"] + # if self.__isSamIniExisted(sn): + # self.ramses[k].update({ "TYPE" : "SAM" }) + # self.ramses[k].update({ "samsn" : sn }) + # self.ramses[k].update({ "inifile" : "SAM_"+sn+".ini" }) + # self.ramses[k].update({ "calfile" : "Cal_SAM_"+sn+".dat" }) + # self.ramses[k].update({ "calaqfile" : "CalAQ_SAM_"+sn+".dat" }) + # self.ramses[k].update({ "backfile" : "Back_SAM_"+sn+".dat" }) + # if self.__isSamIPIniExisted(sn): + # self.ramses[k].update({ "TYPE" : "SAMIP" }) + # samsn = self.__getSAMSN(sn) + # if samsn== None: + # log.warning(f"Cannot get samsn from Sensor: {sn}", __name__, "", "" ) + # raise Exception(f"Cannot get samsn from Sensor: {sn}") + # self.ramses[k].update({ "samsn" : samsn }) + # self.ramses[k].update({ "inifile" : "SAMIP_"+sn+"_ALL.ini" }) + # self.ramses[k].update({ "calfile" : "Cal_SAM_"+samsn+".dat" }) + # self.ramses[k].update({ "calaqfile" : "CalAQ_SAM_"+samsn+".dat" }) + # self.ramses[k].update({ "backfile" : "Back_SAM_"+samsn+".dat" }) + # if not self.__isSamIniExisted(sn) and not self.__isSamIPIniExisted(sn): + # log.warning(f"Cannot find ini file for Sensor: {sn}", __name__, "", "" ) + # raise Exception(f"Cannot find ini file for Sensor: {sn}") + + # pass + + # def __init_configuration_cal2(self ) -> None: + # # self.cfgtool = Config() + # for k in self.ramses.keys(): + # sn = self.ramses[k]["SN"] + # # Device File + # calpath = CAL_DIR.joinpath(self.device, self.ramses[k]["calfile"]) + # if calpath.exists( ): + # res = Readfile.read_columns_set_by_mark( calpath, FILE_MARK, 1 ) + # self.ramses[k].update({ "cal" : res[1][0] }) + # calaqpath = CAL_DIR.joinpath(self.device, self.ramses[k]["calaqfile"]) + # if calaqpath.exists( ): + # res = Readfile.read_columns_set_by_mark( calaqpath, FILE_MARK, 1 ) + # self.ramses[k].update({ "calaq" : res[1][0] }) + # backpath = CAL_DIR.joinpath(self.device, self.ramses[k]["backfile"]) + # if calaqpath.exists( ): + # res = Readfile.read_columns_set_by_mark( backpath, FILE_MARK, 1,2 ) + # self.ramses[k].update({ "b0" : res[1][0] }) + # self.ramses[k].update({ "b1" : res[1][1] }) + # pass + + # def __init_configuration_IP_SAM2(self ) -> None: + # # self.cfgtool = Config() + # for j in self.ramses.keys(): + # # log.debug(f"__init_configuration_IP_SAM {j}", __name__, "", "" ) + # inipath = CAL_DIR.joinpath(self.device, self.ramses[j]["inifile"]) + # # log.debug(f"__init_configuration_IP_SAM {inipath}", __name__, "", "" ) + # sam = Readfile.readSAMCalFromIni(inipath) + # # log.debug(f"__init_configuration_IP_SAM {sam}", __name__, "", "" ) + # for k,v in sam.items(): + # self.ramses[j].update({ k : v }) + # if self.ramses[j]["TYPE"] == "SAMIP": + # ip = Readfile.readIPCalFromIni(inipath) + # for k,v in ip.items(): + # self.ramses[j].update({ k : v }) \ No newline at end of file diff --git a/myconfig.py b/myconfig.py new file mode 100644 index 0000000..db93dec --- /dev/null +++ b/myconfig.py @@ -0,0 +1,322 @@ +#! python3 +# -*- encoding: utf-8 -*- +''' +@File : myconfig.py +@Time : 2023/03/01 15:28:20 +@Author : Jim @ Yiwin +@Version : 1.0 +@Contact : jim@yi-win.com +@Descrip : SysConfig +''' + +import yaml +from enum import Enum +from pathlib import Path + +DEVICE_ID = [2] +CURRENT_DIR = Path() +DATA_DIR = Path("data") +CAL_DIR = Path("calfile") +OUTPUT_DIR = Path("data", "output") +YAML_FILE_NAME = "config.yml" +RETRIEVE_CFG_FILE = "retrieve.yml" +FILE_MARK = ['Spectrum', 'DATA'] +BEGIN_WAVELENGTH = 350 +END_WAVELENGTH = 950 +SAVE_EXT_NAME = ".csv" +INTERVAL = 1.0 +SEPARATOR = ";" +TOKEN = ";" +NEWLINE = "\n" +ROWFACTOR = 0.026 + +class DeviceType(Enum) : + AWRAMS = 1 + SURFACE = 2 + PROFILE = 3 + +class RamsesFunc(Enum): + Lsky = 1 + Esky = 2 + Lwater = 3 + Lw = 4 + Rs = 5 + +class RamsesAWRAMS(Enum): + Lsky = 1 + Esky = 2 + Lwater = 3 + Lw = 4 + Rs = 5 + +class RamsesSURFACE(Enum): + Lsky = 1 + Esky = 2 + Lwater = 3 + Lw = 4 + Rs = 5 + +class RamsesPROFILE(Enum): + Ed = 1 + Esky = 2 + Lu = 3 #upwelling + Lw = 4 + Rs = 5 + +IP_CAL = { + "Incl_Orientation": "up", + "Incl_Xgain": 1.0, + "Incl_Xoffset": 125, + "Incl_Ygain": 0.9375, + "Incl_Yoffset": 126, + "Incl_KBG": 1.2073, + "Incl_Kref": 0.1275, + "Press_Current_mA": 1.08, + "Press_Surface_bar": 5.57, + "Press_Gain": 2.7, + "WithIncl": 1, + "WithPress": 1, + "Press_Sens_mV_bar_4mA": 71.36, + "Press_Sens_mV_bar_1mA": 17.84, + "Press_Type": "PA-10/TAB/10bar", + "CalibrationDate": "08.06.2018", +} + +RAMSES_CAL = { + "SN": "", + "TYPE": "SAM", # SAMIP or SAM + "FUNC": "Lsky", + "inifile": "", + "calfile": "", + "calaqfile": "", + "backfile": "", + "samsn": "", + "b0": [], + "b1": [], + "cal": [], + "calaq": [], + "DarkPixelStart": 237, + "DarkPixelStop": 254, + "Firmware": 2.06, + "IDDataBack": "DLAB_2016-11-29_14-47-59_729_812", + "IDDataCal": "DLAB_2016-12-07_12-00-24_364_510", + "IDDataCalAQ": "DLAB_2016-12-07_12-02-43_591_545", + "IntegrationTime": 0, + "Reverse": 0, + "SerialNo_MMS": 103307, + "WavelengthRange": "310..1100", + "c0s": 299.895, + "c1s": 3.31161, + "c2s": 0.00031652, + "c3s": -1.73194e-06, + "c4s": +0.000000000E+00, + "cs": 102842, + "savefile": "" +} + +class MyConfig(object): + """ + 设置 ID对应的传感器 + """ + + def __init__(self) -> None: + self.device_id = [] + self.device_type = None + self.current_device_id = None + self.system_cfg = {} + self.cfg_path = Path() + self.yml_cfg_file = YAML_FILE_NAME + self.retrieve_cfg_file = Path(RETRIEVE_CFG_FILE) + self.system_cal_cfg = {} + self.validate = { } + + def addDeviceID(self, id:int) -> None: # + self.device_id.append(id) + pass + + def setDeviceID(self, id:int) -> bool: # + if id in self.device_id: + self.current_device_id = id + return True + else: + self.current_device_id = None + return False + pass + + def setDeviceType(self, device_type:DeviceType) -> None: + self.device_type = device_type + pass + + def setRetrieveCfg(self, rtv_yml:str="") -> None: + if rtv_yml =="": + return None + self.retrieve_cfg_file = Path(rtv_yml) + pass + + def getSystemCfg(self,)->None: + ''' + 不同系统，修改此函数，或添加函数 getSystemCfg***()供调用 + ''' + if self.current_device_id == None: + self.system_cfg = None + if self.device_type == None: + self.system_cfg = None + temp_cfg = {} + if self.device_type == DeviceType.AWRAMS: + temp_cfg = { + 1: {"SN": "85B5", "FUNC": RamsesAWRAMS(1).name}, + 2: {"SN": "50ED", "FUNC": RamsesAWRAMS(2).name}, + 3: {"SN": "852F", "FUNC": RamsesAWRAMS(3).name} + } + if self.device_type == DeviceType.SURFACE: + temp_cfg = { + 1: {"SN": "85B5", "FUNC": RamsesSURFACE(1).name}, + 2: {"SN": "50ED", "FUNC": RamsesSURFACE(2).name}, + 3: {"SN": "852F", "FUNC": RamsesSURFACE(3).name} + } + if self.device_type == DeviceType.PROFILE: + temp_cfg = { + 1: {"SN": "85B5", "FUNC": RamsesPROFILE(1).name}, + 2: {"SN": "50ED", "FUNC": RamsesPROFILE(2).name}, + 3: {"SN": "852F", "FUNC": RamsesPROFILE(3).name} + } + self.system_cfg.update( { self.current_device_id : temp_cfg } ) + pass + + def setCfgRamsesSN(self, sn_cfg: dict)->None: + if len(sn_cfg) == 0: + return None + + for k in self.system_cfg[self.current_device_id].keys(): + if str(k) in sn_cfg.keys() : + self.system_cfg[self.current_device_id][k]["SN"] = sn_cfg[str(k)] + else: + self.system_cfg[self.current_device_id][k]["SN"] = None + # if k in sn_cfg.keys() : + # self.system_cfg[self.current_device_id][k]["SN"] = sn_cfg[k] + pass + + def setSystemCalCfg(self, sn_cfg: dict)->None: + if len(sn_cfg) == 0: + pass + + def getDictByAttr(self, *args) -> dict: + ret = {} + if len(args) == 0: + return ret + if len(args) == 1: + if not hasattr(self, args[0]): + return ret + tmp = getattr(self, args[0]) + if isinstance(tmp, dict): + ret.update(tmp) + return ret + if len(args) == 2: + if not hasattr(self, args[0]): + return ret + if not isinstance(getattr(self, args[0]), dict): + return ret + tmp: dict = getattr(self, args[0]) + if not tmp.__contains__(args[1]): + # print(f"------------{args[1]}") + return ret + tmp2 = tmp[args[1]] + if isinstance(tmp2, dict): + ret.update(tmp2) + return ret + if len(args) > 2: + return ret + pass + + # 设置字典对应的键值 + def set_attr(self, d: dict, k, v) -> bool: + if d.__contains__(k): + d.update({k: v}) + return True + return False + + def write_yaml(self, d: dict): + with open(self.yml_cfg_file, "w", encoding="utf-8") as f: + yaml.dump(d, f) + + def read_yaml(self ) -> dict: + with open(self.yml_cfg_file, "r", encoding="utf-8") as f: + content = f.read() # conent 读出来是字符串 + d = yaml.load(content, Loader=yaml.FullLoader) # 用load方法转字典 + return d + + def write_rtv_yaml(self, d: dict): + with open(self.retrieve_cfg_file, "w", encoding="utf-8") as f: + yaml.dump(d, f) + + def read_rtv_yaml(self ) -> dict: + with open(self.retrieve_cfg_file, "r", encoding="utf-8") as f: + content = f.read() # conent 读出来是字符串 + d = yaml.load(content, Loader=yaml.FullLoader) # 用load方法转字典 + return d + + def get_retrieve(self) -> dict: + retrieve = {} + retrieve.update({"beginWL": BEGIN_WAVELENGTH}) + retrieve.update({"endWL": END_WAVELENGTH}) + retrieve.update({"interval": INTERVAL}) + retrieve.update({"rowFactor": ROWFACTOR}) + return retrieve + pass + +ramses_buf_str= "\ + 23a0000007fefe0a0781067d067d068e0693069c069c06b006b506cb06e40619076607e1076c081509cd09bb0a7d0bee0b1d0c6d0cca0ca40ddc0f30135b18b4224d320e43f852c8\ + 23a0000006fefe17639c71c97c9484bb89358be98e5d98b1a37eadccb66abd26be31b97db124aa18a3f29c0499349735968e93a48eea8a028bc28cec8d048f1c92c096de9ab99d43\ + 23a0000005fefee0a157a9ecb1b1b97dc034c507c741c7d2c65ec550c20dbde9b535ae56a60a9e2296cd8ee887278129798170c669b16503632e61605f3a5def5a8e5862561154be\ + 23a0000004fefe3c51f54dda4a0c48634595426a3f5a3cc539903767362b36cc351f356634b633fc32c4310630fb2ec32fe63199349037e03ac03dc03eae3c303a9639d13a413da7\ + 23a0000003fefe97412746bc49404caa4dc54d374ca646d63bde326b332739e43c503d2b3c363aca37ef342d32c22f782d1d2bd328ec250222101e691b2b1a7119d3184018fa174d\ + 23a0000002fefed3177a17e3160616f81423149313f11233126711d8103b10a40f160f8a0efb0d580d660c6f0bd80aa10a580a000ab4099b0994097109e2082c087a0736072c077d\ + 23a0000001fefe2a071e071607140718071e073c074d076a076e076b076007670763076c0760075c073d07350724071207fb06ef06d706cc06b706a506960684067b0672066306a2\ + 23a0000000fefe58065406500642064e064306470642063f064806410644064306430641064206460640063c063e063e063d064406430644063c063c063c063c06400640064906ed\ + 23a0000007fefe0a071b071c07260737073e0749075907650774078b07c0072c08e3082b0ada0be80d9c107b147318931ae51add1a271a77192f1bfb1d99207c265c30c738f03d53\ + 23a0000006fefe07438f489d4d8452d957945cff639171a5832a97deab7fbe09c86cc5acb9d5ab299e90915287de7f8b7ae275ed6f056aff664f662665bc63936335655767926879\ + 23a0000005fefec269206d3e724377fe7b7a8050831f85b68637881689e4884787008580823d7f887bee773d7496702a6cde654c5f895a1f570d5416514d4e9d4bde4897468c44c3\ + 23a0000004fefe71421640db3dc83bbb39533777346c31b92e912c152b5b2aff292129e727cb26d925fd249223b4214020da1fe91ff41f28209420dc201520071e671c221c8d1cfe\ + 23a0000003fefecd1dd31fe72164235924cb24b924f72331211e1ca018ef19221d121f621fe91e081ed51c761b311a2f193c18651788166015a013de11d1106e1041101810e80f2e\ + 23a0000002fefee50fd90fbc0f6d0ff70e610ee00d930d460de40c900c450c010cbd0b7c0b480bfc0aab0a2c0a9e094d092b09fb08d008a208990881087d083908e6078d07660747\ + 23a0000001fefe5b075a07500751074d074e0751075d076807730788077e077c077c07810777077a076c0765075c075607480740073607270723070d070d070b07ff06fd06f0062f\ + 23a0000000fefef606f106ed06f506ef06eb06f106ec06f006e906e806ed06e606ef06e506ea06ec06ee06eb06e906ee06f006ef06ea06e906e606e606e606ec06ea06ea06f506f2\ + 23a0000007fefe0605830479047d047e04810482048204850487048b048c04850487048c049304950498049c04a804a904b604b804c604d004e104f104190546059805ff059a06bb\ + 23a0000006fefe90073209ea0b7610c117cc22b2324a487b645e8798acf2c927d28dc1f2a1ac809664804e543d7a308327a321d81dab1b0f1bca1bb41dba2098243329382e773335\ + 23a0000005fefeca38213e544329486f4c1c500653335592564f577d573b57a556b4557e540a534451424f004d714a9b47994458410e3e9c3af8368c33b630982eca2c052b3429ae\ + 23a0000004fefe632763255a234821321f0e1d061b291983171e16d814a91379124b1138104b0f610e990dd80c240c880bfc0a7b0af8097709020990082508de07c507ba07a50762\ + 23a0000003fefe950781076407460727070007d306ab066e06270605060d0614061306fe05e805d105b405a00586057105630549053d0528051105fd04f304ee04e404dc04dd04c3\ + 23a0000002fefed804d904d004c604bf04b904b904b104a804a604a6049d04980496048f04970491048c04880486047e0484047a047d047b047c0478047a04760476047004700430\ + 23a0000001fefe6f0474046c046d0472046d047304750470046e04760472046f047604700473046f04730470047504700470046d046b047004710470046e046d046a046b046904e4\ + 23a0000000fefe690470046e046b046c04680468046e0466046a046a0469046b046d04640466046c046c046b046a04660466046b046c046a046a0468046f046804740471048d0417\ + " + + +if __name__ == "__main__": + cfg = MyConfig() + cfg.addDeviceID(2) + cfg.addDeviceID(3) + cfg.setDeviceType(DeviceType.AWRAMS) + cfg.setDeviceID(2) + cfg.getSystemCfg() + print(cfg.system_cfg) + + d = {"1":"8888","2":["7777"],"3":["9999"]} + cfg.setCfgRamsesSN(d) + print("修改后。。。。。\n") + print(cfg.system_cfg) + + # cfg.write_yaml( cfg.system_cfg) + + dd = cfg.read_yaml() + # dd 作为cfg.system_cfg 使用 + for k,v in dd.items(): + print(k) + print(type(k)) + + retrieve = { + "beginWL": 350, + "endWL": 950, + "interval": 1, + "rowFactor": 0.026 + } diff --git a/profiler.py b/profiler.py new file mode 100644 index 0000000..8c1462c --- /dev/null +++ b/profiler.py @@ -0,0 +1,1090 @@ +#! python3 +# -*- encoding: utf-8 -*- +''' +@File : handheld.py +@Time : 2023/02/24 17:20:59 +@Author : Jim @ Yiwin +@Version : 1.0 +@Contact : jim@yi-win.com +''' + +CUR_TIME_STR_FMT = "%Y-%m-%d %H:%M:%S" + +import time +import locale +import struct +import numpy as np +from pathlib import PurePath,Path +from myconfig import CURRENT_DIR,DATA_DIR,OUTPUT_DIR,NEWLINE,ROWFACTOR,SAVE_EXT_NAME,TOKEN +from myconfig import DeviceType,RamsesSURFACE,RamsesAWRAMS,RamsesPROFILE +from tools.mylogger import log +from tools.myexception import MyException +from tools.mytime import MyTime +from tools.mypath import MyDir +from Ramses import Ramses + +class HandHeldBuf: + def __init__(self,) -> None: + self.__buf = b'' + self.__head = {} + self.__begin_sign = b'\x23' + self.__end_sign = b'\x0D' + self.data_ip = b'' + self.measure_group = { + "Lsky": b'', + "Esky": b'', + "Lwater": b'', + } + self.one_group_data= b'' + self.state = 0 + + def readFile2Buf(self, fpath) -> None: + with open(fpath,"rb") as f: + self.__buf = f.read() + pass + pass + + def read_buf(self, size: int) -> bytes: + if size > self.__buf.__len__(): + return b'' + ret = self.__buf[0:size] + self.__buf = self.__buf[size:] + return ret + + def write_buf(self, buf: bytes) -> None: + len = buf.__len__() + # logging.info(f'Received ID:{id} Size:{len}') + self.__buf = self.__buf+buf + + def get_buf_size(self) -> int: + return self.__buf.__len__() + + def back_bytes(self, buf: bytes) -> None: + self.__buf = buf+self.__buf + + def reset_head(self) -> None: + self.__head = {} + + def reset_buf(self) -> None: + self.__buf = b'' + + def getResult(self) -> str: + return self.res + + def resetMeasureGroup(self) -> None: + self.measure_group['Lsky'] = b'' + self.measure_group['Esky'] = b'' + self.measure_group['Lwater'] = b'' + + def getMeasureGroup(self) -> dict: + return self.measure_group + + def decode_handheld(self) -> bool: + '''以26个字节开始,一般26个00 ,然后23... 07.... ''' + if self.get_buf_size() < 1754: + self.__buf = b'' + return False + + self.data_ip = self.read_buf(26) + Lsky = self.read_buf(576) + Esky = self.read_buf(576) + Lwater = self.read_buf(576) + + self.measure_group['Lsky'] = self.deal_576_to_512(Lsky) + self.measure_group['Esky'] = self.deal_576_to_512(Esky) + self.measure_group['Lwater'] = self.deal_576_to_512(Lwater) + + if self.measure_group['Lsky'] == b'' \ + or self.measure_group['Esky'] == b'' \ + or self.measure_group['Lwater'] == b'' : + return False + + return True + + def decode_one_group_handheld(self) -> bool: + '''以26个字节开始,一般26个00 ,然后23... 07.... ''' + if self.get_buf_size() < 1754: + self.__buf = b'' + return False + + self.data_ip = self.read_buf(26) + self.one_group_data = self.read_buf(1728) + + return True + + def deal_576_to_512(self,data:bytes) -> bytes: + ''' 576字节校验,拆分成字典 23... 07.... ,然后254*2''' + ret = {} + index = 72 + for i in range(8): + temp = data[i*index : i*index+index] + # print( temp.hex()) + if temp[0] != 35 and temp[0]<8 and temp>0 : + return b'' + pass + ret.update( { temp[4]: temp} ) + if len(ret) != 8: + return b'' + ret_byte = ret[7][7:71]+ ret[6][7:71]+ ret[5][7:71]+ ret[4][7:71] \ + + ret[3][7:71]+ ret[2][7:71]+ ret[1][7:71]+ ret[0][7:71] + return ret_byte + + + def decode(self) -> str: + ret = '' + temp_buf = b'' + token = ";" + if TOKEN: + token = TOKEN + + if self.state == 0: + while self.get_buf_size() >= 1: + if self.read_buf(1) != self.__end_sign: + continue + self.state = 1 + break + + if self.state == 1: + while self.get_buf_size() >= 1: + buf = self.read_buf(1) + if buf != self.__end_sign: + temp_buf += buf + if buf == self.__end_sign: + ret = temp_buf[0:20].decode( + 'utf-8').strip(" ") + token + temp_buf[-9:-1].decode('utf-8').strip(" ") + log.info(f"decode : {ret}") + temp_buf = b'' + self.back_bytes(temp_buf) # 写回临时buf到 + self.res = ret + return ret + pass + + +class HandHeldPath(object): + '''处理一次测量''' + def __init__(self, ): + self.mode = 0 + self.cfg ={} + self.mydir = MyDir() + self.base_path:Path = Path() + self.output_path:Path = Path() + self.data_path:Path = Path() + self.filelist = [] + self.error_result = [] + pass + + def setMode( self, mode:int = 0 ): + self.mode = mode + pass + + def setBasePath( self, fpath:Path ): + self.base_path = fpath + pass + + def setDataPath( self, fpath:Path ): + self.data_path = fpath + pass + + def setOutputPath( self, fpath:Path ): + self.output_path = fpath + pass + + def getSensorPathFromInfoPath( self, info_path:Path, ) -> Path: + # 服务器上转移后的模式 + if self.mode == 1: + sensor_path = self.base_path.joinpath( + info_path.parent, + "sensor.bin" + ) + elif self.mode == 0: + sensor_path = self.base_path.joinpath( + "data", + info_path.parts[1][:4] + "_" + info_path.parts[1][-2:], + info_path.parts[2], + "sensor", + info_path.name + ) + else: + sensor_path = None + return sensor_path + pass + + def getOutputPathFromSensorPath(self,sensor_path:Path ) -> Path: + if self.mode == 1: + output_path = self.output_path.joinpath( + self.info_dict['year'] + "_" + self.info_dict['month'] + + "_" + self.info_dict['day'] + "_" + self.info_dict['hour'] + + "_" + self.info_dict['minute'] + "_" + self.info_dict['second'] + ) + elif self.mode == 0: + output_path = self.output_path.joinpath( + sensor_path.parts[1][:4] + "_" + sensor_path.parts[1][-2:] + + "_" +sensor_path.parts[2] + "_" +sensor_path.name + ) + else: + output_path = None + return output_path + pass + + def getCurrentMeasureTimeFromPath(self,fpath:Path ) -> str: + ret = '' + if self.mode == 1: # 读信息txt获得时间 + txt_path = fpath.parent.glob("*.txt") + txt_stem = txt_path[0].stem + ret = fpath.parts[1][:4]+"-"+fpath.parts[1][-2:]+"-"+fpath.parts[2]+" " \ + + txt_stem[-9:-7] + ":" + txt_stem[-6:-4] + ":" + txt_stem[-3:-1] + pass + elif self.mode == 0: + ret = fpath.parts[1][:4]+"-"+fpath.parts[1][-2:]+"-"+fpath.parts[2]+" " \ + + fpath.name[0:2] + ":" + fpath.name[3:5] + ":" + fpath.name[6:8] + else: + pass + return ret + pass + + + + def getDataFileList(self, ): + ''' + 获得成对的info sensor 文件 + [目录名，文件名，年月日，时间, measure_id] + ''' + # ret = [] + fs = None + self.filelist = [] + if self.mode == 1: + fs = self.data_path.glob( "*/*/*/*/info.bin" ) + else: + fs = self.data_path.glob( "*/*/info/*" ) + + for f in fs: + error_file = {} + if f.stat().st_size==0: + error_file.update( {"path": f } ) + error_file.update( {"error":"info file size is zero"} ) + self.error_result.append(error_file) + continue + + # self.info_path_fname = f + sensor_path = self.getSensorPathFromInfoPath(f) + # sensor_path = Path(sensor_purepath) + # sensor 文件不存在 + if not sensor_path.exists(): + error_file.update( {"path":f} ) + error_file.update( {"error":"cannot find the sensor file "} ) + self.error_result.append(error_file) + continue + + # sensor文件大小为0 + if sensor_path.stat().st_size==0: + error_file.update( {"path":sensor_path} ) + error_file.update( {"error":"sensor file size of the sensor is zero' "} ) + self.error_result.append(error_file) + continue + self.setFilelist(f,sensor_path ) + pass + + + def setFilelist(self, info_path:Path, sensor_path:Path): + temp = {} + temp.update( {"info_path" : info_path } ) + temp.update( {"name" : info_path.name} ) + temp.update( {"parent" : info_path.parent} ) + + if self.mode==1: # 服务器转移后目录 + temp.update( {"year" : info_path.parts[-5] } ) + temp.update( {"month" :info_path.parts[-4] } ) + temp.update( {"day" :info_path.parts[-3] } ) + + temp.update( {"year" : info_path.parts[1][:4] } ) + temp.update( {"month" :info_path.parts[1][-2:] } ) + temp.update( {"day" :info_path.parts[2] } ) + temp.update( { "sensor_path" :sensor_path } ) + self.filelist.append(temp) + + def getFilelist(self, ): + return self.filelist + + def printTest(self,d,sleep_time:int=5): + log.info( f"***** : I am testing from HandheldPath ********", __name__ ) + print(d) + log.info( f"***** : Ending testing from HandheldPath ********", __name__ ) + time.sleep(sleep_time) + pass + +class Profiler(object): + def __init__(self, ): + """ + @description : 手持数据初始化 + 入口： dealOneMeasurement_Handheld() + 参考： 分组写成函数,一次测量多组数据分离出来。 + 保存： 按时间文件夹保存 + """ + self.device_type = DeviceType.PROFILE.name + self.device_enum = None + self.device_id = {} + self.syscfg = {} + self.retrieve ={} + self.error_result=[] + self.mode = 0 # 0:默认SD数据， 1:服务器数据 + self.base_path = CURRENT_DIR + self.current_path = CURRENT_DIR + self.data_path = DATA_DIR + self.output_path = OUTPUT_DIR + self.info_path_fname:Path = self.base_path + self.sensor_path_fname:Path = self.base_path + self.filelist = [] # 包含了全部路径信息 + self.intensity_before_avg = None + self.intensity_after_avg = {} # 最终结果{lsky: esky: lwater: } + self.intensity_after_interpo = {} # 最终结果{lsky: esky: lwater: } + + self.one_group_result = {} # 手持式 多次间隔 + self.wavelength = np.array([]) + self.real_wavelength = {} # 最终结果{lsky: esky: lwater: } + self.current_filepath = '' # 不含后缀 + self.current_measure_time = None # 当前测量时间 + self.current_group_num = 0 + self.measurement_interval = 0 + self.measurement_repeat = 1 + # self.sl2f = SaveList2File() + self.mydir = MyDir() + self.my_time = MyTime() + self.hhb = HandHeldBuf() + self.hhp = HandHeldPath() + self.ramses = Ramses() + self.info_dict = {} + self.res = {} # 最终结果{lsky: esky: lwater: Lw: Rs:} + + # 方法 + self.setDeviceEnum() + self.setMode() + self.set_hhp_path() + pass + + def setMode(self,mode=0 ): + self.hhp.setMode (mode) # 0 ：读SD卡 1：读服务器数据 + + def setMeasureID(self, mid:int): + self.measure_id = mid + + def setOldFolder(self, tuple_path:tuple): + self.old_folder = self.base_path.joinpath( *tuple_path ) + + def transferFromOldFolder(self, ): + log.info( f"transferFromOldFolder: ", __name__ ) + self.getNewFolderFromOldFolder() + bin_files = self.old_folder.glob('*.bin') + for bf in bin_files: + if bf.name == "pic.bin": + bf.replace( self.new_folder.joinpath( "pic.jpg" ) ) + else: + bf.replace( self.new_folder.joinpath(bf.name) ) + pass + + def deleteOldFolder(self, ): + log.info( f"deleteOldFolder: ", __name__ ) + self.mydir.setBaseDir(self.old_folder) + self.mydir.deleteDir() + self.old_folder = DATA_DIR + + def getNewFolderFromOldFolder( self, ) -> Path: + # 服务器上转移后的模式 + tmp_folder = self.old_folder.parent.parent + path_tuple = ("20"+f"{self.info_dict['year']:02d}" + ,f"{self.info_dict['month']:02d}" + ,f"{self.info_dict['day']:02d}" + ,self.old_folder.parts[-1] ) + self.new_folder =tmp_folder.joinpath(*path_tuple) + self.mydir.setBaseDir(self.new_folder) + self.mydir.newDirIfNot() + pass + + def getInfoDict(self, ): + info_path = self.old_folder.joinpath("info.bin") + hexbin = self.read_bin(info_path) + self.info_dict = self.decode_info(hexbin) + self.new_folder = self.getNewFolderFromOldFolder() + + def getCurrentMeasureTimeFromInfoDict(self, ) -> str: + ret = "20"+ str(self.info_dict['year']) + '-' \ + + str(self.info_dict['month']) + '-' \ + + str(self.info_dict['day']) + ' ' \ + + str(self.info_dict['hour']) + ':' \ + + str(self.info_dict['minute']) + ':' \ + + str(self.info_dict['second']) + # ret = "20"+ f"{self.info_dict['year']:02d}" + '_' \ + # + f"{self.info_dict['month']:02d}" + '_' \ + # + f"{self.info_dict['day']:02d}" + '_' \ + # + f"{self.info_dict['hour']:02d}" + '_' \ + # + f"{self.info_dict['minute']:02d}" + '_' \ + # + f"{self.info_dict['second']:02d}" + return ret + pass + + def set_hhp_path(self, ): + self.hhp.setBasePath (self.base_path) + self.hhp.setDataPath (self.data_path) + self.hhp.setOutputPath (self.output_path) + + def setDeviceEnum(self, ): + if self.device_type == DeviceType.SURFACE.name: + self.device_enum = RamsesSURFACE + if self.device_type == DeviceType.AWRAMS.name: + self.device_enum = RamsesAWRAMS + if self.device_type == DeviceType.PROFILE.name: + self.device_enum = RamsesPROFILE + + def getDataFileList(self, ): + self.hhp.getDataFileList() + self.filelist = self.hhp.getFilelist() + # self.printTest(self.filelist) + pass + + # def dealAllMeasurements(self, ): + # log.info(f" 所有测量文件", __name__, "dealAllMeasurements", ) + # if len(self.filelist)<1: + # pass + # # 前面已经考虑各种文件错误 + # for df in self.filelist: + # # 处理信息帧 + # self.info_dict = {} + # self.info_path_fname:Path = df["info_path"] + # self.sensor_path_fname:Path = df["sensor_path"] + # hex_info = self.read_bin( self.info_path_fname ) + # try: + # self.info_dict = self.decode_info( hex_info ) + # except: + # log.error( f"处理信息文件" + # + "/" +self.info_path_fname.stem + # + "出现错误", __name__, "dealAllMeasurements" ) + # raise MyException( "处理文件"+ self.info_path_fname + "出现错误") + # pass + + # try: # awrams每次只有一组数据 + # self.dealOneMeasurement(self.sensor_path_fname ) + # # self.measurement_interval = int(self.info_dict['Measure_Interval']) + # # self.measurement_repeat = int(self.info_dict['Measure_Repeat']) + # # self.dealOneHandheldMeasurement(self.sensor_path_fname ) + # except Exception as e: + # log.error( "处理传感器文件" + self.sensor_path_fname.name + " 出现错误",__name__,"dealAllMeasurements") + # raise MyException( "处理传感器文件" + self.sensor_path_fname.name + " 出现错误" ) + # pass + # log.info(f"Finished !! ", __name__, "dealAllMeasurements") + # return True,self.error_result + # pass + + # def dealOneMeasurement_Handheld(self, fpath:Path): + # '''handheld一次测量包含多组数据''' + # # 调用handheldbuf 处理，将一组数据提交出来 + # log.info(f" 手持测量数据", __name__, "dealOneMeasurement_Handheld") + + # info_path = fpath.joinpath("info.bin") + # sensor_path = fpath.joinpath("sensor.bin") + + # # 信息帧解析 + # hex_info = self.read_bin( info_path ) + # try: + # self.info_dict = self.decode_info( hex_info ) + # except: + # log.error( f"处理信息文件" + # + "/" +self.info_path_fname.stem + # + "出现错误", __name__, "dealOneMeasurement_Handheld" ) + # raise MyException( "处理文件"+ self.info_path_fname + "出现错误") + # pass + + # # 当前文件名 ？ + + # self.hhp.setMode(1) + # self.current_measure_time = self.getCurrentMeasureTimeFromInfoDict() + # self.get_ymdhms() + + # # self.output_path 定位到 /data/output 目录？ + # self.output_path = OUTPUT_DIR.joinpath( self.ymdhms) + + # log.warning(f"current_measure_time: {self.current_measure_time}", __name__, "dealOneMeasurement_Handheld") + + # self.hhb.readFile2Buf(sensor_path) + # log.debug(f"buf: {self.hhb.get_buf_size()}", __name__, "dealOneMeasurement_Handheld") + # self.decode_sensor_buf() + + # # 解析Buf, 对buf分组，获得[{lsky: esky : lwater} .... ] + # len_total = len(self.intensity_before_avg) + # if len_total%self.measurement_repeat != 0: + # self.res = {} + # return # 返回退出 + # group_num = int(len_total/self.measurement_repeat) + + # log.debug(f"group_num...: {group_num}", __name__, "dealOneMeasurement_Handheld") + # self.real_wavelength = self.getWavelenthDict() + # # if group_num == 1: + # # self.dealOneGroup() # self.intensity_before_avg + # # return + + # self.dealMultiGroup(group_num) + + def dealOneMeasurement_Profiler(self, fpath:Path): + ''' + Profiler一次测量包含多组数据 + 只有一个开始十进制 + 需要处理深度，对深度分组 + ''' + # 调用handheldbuf 处理，将一组数据提交出来 + log.info(f" 剖面测量数据", __name__, "dealOneMeasurement_Profiler") + + info_path = fpath.joinpath("info.bin") + sensor_path = fpath.joinpath("sensor.bin") + + # 信息帧解析 + hex_info = self.read_bin( info_path ) + try: + self.info_dict = self.decode_info( hex_info ) + except: + log.error( f"处理信息文件" + + "/" +self.info_path_fname.stem + + "出现错误", __name__, "dealOneMeasurement_Profiler" ) + raise MyException( "处理文件"+ self.info_path_fname + "出现错误") + pass + + # 当前文件名 ？ + + self.hhp.setMode(1) + self.current_measure_time = self.getCurrentMeasureTimeFromInfoDict() + self.get_ymdhms() + + # self.output_path 定位到 /data/output 目录？ + self.output_path = OUTPUT_DIR.joinpath( self.ymdhms) + + log.warning(f"current_measure_time: {self.current_measure_time}", __name__, "dealOneMeasurement_Profiler") + + self.hhb.readFile2Buf(sensor_path) + log.debug(f"buf: {self.hhb.get_buf_size()}", __name__, "dealOneMeasurement_Profiler") + self.decode_sensor_buf() + + # 解析Buf, 对buf分组，获得[{lsky: esky : lwater} .... ] + # 可能需要 传感器序列号数据，判断几个传感器。 如何判断几个传感器？？ + len_total = len(self.intensity_before_avg) + if len_total%self.measurement_repeat != 0: + self.res = {} + return # 返回退出 + + # 剖面 - 校正文件要用水的，计算深度值 ？？ + + # group_num = int(len_total/self.measurement_repeat) + + # log.debug(f"group_num...: {group_num}", __name__, "dealOneMeasurement_Profiler") + # self.real_wavelength = self.getWavelenthDict() + # # if group_num == 1: + # # self.dealOneGroup() # self.intensity_before_avg + # # return + + # self.dealMultiGroup(group_num) + + def dealMultiGroup(self, group_num:int ): + log.info(f"group_num: {group_num}", __name__, "dealMultiGroup") + # 分组进行处理 + for i in range(group_num): + self.current_group_num = i + # 重设当前测量时间 ,measurement_interval 间隔为s + self.real_time = self.get_current_time_by_i_interval( i,self.measurement_interval) + log.warning(f" real time -> {self.real_time}") + self.res = {} + tmp_before_avg = [] + for j in range( self.measurement_repeat ): + tmp_before_avg.append( self.intensity_before_avg[j+i*self.measurement_repeat] ) + pass + self.getAvg(tmp_before_avg) + self.__do_sensor_dict_interpo() + + # 选择追加保存， 一个Lsky 可能多组文件，只能单独保存为 Lsky Esky ..Rs + # self.appendSave() + self.getLwRs() + self.checkAndSaveData( ) + path_info_txt = self.output_path.joinpath( "info.txt" ) + self.save_dict_to_file( self.info_dict, path_info_txt ) + + def get_current_time_by_i_interval(self, i, interval): + self.my_time.setCurrentTimeStr(self.current_measure_time) + self.my_time.timeDelta(seconds=i*interval) + return self.my_time.cur_time_str + pass + + def dealOneGroup(self, ): + # 分组，并获得平均值， 255个未插值结果 (依据 measurement_interval measurement_repeat) + self.getAvg( self.intensity_before_avg ) + + # 插值 + self.real_wavelength = self.getWavelenthDict() + self.__do_sensor_dict_interpo() + + self.ymdhms = "20"+ str(self.info_dict['year']) + '_' \ + + str(self.info_dict['month']) + '_' \ + + str(self.info_dict['day']) + '_' \ + + str(self.info_dict['hour']) + '_' \ + + str(self.info_dict['minute']) + '_' \ + + str(self.info_dict['second']) + + self.output_path = self.output_path.joinpath( self.ymdhms ) + self.appendSave() + + def dealOneMeasurement(self, fpath:Path): + '''handheld一次测量包含多组数据''' + # 调用handheldbuf 处理，将一组数据提交出来 + log.info(f"dealOneMeasurement: 一组测量数据", __name__, "dealOneMeasurement") + if len(self.filelist)<1: + pass + + # 当前文件名 + self.output_path = OUTPUT_DIR + self.current_filepath = fpath + self.current_measure_time = self.hhp.getCurrentMeasureTimeFromPath(fpath) + self.ymdhms = "20"+ str(self.info_dict['year']) + '_' \ + + str(self.info_dict['month']) + '_' \ + + str(self.info_dict['day']) + '_' \ + + str(self.info_dict['hour']) + '_' \ + + str(self.info_dict['minute']) + '_' \ + + str(self.info_dict['second']) + self.output_path = self.output_path.joinpath( self.ymdhms ) + log.warning(f"current_measure_time: {self.current_measure_time}", __name__, "dealOneHandheldMeasurement") + + self.hhb.readFile2Buf(fpath) + self.decode_sensor_buf() + self.real_wavelength = self.getWavelenthDict() + self.dealOneGroup() + + def dealOneMeasurement_Online(self, ): + '''handheld一次测量包含多组数据''' + # 调用handheldbuf 处理，将一组数据提交出来 + log.info(f"dealOneMeasurement_Online: 一组测量数据", __name__, "dealOneMeasurement_Online") + + # 当前文件名 + self.output_path = self.new_folder + # self.current_measure_time = self.hhp.getCurrentMeasureTimeFromPath(fpath) + self.ymdhms = self.getTimeFnameFromInfoDict() + + # getBuf + self.get_sensor_buf() + self.decode_sensor_buf() + self.real_wavelength = self.getWavelenthDict() + + self.getAvg( self.intensity_before_avg ) + self.real_wavelength = self.getWavelenthDict() + self.__do_sensor_dict_interpo() + + self.saveOnefileForLskyEskyLwaterLwRS() + + def get_sensor_buf(self,) : + buf = b"" + self.mydir.setBaseDir(self.new_folder) + my_bin_list = self.mydir.get_files_from_currentdir("*.bin") + my_bin_list = self.mydir.sort_filepath_and_check(my_bin_list) + for mbl in my_bin_list: + buf = buf + self.read_bin(mbl) + self.hhb.write_buf(buf) + pass + + def get_ymdhms(self, ): + self.ymdhms = "20"+ f"{self.info_dict['year']:02d}" + '_' \ + + f"{self.info_dict['month']:02d}" + '_' \ + + f"{self.info_dict['day']:02d}" + '_' \ + + f"{self.info_dict['hour']:02d}" + '_' \ + + f"{self.info_dict['minute']:02d}" + '_' \ + + f"{self.info_dict['second']:02d}" + + def getTimeFnameFromInfoDict(self,) : + ymdhms = "20"+ str(self.info_dict['year']) + '_' \ + + str(self.info_dict['month']) + '_' \ + + str(self.info_dict['day']) + '_' \ + + str(self.info_dict['hour']) + '_' \ + + str(self.info_dict['minute']) + '_' \ + + str(self.info_dict['second']) + return ymdhms + + def decode_sensor_buf(self,) : + # 处理Buf，对多组数据取平均 + self.intensity_before_avg = [] + # res_before_avg = [] + self.clearRes() # 清空数据 + while True: + if not self.hhb.decode_one_group_handheld() : + break # 清空数据 + + res = {} + buf = self.hhb.one_group_data[26:] + for i in range(1,4,1): + temp_buf = buf[7:71] + buf[79:143] + \ + buf[151:215] + buf[223:287] + \ + buf[295:359] + buf[367:431] + \ + buf[439:503] + buf[511:575] + + # Ramses类 标定处理，设置buf 标定文件 + self.ramses.setBuf(temp_buf) + func = self.getFuncBySeq(i) + cfg = self.getCfgByFunc( func) + self.ramses.setCalCfg(cfg) + self.ramses.resetItSpectrum() + self.ramses.ConvertAndCalibrate() + res.update({ func : self.ramses.spectrum }) + self.intensity_before_avg.append( res ) + pass + + def getAvg( self, d:list) : + log.info(f"getAvg: 平均多组数据", __name__, "getAvg") + data = d + ret = {} + len_result = len(data) + if len_result == 0: + self.intensity_after_avg ={} + return None + if len_result == 1: + self.intensity_after_avg = data[0] + return None + ret = data[0] + + res_dict = self.getRetDict() + + for k in res_dict.keys(): + for i in range(1,len_result,1): + data[0][k] = data[0][k] + data[i][k] + ret = data[0][k]/len_result + self.intensity_after_avg.update( { k : ret } ) + log.debug(f"getAvg: {self.intensity_after_avg}", __name__, "getAvg") + pass + + def getRetDict(self,) : + ret_dict = { } + ret_dict.update( {self.device_enum(1).name:np.array([])} ) + ret_dict.update( {self.device_enum(2).name:np.array([])} ) + ret_dict.update( {self.device_enum(3).name:np.array([])} ) + # self.one_group_result = ret_dict + return ret_dict + + def __do_sensor_dict_interpo(self,) : + log.info( f"同步处理多个个插值 ", __name__, "__do_sensor_dict_interpo" ) + self.clearRes() + for k in self.intensity_after_avg.keys(): + tmp = np.interp( self.wavelength, self.real_wavelength[k], self.intensity_after_avg[k] ) + self.res.update( { k : tmp } ) + + def getCfgByDid(self, func:str) : + cfg_id:dict = self.cfg.get(int(self.device_id)) + cfg_sensor = cfg_id.get( func) + return cfg_sensor + pass + + def getFuncBySeq(self, seq:int) : + func = "" + if self.device_type == DeviceType.AWRAMS.name: + func = RamsesAWRAMS(seq).name + if self.device_type == DeviceType.SURFACE.name: + func = RamsesAWRAMS(seq).name + if self.device_type == DeviceType.PROFILE.name: + func = RamsesAWRAMS(seq).name + return func + + def getCfgByFunc(self, func:str) : + cfg_id:dict = self.syscfg.get(int(self.device_id)) + cfg_sensor = cfg_id.get( func) + return cfg_sensor + pass + + def getCfgBySeq(self, seq:int) : + func = "" + if self.device_type == DeviceType.AWRAMS.name: + func = RamsesAWRAMS(seq).name + if self.device_type == DeviceType.SURFACE.name: + func = RamsesAWRAMS(seq).name + if self.device_type == DeviceType.PROFILE.name: + func = RamsesAWRAMS(seq).name + cfg_id:dict = self.syscfg.get(int(self.device_id)) + cfg_sensor = cfg_id.get( func) + return cfg_sensor + pass + + def saveOnefileForLskyEskyLwaterLwRS(self, ) -> bool: + log.info(f" ", __name__, "saveOnefileForLskyEskyLwaterLwRS") + self.mydir.setBaseDir(self.output_path) #基路径 + + time_name = self.getTimeFnameFromInfoDict() + Lw = self.res["Lwater"] - ROWFACTOR * self.res["Lsky"] + self.res.update({ self.device_enum(4).name : Lw }) + Rs = self.res[self.device_enum(4).name] / self.res["Esky"] + self.res.update({ self.device_enum(5).name : Rs }) + + self.mydir.setHeader( self.wavelength.tolist(), TOKEN, "device_id_"+str(self.device_id) ) + header = self.mydir.header_str + + save_path_csv = self.new_folder.joinpath(time_name + SAVE_EXT_NAME) + data_str = "" + for k in self.res.keys(): + self.mydir.setContent(self.res[k].tolist(),TOKEN,k ) + data_str = data_str + NEWLINE + self.mydir.content_str + save_path_csv.write_text(header+data_str) + + path_info_txt = self.new_folder.joinpath( time_name + "_info.txt" ) + self.save_dict_to_file( self.info_dict, path_info_txt ) + + def appendSave(self, ) -> bool: + '''self.output_path''' + self.checkAndSaveData( ) + self.getLwRsAndSave() + path_info_txt = self.output_path.joinpath( "info.txt" ) + self.save_dict_to_file( self.info_dict, path_info_txt ) + + def checkAndSaveData(self, ) -> bool: + """ + check self.Lsky Esky Lwater and Save + 处理self.res 的数据 + """ + log.info(f"checkAndSaveData: {self.output_path.parts}", __name__) + self.mydir.setBaseDir(self.output_path) #基路径 + self.mydir.setHeader( self.wavelength.tolist(), TOKEN, "device_id_"+str(self.device_id) ) + # print(f"header_str : {self.mydir.header_str[-1:]}") + + if self.current_group_num == 0: + self.newFileByFunc( self.device_enum(1).name ) + self.newFileByFunc( self.device_enum(2).name ) + self.newFileByFunc( self.device_enum(3).name ) + self.newFileByFunc( self.device_enum(4).name ) + self.newFileByFunc( self.device_enum(5).name ) + pass + + self.appendFileByFunc( self.device_enum(1).name ) + self.appendFileByFunc( self.device_enum(2).name ) + self.appendFileByFunc( self.device_enum(3).name ) + self.appendFileByFunc( self.device_enum(4).name ) + self.appendFileByFunc( self.device_enum(5).name ) + + def newFileByFunc(self, func:str) -> None: + self.mydir.newFileIfNot( func+SAVE_EXT_NAME) + if self.mydir.checkHeader() == -1: + log.error(f"请备份文件:{self.mydir.current_filepath.parent} {self.mydir.current_filepath.name}, 并删除文件后再试!", __name__) + raise MyException( f"请备份文件:{self.mydir.current_filepath.parent} {self.mydir.current_filepath.name}, 并删除文件后再试!" ) + return False + if self.mydir.checkHeader() == 1: + pass + if self.mydir.checkHeader() == 0: + self.mydir.writeHeader() + pass + + # 写入数据content + log.warning(f" write content time {self.real_time}") + self.mydir.setContent(self.res[func].tolist(), TOKEN, self.real_time) + self.mydir.writeContent() + + def appendFileByFunc(self, func:str) -> None: + # 追加写入数据content + self.mydir.setContent(self.res[func].tolist(), TOKEN, self.real_time) + self.mydir.writeContent() + + def clearRes(self, ) -> None: + self.res = { } + + def getLwRs(self, ) -> bool: + """ + 并计算Lw Rs并保存 + """ + Lw = self.res["Lwater"] - ROWFACTOR * self.res["Lsky"] + self.res.update({ self.device_enum(4).name : Lw }) + Rs = self.res["Lw"] / self.res["Esky"] + self.res.update({ self.device_enum(5).name : Rs }) + + return True + + def getLwRsAndSave(self, ) -> bool: + """ + 并计算Lw Rs并保存 + """ + Lw = self.res["Lwater"] - ROWFACTOR * self.res["Lsky"] + self.res.update({ "Lw" : Lw }) + Rs = self.res["Lw"] / self.res["Esky"] + self.res.update({ "Rs" : Rs }) + self.mydir.setBaseDir(self.output_path) #基路径 + + # 保存 + + if self.current_group_num == 0: + self.newFileByFunc( "Lw" ) + self.newFileByFunc( "Rs" ) + pass + + self.appendFileByFunc( "Lw" ) + self.appendFileByFunc( "Rs" ) + + return True + + def read_bin(self,fpath: Path): + log.debug(f" readbin: ", __name__, "", "" ) + ret = None + if not fpath.exists() : + log.info(f"not find file: {fpath} ") + return ret + with open(fpath, 'rb') as file: + ret = file.read() + return ret + log.debug(f" readbin: {ret} ", __name__, "", "" ) + return ret + pass + + def decode_info( self,info: bytes ) -> dict: + ret = {} + # 剖面型加了1311+24个字节共26个字节 + # 保留字节有所变化，改为序列号，每个序列号两个字节 + try: + temp = struct.unpack(" None: + temp_str = "" + for key, value in info_dict.items(): + temp_str = temp_str + key + " : " + str(value) + "\n" + with open(fpath, "w+") as f: + f.write(temp_str) + + def save_error_to_file(self, errlist: list, fpath: Path) -> None: + temp_str = "" + if len(errlist) <1: + return None + for errdict in errlist: + temp_str = temp_str + errdict["path"] +" : "+ errdict["error"] + "\n" + pass + with open(fpath, "w+") as f: + f.write(temp_str) + return None + pass + + def getWavelenthDict( self, ) -> dict: + ret_dict = self.getRetDict() + cfg_id:dict = self.syscfg.get( int(self.device_id)) + for k in ret_dict.keys(): + tmp = self.getWavelength( cfg_id.get(k) ) + ret_dict.update({ k : tmp }) + return ret_dict + + def getWavelength(self,ramsesdict:dict) -> np.ndarray: + ret = [] + for i in range(1,256): + tmp = float(ramsesdict['c0s']) + float(ramsesdict['c1s'])*i \ + + float(ramsesdict['c2s'])*i*i + float(ramsesdict['c3s'])*i*i*i + ret.append(tmp) + pass + return np.array(ret) + + def setSyscfg(self, syscfg:dict): + self.syscfg = syscfg + + def setRetrieve(self, retrieve:dict): + log.debug( f"setRetrieve : {retrieve}......", __name__ ) + self.retrieve = retrieve + self.setNewWavelength() + # self.wavelength:np.ndarray = np.arange ( self.retrieve['beginWL'], self.retrieve['endWL'], self.retrieve['interval'] ) + + def setNewWavelength(self, ): + self.wavelength:np.ndarray = np.arange ( + self.retrieve['beginWL'], self.retrieve['endWL'], self.retrieve['interval'] ) + + def setDeviceID(self, did:int): + self.device_id = did + + def setDeviceType(self, device_type): + self.device_type = device_type + + def getErrorInfoDict(self, ): + return self.error_result + + def printResult(self,): + log.info( f"***** : Print Lsky Esky Lwater Lw Rs......", __name__ ) + print(self.res[self.device_enum(1)]) + print(self.res[self.device_enum(2)]) + print(self.res[self.device_enum(3)]) + print(self.res[self.device_enum(4)]) + print(self.res[self.device_enum(5)]) + pass + + def printTest(self,d,sleep_time:int=5): + log.info( f"***** : I am testing ********", __name__ ) + print(d) + log.info( f"***** : Ending testing ********", __name__ ) + time.sleep(sleep_time) + pass + +if __name__ == "__main__": + # hh = HandHeld() + # hh.getDataFileList() + # hh.dealAllMeasurements() + + # data/2023_02/07/sensor/14_02_46 + # hhb= HandHeldBuf() + # hhb.readFile2Buf("data/2023_02/07/sensor/14_02_46") + # hhb.decode_handheld() + # print(hh.error_result) + # print("======") + # print(hh.filelist) + + # print( hh.getDataFname() ) + + # t = b'\x07' + # print( int(t[0])) + # print( 2 << int(t[0])) + + + pass + diff --git a/readcal.py b/readcal.py new file mode 100644 index 0000000..59f6fcc --- /dev/null +++ b/readcal.py @@ -0,0 +1,1008 @@ +from tools.mylogger import log +from tools.myexception import MyException +from typing import Tuple,List,Optional,Union,Callable +from pathlib import Path, PurePath +from myconfig import MyConfig,DeviceType,FILE_MARK,CURRENT_DIR,CAL_DIR,DATA_DIR,SAVE_EXT_NAME,\ + RamsesFunc,INTERVAL,SEPARATOR + +class ReadCal(): + def __init__(self) -> None: + pass + + # 数据加时间值 + @staticmethod + def read_columns_set_by_mark(fpath:Path, mark:list, *column ) -> Tuple[str,List] : + # def read_column_1set_by_mark(fpath:str, column:int, *args ) -> Tuple[ str, List ] : + ''' + return: time:str List + mark : ['Spectrum','DATA'] + default end with "[END]" + ''' + log.info(f"read_column_1set_by_mark : \ + {fpath} , mark:{mark} , column:{column}", __name__, "", "") + + # 处理mark args[0] = Spectrum + if len(mark) != 2: + log.error( f"read_column_1set_by_mark() wrong args ",__name__, "", "" ) + raise MyException( f"read_column_1set_by_mark() wrong args " ) + + if len(column) == 0: + log.error(f"read_column_1set_by_mark(), no column para, pls input column ",__name__, "", "" ) + raise MyException( f"read_column_1set_by_mark() , pls input column " ) + + if len( mark ) == 0: + mark_1 = '[Spectrum]' + mark_2 = '[DATA]' + + mark_1 = '['+ mark[0] +']' + mark_2 = '['+ mark[1] +']' + res = [] + sn = ReadCal.readFileSNbyIDDevice(fpath) + # 预备返回数据结构 + # res_time = [] + res_data = [] + columnLen = len(column) + for i in range(columnLen): + res_data.append( [] ) + + with fpath.open('r') as f_handle: + rflag = 0 + # res = [] + # res_time = [] + # res_data = [] + + for line in f_handle: + if mark_1 in line and rflag == 0: + log.debug(f" find {mark_1}", __name__, "", "" ) + rflag += 1 + continue + + if mark_2 in line and rflag > 3 : + log.debug(f" find {mark_1} end ", __name__, "", "" ) + rflag = 0 + continue + + # 获取时间 + if rflag == 1 : + data = line.strip('\n').strip(' ').split('=') + if data[0].strip() == "DateTime" : + log.debug(f" find {mark_1} -> DateTime {data[1]} ", __name__, "", "" ) + time_ = data[1].strip() + rflag += 1 + continue + + if rflag == 2 and ( mark_2 in line ) : + log.debug(f" find {mark_2} -> ", __name__, "", "" ) + rflag += 1 + continue + + if rflag == 3 : + if mark_2 not in line: + # log.debug(f" get data {line}", __name__, "", "" ) + data = line.strip('\n').strip(' ').split(' ') + # 忽略第一行，第一行0开头，保存积分时间的 + if data[0].strip() == "0" : + continue + else: + # if data[1] != "-NAN" and data[1] != "NAN" and data[1] != "+NAN" : + # 处理多列 + for i in range(columnLen): + # res_time.append( time_ ) + res_data[i].append( data[column[i]] ) + continue + + if rflag == 3 and ( mark_2 in line ) : + log.debug(f" find {mark_2} end , line: {line} ", __name__, "", "" ) + rflag += 1 + continue #需要继续让其找到 + + if rflag == 4: + log.debug(f" job done......", __name__, "", "" ) + res_time = time_ + # res_data.append( res ) + # 只读一组数据就 break + rflag += 1 + # res = [] + # rflag = 0 + + if rflag == 5: + log.debug(f" job done, break...", __name__, "", "" ) + break + return ( res_time, res_data) + pass + + # 数据加时间值 + @staticmethod + def read_columns_sets_by_mark(fpath:Path, mark:list, *column + ) -> Tuple[List[str],List[List]] : + # def read_column_1set_by_mark(fpath:str, column:int, *args ) -> Tuple[ str, List ] : + ''' + @desc: 获取多列数据，且为多套数据，非单套数据 + @return: time:List, List + @args : ['Spectrum','DATA'] + default end with "[END]" + ''' + log.info(f"read_column_1set_by_mark : \ + {fpath} , mark:{mark} , column:{column}", __name__, "", "") + + # 处理mark args[0] = Spectrum + if len(mark) != 2: + log.error( f"read_columns_sets_by_mark() wrong args ",__name__, "", "" ) + raise MyException( f"read_columns_sets_by_mark() wrong args " ) + + if len(column) == 0: + log.error(f"read_columns_sets_by_mark(), no column para, pls input column ",__name__, "", "" ) + raise MyException( f"read_columns_sets_by_mark() , pls input column " ) + + if len( mark ) == 0: + mark_1 = '[Spectrum]' + mark_2 = '[DATA]' + + mark_1 = '['+ mark[0] +']' + mark_2 = '['+ mark[1] +']' + res = [] + sn = ReadCal.readFileSNbyIDDevice(fpath) + # 预备返回数据结构 + res_data = [] + res_time = [] + res = [] + columnLen = len(column) + for i in range(columnLen): + # res_time.append( [] ) + res.append( [] ) + + with fpath.open('r') as f_handle: + rflag = 0 + + for line in f_handle: + if mark_1 in line and rflag == 0: + log.debug(f" find {mark_1}", __name__, "", "" ) + rflag += 1 + continue + + if mark_2 in line and rflag > 3 : + log.debug(f" find {mark_1} end ", __name__, "", "" ) + rflag = 0 + continue + + # 获取时间 + if rflag == 1 : + data = line.strip('\n').strip(' ').split('=') + if data[0].strip() == "DateTime" : + log.debug(f" find {mark_1} -> DateTime {data[1]} ", __name__, "", "" ) + time_ = data[1].strip() + rflag += 1 + continue + + if rflag == 2 and ( mark_2 in line ) : + log.debug(f" find {mark_2} -> ", __name__, "", "" ) + rflag += 1 + continue + + if rflag == 3 : + if mark_2 not in line: + # log.debug(f" get data {line}", __name__, "", "" ) + data = line.strip('\n').strip(' ').split(' ') + # 忽略第一行，第一行0开头，保存积分时间的 + if data[0].strip() == "0" : + continue + else: + # if data[1] != "-NAN" and data[1] != "NAN" and data[1] != "+NAN" : + # 处理多列 + for i in range(columnLen): + # res_time.append( time_ ) + res[i].append( data[column[i]] ) + continue + + if rflag == 3 and ( mark_2 in line ) : + log.debug(f" find {mark_2} end , line: {line} ", __name__, "", "" ) + rflag += 1 + continue #需要继续让其找到 + + if rflag == 4: + log.debug(f" job done......", __name__, "", "" ) + res_time.append( time_ ) + res_data.append( res ) + rflag += 1 + # res = [] + # rflag = 0 + + if rflag == 5: + log.debug(f" job done, break...", __name__, "", "" ) + rflag = 0 + continue + + log.debug(f" res.time len {len(res_time)} , \ + res.data len {len(res_data)} , ", __name__, "", "" ) + + return ( res_time, res_data) + + # 数据加时间值 + @staticmethod + def read_columns_sets_by_mark_callback(fpath:Path, mark:list, callback:Callable, *column ) : + ''' + @desc: 获取多列数据，且为多套数据，非单套数据 + @return: time:List, List + @args : ['Spectrum','DATA'] + default end with "[END]" + ''' + log.debug(f"read_column_1set_by_mark : \ + {fpath} , mark:{mark} , column:{column}", __name__, "", "") + + # 处理mark args[0] = Spectrum + if len(mark) != 2: + log.error( f"read_columns_sets_by_mark() wrong args ",__name__, "", "" ) + raise MyException( f"read_columns_sets_by_mark() wrong args " ) + + if len(column) == 0: + log.error(f"read_columns_sets_by_mark(), no column para, pls input column ",__name__, "", "" ) + raise MyException( f"read_columns_sets_by_mark() , pls input column " ) + + if len( mark ) == 0: + mark_1 = '[Spectrum]' + mark_2 = '[DATA]' + + mark_1 = '['+ mark[0] +']' + mark_2 = '['+ mark[1] +']' + + # 预备返回数据结构 + res_data = [] + res_time = [] + res = [] + sn = ReadCal.readFileSNbyIDDevice(fpath) + + columnLen = len(column) + for i in range(columnLen): + # res_time.append( [] ) + res.append( [] ) + + with fpath.open('r') as f_handle: + rflag = 0 + + for line in f_handle: + if mark_1 in line and rflag == 0: + log.debug(f" find {mark_1}", __name__, "", "" ) + rflag += 1 + continue + + if mark_2 in line and rflag > 3 : + log.debug(f" find {mark_1} end ", __name__, "", "" ) + rflag = 0 + continue + + # 获取时间 + if rflag == 1 : + data = line.strip('\n').strip(' ').split('=') + if data[0].strip() == "DateTime" : + log.debug(f" find {mark_1} -> DateTime {data[1]} ", __name__, "", "" ) + time_ = data[1].strip() + rflag += 1 + continue + + if rflag == 2 and ( mark_2 in line ) : + log.debug(f" find {mark_2} -> ", __name__, "", "" ) + rflag += 1 + continue + + if rflag == 3 : + if mark_2 not in line: + # log.debug(f" get data {line}", __name__, "", "" ) + data = line.strip('\n').strip(' ').split(' ') + # 忽略第一行，第一行0开头，保存积分时间的 + if data[0].strip() == "0" : + continue + else: + # if data[1] != "-NAN" and data[1] != "NAN" and data[1] != "+NAN" : + # 处理多列 + for i in range(columnLen): + res[i].append( data[column[i]] ) + continue + + if rflag == 3 and ( mark_2 in line ) : + log.debug(f" find {mark_2} end , line: {line} ", __name__, "", "" ) + rflag += 1 + continue #需要继续让其找到 + + if rflag == 4: + log.debug(f" job done......", __name__, "", "" ) + # callable , not return。 每读出一组就回调，不建议。频繁读写硬盘 + # callback(time_, res) + res_time.append( time_ ) + res_data.append( res ) + rflag += 1 + # res = [] + # rflag = 0 + + if rflag == 5: + log.debug(f" job done, break...", __name__, "", "" ) + rflag = 0 + continue + + # callable , not return。 一个文件读完直接处理 + + callback(sn, res_time, res_data) + + log.debug(f" res.time len {len(res_time)} , \ + res.data len {len(res_data)} , ", __name__, "", "" ) + + # return ( res_time, res_data) + + @staticmethod + def readDataIPinfo(fpath:Path, mark:list, callback:Callable ): + ''' + SAMIP sensor ,Inclination Pressure, InclX InclY Pressure + ''' + log.debug(f"readDataIPinfo : {fpath} , mark:{mark} , ", __name__, "", "") + + # 处理mark args[0] = Spectrum + if len(mark) != 2: + log.error( f"readDataIPinfo() wrong args ",__name__, "", "" ) + raise MyException( f"readDataIPinfo() wrong args " ) + + if len( mark ) == 0: + mark_1 = '[Spectrum]' + mark_2 = '[DATA]' + + mark_1 = '['+ mark[0] +']' + + # 预备返回数据结构 + res_data = [] + res_time = [] + res = [] + tags = ['InclX' , 'InclY', 'Pressure'] + + with fpath.open('r') as f_handle: + rflag = 0 + for line in f_handle: + if mark_1 in line and rflag == 0: + log.debug(f" find {mark_1}", __name__, "", "" ) + rflag += 1 + continue + if mark_1 in line and rflag > 5: + log.debug(f" find {mark_1} end", __name__, "", "" ) + rflag = 0 + continue + # 获取时间 + if rflag == 1 : + data = line.strip('\n').strip(' ').split('=') + if data[0].strip() == "DateTime" : + log.debug(f" find {mark_1} -> DateTime {data[1]} ", __name__, "", "" ) + time_ = data[1].strip() + rflag += 1 + continue + if rflag == 2: + if "[" not in line : + data = line.strip('\n').strip(' ').split('=') + # RAMSES 数据通过MethodName 获得传感器序列号 + if data[0].strip() in tags : + res.append( data[1].strip(' ') ) + rflag += 1 + if rflag == 5: + res_time.append(time_) + res_data.append(res) + rflag = 0 + pass + callback(res_time,res_data) + return (res_time,res_data) + + @staticmethod + def readSamSNFromIni( fpath:Path ): + """ + @description : 依据文件路径获得samsn, 兼容 SAM SAMIP传感器 + """ + + with fpath.open('r') as f_handle: + for line in f_handle: + # IDDevice = SAM_85AF + ln = line.strip('\n').strip(' ').split('=') + if ln[0].strip() == "IDDevice": + if ln[1].strip().split('_')[0] == "SAM": + return ln[1].strip().split('_')[1] + return None + pass + + @staticmethod + def readIPCalFromIni( fpath:Path ): + """ + @description : 依据文件路径获得IPCal IP标定信息 + """ + ipcal = {} + with fpath.open('r') as f_handle: + flag = 0 + for line in f_handle: + # IDDevice = SAM_85AF + ln = line.strip('\n').strip(' ').split('=') + + if flag == 0 and ln[0].strip() == "IDDevice": + if ln[1].strip().split('_')[0] == "IP": + ipcal["IPSN"] = ln[1].strip().split('_')[1] + flag += 1 + continue + + if flag == 1 and "[" in line: + flag += 1 + continue + + if flag == 2: + if "[" in line: + flag = flag + 1 + + if not "[" in line: + line_ = line.strip('\n').strip(' ').split('=') + ipcal.update({line_[0].strip():line_[1].strip()}) + + if flag == 3: + break + + return ipcal + + @staticmethod + def readSAMCalFromIni( fpath:Path ): + """ + @description : 依据文件路径获得SAMCal SAM标定信息 + """ + samcal = {} + with fpath.open('r') as f_handle: + flag = 0 + for line in f_handle: + # IDDevice = SAM_85AF + line_ = line.strip('\n').strip(' ').split('=') + if flag == 0 and line_[0].strip() == "IDDevice": + if line_[1].strip().split('_')[0] == "SAM": + samcal["SAMSN"] = line_[1].strip().split('_')[1] + flag += 1 + continue + + if flag == 1 and "[" in line: + flag += 1 + continue + + if flag == 2: + if "[" in line: + flag = flag + 1 + + if not "[" in line: + line_ = line.strip('\n').strip(' ').split('=') + samcal.update({line_[0].strip():line_[1].strip()}) + + if flag == 3: + break + return samcal + + @staticmethod + def readFileSNbyIDDevice(fpath:Path): + sn_from_file = '' + with fpath.open('r') as f_handle: + rflag = 0 + for line in f_handle: + if '[Spectrum]' in line and '[END]' not in line and rflag == 0: + rflag += 1 + pass + if rflag == 1: + data = line.strip('\n').strip(' ').split('=') + # RAMSES 数据通过MethodName 获得传感器序列号 + if data[0].strip() == "IDDevice": + sn_from_file = data[1].strip(' ').split('_')[1] + break + return sn_from_file + + @staticmethod + def readDatafileSNbyMethodName(fpath:Path): + ''' + SAMIP SAM的数据文件中, MethodName对应的值都是 SAM_{SAMSN} + 读SAMIP的数据,建议用 {SAMSN}的处理 + ''' + sn_from_file = '' + with fpath.open('r') as f_handle: + rflag = 0 + for line in f_handle: + if '[Spectrum]' in line and '[END]' not in line and rflag == 0: + rflag += 1 + pass + if rflag == 1: + data = line.strip('\n').strip(' ').split('=') + # RAMSES 数据通过MethodName 获得传感器序列号 + if data[0].strip() == "MethodName": + sn_from_file = data[1].strip(' ').split('_')[1] + break + return sn_from_file + + +if __name__ == "__main__": + log.info(f"******** main read *********", __name__, "", "") + + # path = PurePath() + # print(path) + + pass + + # def read_data_sn(self,fpath): + # with open(fpath, 'r') as f_handle: + # rflag = 0 + # for line in f_handle: + # if '[Spectrum]' in line and '[END]' not in line and rflag == 0: + # rflag += 1 + # pass + # if rflag == 1 : + # data = line.strip('\n').strip(' ').split('=') + # if data[0].strip() == "IDDevice" : + # return data[1].strip(' ').split('_')[1] + # pass + + # def read_data1(self,fpath): + # with open(fpath, 'r') as f_handle: + # rflag = 0 + # res = [] + # res_time = [] + # res_data = [] + + # for line in f_handle: + # if '[Spectrum]' in line and '[END]' not in line and rflag == 0: + # rflag += 1 + # pass + + # if '[Spectrum]' in line and '[END]' in line : + # rflag = 0 + # pass + + # # 获取时间 + # if rflag == 1 : + # data = line.strip('\n').strip(' ').split('=') + # if data[0].strip() == "DateTime" : + # time_ = data[1].strip() + # rflag += 1 + + # if rflag == 2 and ( '[DATA]' in line or '[Data]' in line ) : + # rflag += 1 + + # if rflag == 3 : + # data = line.strip('\n').strip(' ').split(' ') + + # if int(data[0].strip()) == 0 : + # pass + # else: + # # if data[1] != "-NAN" and data[1] != "NAN" and data[1] != "+NAN" : + + # res.append( float(data[1] ) ) + # if rflag == 3 and '[' in line and ']' in line: + # rflag += 1 + # if rflag == 4: + # # 这个地方 + # # self. __save_absorbance_data( res_time, res, sn ) + # res_time.append( time_ ) + # res_data.append( res ) + # # ?? 可以调用函数直接处理 + # # DealResult.deal_one_data(time_,res) + # res = [] + # rflag = 0 + # # return [['data']] + # pass + + + # def read_data_column( self,fpath, column=1 ): + # column_ = 1 + # if int(column_) : + # column_ = int(column_) + # pass + # else: + # pass + + # with open(fpath, 'r') as f_handle: + # rflag = 0 + # res = [] + # res_time = [] + # res_data = [] + + # for line in f_handle: + # if '[Spectrum]' in line and '[END]' not in line and rflag == 0: + # rflag += 1 + # pass + + # if '[Spectrum]' in line and '[END]' in line : + # rflag = 0 + # pass + + # # 获取时间 + # if rflag == 1 : + # data = line.strip('\n').strip(' ').split('=') + # if data[0].strip() == "DateTime" : + # time_ = data[1].strip() + # rflag += 1 + + # if rflag == 2 and ( '[DATA]' in line or '[Data]' in line ) : + # rflag += 1 + + # if rflag == 3 : + # data = line.strip('\n').strip(' ').split(' ') + + # if int(data[0].strip()) == 0 : + # pass + # else: + # res.append( float(data[column_] ) ) + # if rflag == 3 and '[' in line and ']' in line: + # rflag += 1 + # if rflag == 4: + # # 这个地方 + # # self. __save_absorbance_data( res_time, res, sn ) + # # res_time.append( time_ ) + # # res_data.append( res ) + # # ?? 可以调用函数直接处理 + # return time_, res + + # # return [['data']] + # pass + + # def read_data_wl( self, fpath ): + # wl = self.read_data_column( fpath, column=0 ) + # pass + + # def get_cal_file(self,sn): + # sn_ = sn + # file_ = {} + # if open( os.path.join( CAL_PATH, "SAM_"+sn_+"ini" ),"r"): + # file_ = { + # "type": "SAM", + # "back":"Back_SAM_", + # "cal":"Cal_SAM_", + # "calaq":"CalAQ_SAM_", + # "ini":"SAM_", #ini + # "immersion":"immersion_factors_Lu.DAT" + # } + # return file_ + # if open( os.path.join( CAL_PATH, "SAMIP_"+ sn_ +"_ALL.ini" ),"r"): + # file_ = { + # "type": "SAMIP", + # # "back":"Back_SAM_" + sn_ + CAL_EXT_NAME, + # # "cal":"Cal_SAM_" + sn_ + CAL_EXT_NAME, + # # "calaq":"CalAQ_SAM_" + sn_ + CAL_EXT_NAME, + # "ini":"SAMIP_"+sn+"_ALL.ini", #ini + # "immersion":"immersion_factors_Lu.DAT" + # } + # if file_['ini'] : + # samsn = self.read_ini(file_['ini'], "SAMSN") + # file_.update("cal", "Cal_SAM_" + samsn + CAL_EXT_NAME) + # file_.update("calaq", "CalAQ_SAM_" + samsn + CAL_EXT_NAME) + # file_.update("back", "Back_SAM_" + samsn + CAL_EXT_NAME) + # return file_ + # return + + # def read_cal(self,sn): + # # SAM SAMIP 不同 + # self.cal_data= {} + # # 1. 依据序列号读 ini, 判读是否有文件 SAM_8578.ini SAMIP_50BB_ALL.ini + # self.calfile = self.get_cal_file(sn) # 其中sam的是错误的 + + # # 2 读ini 文件错误 + + # cal_data = {} + # for key in self.calfile: + # if key == "immersion" : + # path_ = os.path.join( CAL_PATH, self.calfile[key] ) + # content_ = [self.read_data_column( path_, column=0 )[1]] + # content_.append( self.read_data_column( path_, column=0 )[1] ) + # cal_data.update({key:content_}) + + # if key == "cal" : + + # path_ = os.path.join( CAL_PATH, self.calfile[key] ) + # content_ = self.read_data_column( path_, column=0 ) + # cal_data.update({key:content_}) + + # if key == "back" : + # path_ = os.path.join( CAL_PATH, self.calfile[key] ) + # content_ = self.read_data_column( path_, column=0 ) + # cal_data.update({key:content_}) + + # if key == "calaq" : + # path_ = os.path.join( CAL_PATH, self.calfile[key] ) + # content_ = self.read_data_column( path_, column=0 ) + # cal_data.update({key:content_}) + + # if key == "ini" : + # cal_data.update({ "samcal" : self.read_ini( os.path.join( CAL_PATH, self.calfile[key] ), "SAMCAL" ) }) + # cal_data.update({ "ipcal" : self.read_ini( os.path.join( CAL_PATH, self.calfile[key] ), "IPCAL" ) }) + + # pass + + # DealResult.deal_cal(cal_data) + + # pass + + # def read_data(self,fname,lst): + # """ + # @description : 线程守护读取数据文件 ，循环读取 + # 考虑一次读取多组,只读一列的情况 + # sn ,begin,end, 指定读的列 + # 返回：[ [[第一组时间],[第二组时间] .... ], [ [第一组data],[第二组data] ...] ] + # RAMSES 要考虑 : InclX = -6.42, InclY = -6.5625 , Pressure = 6.61394049205538 + # 需要读一组波长出来 + # """ + # log.info( ":::::::::::: Class : %s -> Function :__read_data " % ( __name__, ) ) + # log.info( " __read_data : lst %s" % ( lst, ) ) + # sn = lst[0] + # begin = lst[1] + # end = lst[2] + # column = lst[3] + # samip_sn= lst[4] + + # # 检查 sn--- 不检查sn ，读数据就 成csv + + # # 读取一组数据的0列作为波长？？？ + + # # if is_sn_ok : + + # log.info( "正在处理文件 .... " + fname ) + # with open(fname, 'r') as f_handle: + # time_ = "" + # rflag = 0 + # intlx = 0.0 + # intly = 0.0 + # pressure = 0.0 + # res = [] + # res_time = [] + # res_data = [] + + # for line in f_handle: + # if rflag == 0 and '[Spectrum]' in line and '[END]' not in line : + # rflag += 1 + # pass + + # if '[Spectrum]' in line and '[END]' in line : + # rflag = 0 + # pass + + # # 获取时间, InclX, InclY ,Pressure + # if rflag == 1 : + # data = line.strip('\n').strip(' ').split('=') + # if data[0].strip() == "DateTime" : + # time_ = data[1].strip() + # rflag += 1 + # # InclX, InclY ,Pressure + + # if rflag == 2: + # data = line.strip('\n').strip(' ').split('=') + + # if data[0].strip() == "InclX" : + # intlx = data[1].strip() + + # if data[0].strip() == "InclY" : + # intly = data[1].strip() + + # if data[0].strip() == "Pressure" : + # pressure = data[1].strip() + + + # # DATA set 结束添加数据 + # if rflag == 4 and '[DATA]' in line and '[END]' in line: + # # time.sleep(10) + # res_time.append( [time_,intlx,intly,pressure] ) + # res_data.append( res ) + # res = [] + # rflag += 1 + + # if rflag == 3 : + # data = line.strip('\n').strip(' ').split(' ') + # # print("rflag %s +++++++++++++++++++++++++++++++++ %s" %(rflag,data) ) + # if data[0] != "0" and data[1] != "-NAN" and data[1] != "NAN" and data[1] != "+NAN" : #去掉第一行 0， 对应的积分时间 2^（n+1） + # # print("append %s -- %s" %(data[0],data[1]) ) + # res.append( float( data[column-1] ) ) + # if data[0] != "0" and float(data[0]) > 950 : + # # print("> 950... %s %s" %(data[0],data[1]) ) + # rflag += 1 + + # # 处理data 部分 + # if rflag == 2 and '[DATA]' in line and '[END]' not in line: + # # print("rflag %s +++++++++++++++++++++++++++++++++ data+1 " %(rflag,) ) + # rflag += 1 + + + # if rflag == 5: + # DealResult.deal_ramses_data(time_, [[intlx,intly,pressure], res]) + # # 这个地方 + # # self. __save_absorbance_data( res_time, res, sn ) + # # print("rflag 4 ......................") + # # res_time.append( [time_,intlx,intly,pressure] ) + # # print(" append time %s " %res_time) + # # res_data.append( res ) + # res = [] + # rflag = 0 + # intlx = 0.0 + # intly =0.0 + # pressure =0.0 + # # else: + # # return [] + + # log.info( "已经转换完文件，正在保存... " + fname ) + + # # 分析处理数据，还是返回原始数据 或空 ？？ + # log.info( " Function :__read_data %s - %s " % ( res_time[0] ,res_data[0] ) ) + + + # # # 光强保存文件路径 + # # fpath = os.path.join(INTENSITY_PATH, samip_sn + SAVE_EXT_NAME ) + # # fpath_IP = os.path.join(INTENSITY_PATH, samip_sn + "_IP" + SAVE_EXT_NAME ) + + # # for i in range ( len(res_time) ) : + # # # print("1111111111..............%s %s " %(res_time[i],res_time[i][0]) ) + # # # time.sleep(2) + # # ProcessFile.save_time_list( str(res_time[i][0]) , res_data[i], TOKEN, fpath ) + # # # 不是samip 不保存 IP 文件 + # # if sn != samip_sn: + # # ProcessFile.save_time_list( str(res_time[i][0]) , res_time[i][1:], TOKEN, fpath_IP ) + + + # # log.info( "保存结束 ... " + fname ) + + # # 返回 ok 表示文件处理结束 + # return + # pass + + # def read_ini(self,fname ,type_): + # """ + # @description : 三个不同任务， SAM　SN， SAM ATTR, IP ATTR + # 分别记为 SAMSN SAMCAL IPCAL + # """ + # log.info( " RAMSES __read_ini -> lst -> %s " % type_) + # typ = type_ + # data = [] + + # if typ == "SAMSN" : + + # with open(fname, 'r') as f_handle: + # for line in f_handle: + # # IDDevice = SAM_85AF + # ln = line.strip('\n').strip(' ').split('=') + # if ln[0].strip() == "IDDevice" : + # if ln[1].strip().split('_')[0] =="SAM" : + # return ln[1].strip().split('_')[1] + # pass + + # if typ == "SAMCAL" : + # samcal= {} + # with open(fname, 'r') as f_handle: + # flag = 0 + # for line in f_handle: + # # IDDevice = SAM_85AF + # line_ = line.strip('\n').strip(' ').split('=') + # if flag == 0 and line_[0].strip() == "IDDevice" : + # if line_[1].strip().split('_')[0] =="SAM" : + # samcal["SAMSN"] = line_[1].strip().split('_')[1] + # flag += 1 + + # if flag == 2 and "[END]" in line: + # flag += 1 + + # if flag == 2: + # # print(" 1 %s %s %s" % (fname,line_[0],line_[1])) + # samcal[line_[0]] = line_[1] + + # if flag == 1 and line_[0].strip() == "[ATTRIBUTES]" : + # flag += 1 + + # if flag == 2 and "[END]" in line: + # flag += 1 + + # return samcal + + # pass + + # if typ == "IPCAL" : + # ipcal= {} + # with open(fname, 'r') as f_handle: + # flag = 0 + # for line in f_handle: + # # IDDevice = SAM_85AF + # ln = line.strip('\n').strip(' ').split('=') + + # if flag == 0 and ln[0].strip() == "IDDevice" : + # if ln[1].strip().split('_')[0] =="IP" : + # ipcal["IPSN"] = ln[1].strip().split('_')[1] + # flag += 1 + + # if flag == 2 and "[END]" in line: + # flag += 1 + + # if flag == 2: + # ipcal[ln[0]] = ln[1] + + # if flag == 1 and ln[0].strip() == "[ATTRIBUTES]" : + # flag += 1 + + # return ipcal + + # return + # pass + + # def read_raw( self, fname , lst, uiraw): + # """ + # @description : 三个不同任务， SAM　SN， SAM ATTR, IP ATTR + # 对波长进行标定处理 + # 并存入文件 + # λ(N) = C0s + C1s·N + C2s· N2 + C3s·N3 + # """ + + # sep = uiraw["1"] + # air_water = int(uiraw["2"]) + + # with open(fname, 'r') as f_handle: + # sn = "" + # wl = "" + # is_samip = 0 + # # data = data[2:] + # # cal_data = lst[sn] + # wl_cal = [] + # cal_data = [] + # intg_time = 128 + # ip_cal = [] + # cal = [] + # calaq = [] + # b0 = [] + # b1 = [] + + # for line in f_handle: + # data = line.split(",") + # sn_d = data[0] + + # if sn_d != sn : + # sn = sn_d + # intg_time = int( data[1] ) + # cal_data = lst[sn] + # # 判断sam SAMIP + # if cal_data[0][0] != cal_data[0][1] : + # is_samip = 1 + + # wl_cal = cal_data[1] + # ip_cal = cal_data[2] + + # if air_water == 1 : + # cal = cal_data[3] + # else: + # cal = cal_data[2] + + # b0 = cal_data[4] + # b1 = cal_data[5] + + # for i in range(1,256): + # temp = float(wl_cal["C0s"]) + \ + # float(wl_cal["C1s"]) * i + \ + # float(wl_cal["C2s"]) * i * i + \ + # float(wl_cal["C3s"]) * i *i *i + # wl.append(temp) + + + + # # 处理数据 + # data = data[2:] + # intensity = [] + # Cn = [] + + # for i in range( 255 ): + # # NAN 为 0 + # # if cal[i] = "+NAN": + # # intensity.append(0) + # # else: + # Mn = data[i] / 65535 + # Bn = float( b0[i] ) + float( b1[i] ) * intg_time / 8192 #积分时间比 + # Cn.append(Bn - Mn) + + # offset = 0 # DarkPixelStart = 237 DarkPixelStop = 254的平均值. 数组中236-254 + # for i in range( int( wl_cal['DarkPixelStart']) -1 , int( wl_cal['DarkPixelStop']) ): + # offset = offset + float( Cn[i] ) + # offset = offset/ ( int( wl_cal['DarkPixelStop']) - int( wl_cal['DarkPixelStart']) +1 ) + + # for i in range( 255 ): + # Dn = Cn[i] - offset + # En = 8192 * Dn / intg_time + + # # ???? Sn 来自 cal calaq 文件的部分 + # Sn = cal[i] + # intensity.append( En / Sn ) + + # # 还需要处理传感器的 IP + # if is_samip : + # # ip 数据过来后的格式 + # pass + + # return intensity + + # # 数据分别写到文件 diff --git a/receive.py b/receive.py new file mode 100644 index 0000000..1750e9f --- /dev/null +++ b/receive.py @@ -0,0 +1,476 @@ +#!/usr/bin/env python +# coding:utf-8 + +''' +# 因为图片帧不是必须，必须按帧处理数据。 可能摄像头坏掉，没有传感器数据？？ +# 按帧处理数据，必须在每帧接收完毕判断，数据是否完整， 完整则进一步处理 ！！！！！！！！！！！！ +# 时间作为目录 +''' +import socket +import socketserver +from socketserver import TCPServer,ThreadingMixIn +import threading +# import datetime +import time +# import os +import struct +from tools.mypath import MyDir +from tools.mylogger import log +from pathlib import Path,PurePath +from myconfig import DATA_DIR,DeviceType +from profiler import Profiler + + +IP = "" +PORT = 7887 +ADDRESS = (IP, PORT) # 绑定地址 + +# LOGGING_LEVEL = logging.DEBUG +# LOGGING_LEVEL = logging.INFO +# LOGGING_LEVEL = logging.WARNING + +DATA_FRAME_HEAD = b'\x11\x13\x55\xaa' +DATA_FRAME_TAIL = b'\xff\xd9' +PIC_BEGIN_BYTES = b'\xff\xd8' + +# 连接超时 +TIMEOUT_SECOND = 8 * 3600 + +# 连接线程池 +conn_pool = [] + +# save_path = Path + +class MyTCPServer(TCPServer): + def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True, cfg=None, retrieve=None): + self.cfg = cfg + self.retrieve = retrieve + TCPServer.__init__(self, server_address, RequestHandlerClass, bind_and_activate=True ) + +class MyThreadingTCPServer(ThreadingMixIn, MyTCPServer): pass + +class MyException(Exception): + def __init__(self, message="自定义异常"): + self.message = message + + +class illumination_sensor: + def __init__(self, socket: socket.socket) -> None: + self.__buf = b'' + self.__head = { + # "id" : -1, # 设备id + # 'type' : -1, # 类型 信息 传感器 图片 + # 'num' : -1, # 光学传感器的第几次测量 + # 'con' : -1, # 总的测量序号 + # 'size' : -1, # 字节大小 + # 'packet_con' : -1, # 第几帧 + # 'packet_all' : -1, # 总帧数 + # 'head' : b'', # 帧头 + # 'payload' : b'' # 具体内容 + } + self.state = 0 + self.socket = socket + self.id = 0 + # self.is_data_complete = { + # "info_frame": False, + # 'sensor_frame': False, + # 'pic_frame': False + # } + # self.data = { + # "info_frame": {}, + # 'sensor_frame': {}, + # 'pic_frame': {} + # } + self.timeout_base = int(time.time()) + + def set_id(self, id) -> None: # 对应设备ID + self.id = id + + # def set_socket(self,socket:socket.socket) -> socket.socket: + # tmp = self.socket + # self.socket=socket + # return tmp + + def read_buf(self, size: int) -> bytes: + if size > self.__buf.__len__(): + return b'' + ret = self.__buf[0:size] + self.__buf = self.__buf[size:] + return ret + + def write_buf(self, buf: bytes) -> None: + id = self.id + len = buf.__len__() + # logging.info(f'Received ID:{id} Size:{len}') + self.__buf = self.__buf+buf + + def get_buf_size(self) -> int: + return self.__buf.__len__() + + def back_bytes(self, buf: bytes) -> None: + self.__buf = buf+self.__buf + + def reset_head(self) -> None: + self.__head = {} + # self.__head['id'] = -1 + # self.__head['type'] = -1 + # self.__head['num'] = -1 + # self.__head['con'] = -1 + # self.__head['size'] = -1 + # self.__head['packet_con'] = -1 + # self.__head['packet_all'] = -1 + # self.__head['head'] = b'' + # self.__head['payload'] = b'' + + def reset_data(self) -> None: + self.data['info_frame']: dict = {} + self.data['sensor_frame']: dict = {} + self.data['pic_frame']: dict = {} + + def decode(self) -> dict: + if self.__head == {}: + while self.get_buf_size() >= 15: + if self.read_buf(1) != b'\x11': + continue + c = self.read_buf(1) + if c != b'\x13': + self.back_bytes(c) + continue + c = self.read_buf(1) + if c != b'\x55': + self.back_bytes(c) + continue + c = self.read_buf(1) + if c != b'\xaa': + self.back_bytes(c) + continue + head = self.read_buf(11) + head_list = struct.unpack('> 4 + self.__head['num'] = head_list[1] & 0x0f + self.__head['con'] = head_list[2] + self.__head['size'] = head_list[3] + self.__head['packet_con'] = head_list[4] + self.__head['packet_all'] = head_list[5] + self.__head['head'] = b'\x11\x13\x55\xaa'+head + break + + if self.__head != {}: + # 判断是否是测量结束标志 + if self.__head['type'] == 15: + log.info(f" end frame recieved! ") + self.__head['payload'] = b'\xff\xff' + data = self.__head.copy() + self.__head = {} + self.id = data['id'] + return data + pass + payload = self.read_buf(self.__head['size']) + if payload != b'': + self.__head['payload'] = payload + data = self.__head.copy() + self.__head = {} + self.id = data['id'] + return data + + return {} + +class DealData: + """ + @description : 调用AWRAMS类处理数据 + @param : + @Returns : + """ + + def __init__(self) -> None: + self.device_id = None + self.devie_type = DeviceType.SURFACE.name + self.measure_id = None + self.cfg = {} + self.profiler = None + pass + + def deal(self, id: int, con: int, cfg:dict , retrieve) -> None: # 取字典中的 payload + log.info(f" 接收到数据开始处理数据 device_id {id} ") + self.device_id = id + self.measure_id = con + if self.device_id is None: + self.device_id = id + if self.cfg == {}: + self.cfg = cfg + # self.cfg = cfg.get(self.device_id) + if self.profiler is None: + self.profiler = Profiler() ##处理数据 + self.profiler.setSyscfg(self.cfg) + self.profiler.setRetrieve(retrieve) + self.profiler.setDeviceID(self.device_id) + self.profiler.setMeasureID(self.measure_id) + path_tuple = ( "data", str(id), str(con) ) + self.profiler.setOldFolder( path_tuple ) + self.profiler.getInfoDict( ) + self.profiler.transferFromOldFolder() + self.profiler.deleteOldFolder() + self.profiler.dealOneMeasurement_Profiler(self.profiler.new_folder) + log.info(f" Complete Dealing one group.") + # self.profiler.readOneFolder( ) + + + @staticmethod + def read_bin(fpath:Path): + ret = None + if fpath.exists() == False: + log.warning(f"not find file: {fpath} ") + return ret + ret = fpath.read_bytes() + # with open( fpath, 'rb') as file: + # ret = file.read() + # return ret + return ret + pass + + @staticmethod + def decode_info(info: bytes) -> dict: + ret = {} + try: + temp = struct.unpack( "None: + temp_str = "" + for key, value in info_dict.items(): + temp_str = temp_str + key + " : " + str(value) + "\n" + with open(fpath, "w+") as f: + f.write(temp_str) + + ret = None + if fpath.exists() == False: + log.info(f"not find file: {fpath} ") + return ret + with open(fpath, 'rb') as file: + ret = file.read() + return ret + return ret + pass + + @staticmethod + def check_spectrum_data(dst_dir:Path): + # 判断目录下是否有 0.bin ...15.bin 文件 + sensor_file_list = dst_dir.glob( '*[0-9].bin' ) + fname_without_ext = [] + for fl in sensor_file_list: + temp = fl.stem + if not temp.isdigit: + log.warning( f" {dst_dir} 目录光谱文件的文件名 {temp} 不为数字,type:{type(temp)}，请检查异常" ) + return False + fname_without_ext.append( int(temp) ) + + if len(fname_without_ext) ==0: + log.warning( f" {dst_dir} 目录没有发现光谱文件，请检查异常" ) + return False + + # 排序，然后检查是否有遗漏项 + fname_without_ext.sort() + for i in fname_without_ext: + if fname_without_ext[i] !=i: + log.warning( f" {dst_dir} 目录,序号{i}光谱文件的文件名没有发现，请检查异常" ) + return False + + return False + pass + + @staticmethod + def calibrate_spectrum_data(dst_dir): + ''' + 用ini back 等文件获得标定后数据 + ''' + log.info("calibrate_spectrum_data.... ") + pass + + @staticmethod + def retrieve_data(dst_dir): + ''' + 反演遥感反射率等参数 + ''' + log.info(" retrieve_data.... ") + pass + + # 收到的全部扔到 sensor.bin, 追加保存 + def save(self,data: dict) -> None: + log.info(f"save .....to first dir {str(data['con'])} - type:{data['type']} - num {data['num']}") + # 路径 传感器id/测量序号（唯一） -- 处理数据时候改时间 + saveDir = DATA_DIR.joinpath(str(data['id']), str(data['con']) ) + if saveDir.exists() == False: + saveDir.mkdir(parents=True) + + if data['type'] == 0: + log.debug( f" {data['type']} - {data['num']}") + fpath = saveDir.joinpath("info.bin") + fpath.write_bytes( data['payload'] ) + elif data['type'] == 1: + log.debug( f" {data['type']} - {data['num']}") + senor_path = saveDir.joinpath( "sensor.bin" ) + with open( senor_path, "ab+") as f: + f.write(data['payload']) + pass + elif data['type'] == 2: + pass + else: + pass + + +# class MyServer(socketserver.BaseRequestHandler): +class MyServer(socketserver.BaseRequestHandler): + + def setup(self) -> None: + log.debug(f"retrieve {self.server.retrieve}",__name__, "", "" ) + self.cfg =self.server.cfg + self.retrieve =self.server.retrieve + self.sk: socket.socket = self.request + self.sensor = illumination_sensor(self.request) + self.dealData = DealData() + self.begin_time = time.time() + conn_pool.append(self.client_address) + pass + + def handle(self) -> None: + + log.info('... connected from {}'.format(self.client_address),'__name__') + while True: + # self.request.recv 方法接收客户端发来的消息 + try: + data_byte = self.sk.recv(1000) + except ConnectionResetError as e: + log.warning( + f"recv ConnectionResetError, client {self.client_address} had close .....") + break + except: + log.warning(" sk.recv(1000) exception .....") + pass + + log.debug(len(data_byte)) + + # 客户端主动关闭连接后，会不断接收b'', 所以跳出循环，运行到程序结尾，自动关闭线程 + if data_byte == b'': + log.info( + " b'' is received , maybe client {self.client_address} had close. ") + self.sk.close() + if hasattr(self, "sensor"): + del self.sensor # 销毁对象 + break + else: + break + continue + else: + self.sensor.write_buf(data_byte) + + data_byte = b'' # 客户端掉线后以前数据不长居内存 + + try: + data_ = self.sensor.decode() + except MyException as e: + log.warning(e) + break + except Exception as e: + log.warning("decode data 出现异常 ") + log.warning(e) + break + + if data_ != {}: + id = data_['id'] + data_code = data_['type'] + log.info(f'Received From ID:{id} DATA CODE:{data_code}') + + # 保存当前数据 + self.dealData.save(data_) + head = data_['head'] + log.info(f'Head :{head}') + + # 返回head给服务器 + self.sk.send(data_['head']) + + # 判断是否是最后一帧, 修改目录为时间格式并 + if data_["packet_con"] == data_["packet_all"]: + log.info(f'最后一帧数据已经收到并保存') + # id 为传感器测量id ,con 测量序号 + self.dealData.deal(data_['id'], data_["con"], self.cfg, self.retrieve) + pass + + if time.time() - self.begin_time > TIMEOUT_SECOND: + log.info(f'Received data timeout') + break + + def finish(self) -> None: + # 什么时候执行 finish + # 执行handle 出现意外错误时执行 finish，或handle break时执行 finish + # 关闭连接 + if self.sk: + self.sk.close() + + # conn_pool 连接池销毁 + if self.client_address in conn_pool: + conn_pool.remove(self.client_address) + + log.info( + f"finish(): stop one socket from client {self.client_address} ") + + + +if __name__ == '__main__': + server_ = socketserver.ThreadingTCPServer(ADDRESS, MyServer) + log.info('listening...........') + + try: + server_.serve_forever() + except KeyboardInterrupt: + log.info(" Ctrl+C 退出主程序 ") + server_.server_close() + except Exception as e: + log.info(" 系统异常， 如下: \n ") + log.info(e) + + # 有闪退，可以用线程将server_.serve_forever 包起来 + # threading.Thread(target=server_.serve_forever).start() diff --git a/tcp_profiler.py b/tcp_profiler.py new file mode 100644 index 0000000..1919bfd --- /dev/null +++ b/tcp_profiler.py @@ -0,0 +1,234 @@ +# tcp Handheld 客户端 模拟 ,模拟数据打包 +# 直接平一个小包测试数据 直接写tou + +import socket +import time + +# IP = "39.96.0.224" +IP = "127.0.0.1" +PORT = 7887 +ADDRESS = ( IP, PORT ) # 绑定地址 +INFO_HEADER = "111355AA020000250000004F000107" +DATA_HEADER = "11 13 55 AA 02 00 10 25 00 00 00 DA 06 02 07" +END_FRAME = "11 13 55 AA 02 00 FF 25 00 00 00 00 00 00 00" + + +# 改数据 +class DataContent: + # 11 13 55 AA 0200 00 11 00 00 00 4F00 01 07 + # id type-num 序号 17 字节数 包号 总 + info_frame = "1703140E313518565D1E51D55605CCE812068CDDC347D0239515000000000000000000000064000000000000000000000000000000000000000000F450F450F4500600000300000000000000000000" + + sensor_frame =[ + # 11 13 55 AA 02 00 10 11 00 00 00 DA 06 02 07 + '131123600020000000524844557691E700CB054C054A05A10001\ + 23A000300700000A07FD062A073707440750074D074C07530759075A07610765076A07720790079107A407C3070709BE0AAC0ABF080F080808270849089E0880094F0BB70F2A1901\ + 23A000300600005223CD28242B3830E235813BE9401B46675A3181C68D5C7916715D74C676617658742E71B26C9C674363DA5F1C5E0160F164DF66A262F85AD153764E4F4C014D01\ + 23A00030050000EA4D8C4D314DF44D3C4FC250E453BA5E4486B8C5BCECC4D29A96436BFA56734E4B4A55473446364A845140582A5A9C55A54D91465440303C1B3FDE5783718E7001\ + 23A00030040000DC565046C140F63D21392E32D12B5B28A226F3253525662366219E20DD1FB11E971DC51C2F1C901B921A6419C21882185818E418181B401DEC1C1D1A3917F71501\ + 23A00030030000EE153316F516CA176E18B218CA18B118701845186A1730155913E51314153015E1146C1420147D13C11208128C1128113011BF121614DB12F70F3B0EE50D320E01\ + 23A00030020000FD0D3C0D930C470C0D0CB80B710B1E0BCD0A920A560A1F0A000A140A460A3E0AD0097A096C095F092109B60870085F0834082B081C0808080B08DD07B907950701\ + 23A000300100007007620755074A0740074C074607460748074A07500758075D0757074F0743075F0756075F0758074F073F07240745076207770756073F072A070907E606D60601\ + 23A00030000000F3060F07250722072E07200703070B07EC06FF0619073E07590734071807190719070A07E106DD06F006EC061B072F073E073E07220725070B07F806C806DE0601\ + 23A000300700000A07FD062A073707440750074D074C07530759075A07610765076A07720790079107A407C3070709BE0AAC0ABF080F080808270849089E0880094F0BB70F2A1901\ + 23A000300600005223CD28242B3830E235813BE9401B46675A3181C68D5C7916715D74C676617658742E71B26C9C674363DA5F1C5E0160F164DF66A262F85AD153764E4F4C014D01\ + 23A00030050000EA4D8C4D314DF44D3C4FC250E453BA5E4486B8C5BCECC4D29A96436BFA56734E4B4A55473446364A845140582A5A9C55A54D91465440303C1B3FDE5783718E7001\ + 23A00030040000DC565046C140F63D21392E32D12B5B28A226F3253525662366219E20DD1FB11E971DC51C2F1C901B921A6419C21882185818E418181B401DEC1C1D1A3917F71501\ + 23A00030030000EE153316F516CA176E18B218CA18B118701845186A1730155913E51314153015E1146C1420147D13C11208128C1128113011BF121614DB12F70F3B0EE50D320E01\ + 23A00030020000FD0D3C0D930C470C0D0CB80B710B1E0BCD0A920A560A1F0A000A140A460A3E0AD0097A096C095F092109B60870085F0834082B081C0808080B08DD07B907950701\ + 23A000300100007007620755074A0740074C074607460748074A07500758075D0757074F0743075F0756075F0758074F073F07240745076207770756073F072A070907E606D60601\ + 23A00030000000F3060F07250722072E07200703070B07EC06FF0619073E07590734071807190719070A07E106DD06F006EC061B072F073E073E07220725070B07F806C806DE0601\ + 23A000300700000A07FD062A073707440750074D074C07530759075A07610765076A07720790079107A407C3070709BE0AAC0ABF080F080808270849089E0880094F0BB70F2A1901\ + 23A000300600005223CD28242B3830E235813BE9401B46675A3181C68D5C7916715D74C676617658742E71B26C9C674363DA5F1C5E0160F164DF66A262F85AD153764E4F4C014D01\ + 23A00030050000EA4D8C4D314DF44D3C4FC250E453BA5E4486B8C5BCECC4D29A96436BFA56734E4B4A55473446364A845140582A5A9C55A54D91465440303C1B3FDE5783718E7001\ + 23A00030040000DC565046C140F63D21392E32D12B5B28A226F3253525662366219E20DD1FB11E971DC51C2F1C901B921A6419C21882185818E418181B401DEC1C1D1A3917F71501\ + 23A00030030000EE153316F516CA176E18B218CA18B118701845186A1730155913E51314153015E1146C1420147D13C11208128C1128113011BF121614DB12F70F3B0EE50D320E01\ + 23A00030020000FD0D3C0D930C470C0D0CB80B710B1E0BCD0A920A560A1F0A000A140A460A3E0AD0097A096C095F092109B60870085F0834082B081C0808080B08DD07B907950701\ + 23A000300100007007620755074A0740074C074607460748074A07500758075D0757074F0743075F0756075F0758074F073F07240745076207770756073F072A070907E606D60601\ + 23A00030000000F3060F07250722072E07200703070B07EC06FF0619073E07590734071807190719070A07E106DD06F006EC061B072F073E073E07220725070B07F806C806DE0601', + ] + + end_frame = "11 13 55 AA 02 00 FF 25 00 00 00 00 00 00 00" #0xFF +class MyBuf: + '''一次测量三个传感器数据打一个包''' + def __init__(self,) -> None: + self.__buf = b'' + self.__head = {} + self.__begin_sign = b'\x23' + self.__end_sign = b'\x0D' + self.send_head = b'\x11\x13\x55\xAA' + self.send_id = 2 # 2byte 地位在前 + self.send_type = 0 # info 0 data 1 pic 2 + self.send_num = 0 # 一次测量最多十六次平均 + self.send_con = 0 # 总的测量序号 4byte ，16次平均算几次？每次算一个 + self.send_size = 0 #字节数 2byte + self.send_packet_con = 0 + self.send_packet_all = 0 + + self.repeat = 0 # 依据这个拆包 + self.groups = 0 + + self.state = 0 + + def readFile2Buf(self, fpath) -> None: + with open(fpath,"rb") as f: + self.__buf = f.read() + pass + pass + + def read_buf(self, size: int) -> bytes: + if size > self.__buf.__len__(): + return b'' + ret = self.__buf[0:size] + self.__buf = self.__buf[size:] + return ret + + def write_buf(self, buf: bytes) -> None: + len = buf.__len__() + # logging.info(f'Received ID:{id} Size:{len}') + self.__buf = self.__buf+buf + + def get_buf_size(self) -> int: + return self.__buf.__len__() + + def back_bytes(self, buf: bytes) -> None: + self.__buf = buf+self.__buf + + def reset_head(self) -> None: + self.__head = {} + + def reset_buf(self) -> None: + self.__buf = b'' + + def pack_info_send(self,): + length = 26 + ret = self.read_buf(length) + id = self.send_id.to_bytes(2, byteorder='little') + self.send_type = 0 + type_num = (self.send_type *16 + self.send_num).to_bytes(1,byteorder='little' ) # ?? + con = self.send_con.to_bytes(4, byteorder='little') + size = length.to_bytes(2, byteorder='little') + packet_con = self.send_packet_con.to_bytes(1,byteorder='little' ) + packet_all = self.send_packet_all.to_bytes(1,byteorder='little' ) + ret = self.send_head + id + type_num + con + size + packet_con + packet_all +ret + self.send_packet_con += 1 + + # 取重复次数和测量间隔 + return ret + + def pack_handheld_send(self,): + length = 26+576+576+576 + ret = self.read_buf(length) + id = self.send_id.to_bytes(2, byteorder='little') + self.send_type = 1 + type_num = self.send_type << 4 + self.send_num # ?? + con = self.send_con.to_bytes(4, byteorder='little') + size = length.to_bytes(2, byteorder='little') + packet_con = self.send_packet_con.to_bytes(1,byteorder='little' ) + packet_all = self.send_packet_all.to_bytes(1,byteorder='little' ) + ret = self.send_head + id + type_num + con + size + packet_con + packet_all +ret + self.send_con+=1 # 总测量序号+1 + self.send_num += 1 # 是平均中的第几次 + self.send_packet_con += 1 + return ret + pass + + def pack_end_send(self,): + ret = b'\x11\x13\x55\xAA\x02\x00\xFF\x11\x00\x00\x00\x00\x00\x00\x00' + return ret + pass + +class TcpProfiler(object): + def __init__(self ): + super(TcpProfiler, self).__init__() + self.flag = True + self.delay_time = 15 + self.recv_times = 10 + self.head = b'' + self.socket = socket.socket( socket.AF_INET, socket.SOCK_STREAM ) + self.socket.setblocking(True) + + def connect(self,): + print(f"== connect to server : {ADDRESS}") + self.socket.connect( ADDRESS ) + + def disconnect(self,): + self.socket.close( ) + + def rcv(self,): + recv_data = self.socket.recv( 1024 ) + if recv_data == self.head: + self.flag =True + time.sleep( 0.01 ) + + def send_rcv(self,buf): + self.socket.send( buf ) + self.flag = False + self.head = buf[:15] + # 当前接收到数据就不阻塞了，后加数据不对继续阻塞 + try: + data = self.socket.recv(1024) + if data != b'': + print("not empty") + print(data) + if data==self.head: + print(f"recv.... {data}" ) + self.flag = True + self.head = b'' + time.sleep( 0.1 ) + except BlockingIOError as e: + print(f" 阻塞 中断 ....") + except KeyboardInterrupt as e: + print(f" == ctrl +c , exit") + + + def send_info(self,): + buf = bytes.fromhex (INFO_HEADER+ DataContent.info_frame) + self.flag = False + self.head = buf[:15] + print(f"head ....{self.head.hex()}") + self.send_rcv(buf) + + + def send_data(self,): + buf = b'' + buf = bytes.fromhex (DATA_HEADER+ DataContent.sensor_frame[0]) + self.flag = False + self.head = buf[:15] + print(f"head ....{self.head.hex()}") + self.send_rcv(buf) + # for i in range( len( DataContent.sensor_frame) ): + # print( "sensorframe..." ) + # buf = bytes.fromhex ( DataContent.sensor_frame[i] ) + # self.flag = False + # self.head = buf[:15] + # print(f"head ....{self.head.hex()} ") + # self.send_rcv(buf) + + def send_end_frame(self,): + buf = bytes.fromhex (DataContent.end_frame) + self.flag = False + self.head = buf[:15] + print(f"head ....{self.head.hex()}") + self.send_rcv(buf) + +if __name__ == '__main__': + # print( bytes.fromhex (DataContent.info_frame) ) + tp = TcpProfiler() + tp.connect() + + try : + while True: + tp.send_info() + + tp.send_data() + + tp.send_end_frame() + + time.sleep( 10) + except KeyboardInterrupt as e: + print(e) + + tp.disconnect( ) + + diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/myexception.py b/tools/myexception.py new file mode 100644 index 0000000..1b526c0 --- /dev/null +++ b/tools/myexception.py @@ -0,0 +1,11 @@ +class TCPIPException(Exception): + def __init__(self, message="TCPIP 自定义异常"): + self.message = "TCPIP exception : " + message + +class SerialException(Exception): + def __init__(self, message="Serial 自定义异常"): + self.message = "serial exception : " + message + +class MyException(Exception): + def __init__(self, message=" 自定义异常"): + self.message = "自定义异常: " + message \ No newline at end of file diff --git a/tools/mylogger.py b/tools/mylogger.py new file mode 100644 index 0000000..9c85ec9 --- /dev/null +++ b/tools/mylogger.py @@ -0,0 +1,285 @@ +# coding=utf-8 +''' +单例模式日志 -- 使用一次关闭 handler +这种方法优缺点： +缺点： 输出的format 需要自己定义 ，并过滤 + 过滤要看是否以什么开头，或包含什么 +优点： 不占用文件资源，占用系统资源小 +调用 log.info( ) log.error() +''' + +import logging +import logging.handlers +import os +import time +import threading +# from config import LOG_PATH,LOG_LEVEL,LOG_ENABLED,LOG_FORMAT, \ +# LOG_TO_CONSOLE,LOG_TO_FILE + +MY_LOGGER_NAME = "DefaultLogger" +LOG_ENABLED = True # 是否启用日志 +LOG_TO_CONSOLE = True # 是否启用控制台输出日志 +LOG_TO_FILE = False # 是否启用文件输出 +LOG_COLOR_ENABLE = True # 是否启用颜色日志 + +LOGGER_DIR = "logs" +LOGGER_PATH = os.path.join( os.path.dirname(__file__), LOGGER_DIR ) +LOGGER_FILENAME = os.path.join( LOGGER_PATH, 'logs.log' ) + +""" +logging.INFO , logging.DEBUG , logging.WARNING , logging.ERROR , +""" +LOG_LEVEL = logging.INFO # 日志等级DEBUG INFO WARNIG ERROR +# LOG_LEVEL = logging.DEBUG +# LOG_LEVEL = logging.WARNING + +""" +# LOG_FORMAT = " %(name)s - %(module)s - %(filename)s - %(lineno)d | %(levelname)s : %(message)s" +# LOG_FORMAT = "%(levelname)s - %(asctime)s - process: %(process)d - threadname: %(thread)s " \ +# "- %(filename)s - %(funcName)s - %(lineno)d - %(module)s " \ +# "- %(message)s " +# LOG_FORMAT = "%(asctime)s - %(thread)s " \ +# "- %(levelname)s ::: %(message)s " +# '[%(asctime)s] |%(thread)s| %(levelname)-6s: %(message)s' +# fm = '%(levelname)：%(levelno)s：%(name)s：%(funcName)s：%(asctime)：%(pathname)：%(filename)：%(module)：%(thread)：%(threadName)' +# 此处日志颜色，修改日志颜色是通过 Filter实现的 +""" +LOG_FORMAT = '%(levelname)s\t[%(asctime)s] %(package)s:%(classname)s:%(funcname)s \t>> %(message)s' + +""" +# 此处日志颜色，修改日志颜色是通过 Filter实现的 +""" +LOG_FORMAT_COLOR_DICT = { + 'ERROR' : "\033[31mERROR\033[0m", + 'INFO' : "\033[36mINFO\033[0m", + 'DEBUG' : "\033[1mDEBUG\033[0m", + 'WARN' : "\033[33mWARN\033[0m", + 'WARNING' : "\033[33mWARNING\033[0m", + 'CRITICAL': "\033[35mCRITICAL\033[0m", +} + +""" +# Filter 用法， 以package class function 过滤 __package__ __class__ +# log.error( f"{__package__}::{__class__.__name__}::{sys._getframe().f_code.co_name} >> ") +# log.error( f"PacakgeName::ClassName::FunctionName:: ") +# LOGGER_FILTER_PACKAGE=[] 为空, 则Filter不起作用 +# 不为空，则只显示定义的报名 +# LOGGER_FILTER_CLASS=[] 为空, 则Filter不起作用 +# 不为空，则只显示定义的类或 +# LOGGER_FILTER_FUNCNAME=[] 为空, 则Filter不起作用 +# 不为空，则只显示定义的函数 +""" +# LOGGER_FILTER_PACKAGE = [ "test_logger" ] # 包名，文件名去 .py?? +LOGGER_FILTER_PACKAGE = [ ] +LOGGER_FILTER_CLASS = [ ] # 类名，文件名去 .py?? +# LOGGER_FILTER_CLASS = [ "LogTest" ] +# LOGGER_FILTER_FUNCNAME = [ "test1","test" ] # 函数名 +LOGGER_FILTER_FUNCNAME = [ ] +LOGGER_FILTER_LEVELNAME = [ ] # INFO DEBUG WARNING + +class PackageFilter(logging.Filter): + def __init__(self, filter_word:list = []): + self.filter_word = filter_word + pass + def filter(self, record: logging.LogRecord) -> bool: + if self.filter_word is not None: + return record.package in self.filter_word + +class ClassFilter(logging.Filter): + def __init__(self, filter_word:list = []): + self.filter_word = filter_word + pass + def filter(self, record: logging.LogRecord) -> bool: + if self.filter_word is not None: + return record.classname in self.filter_word + + pass + +class FunctionFilter(logging.Filter): + def __init__(self, filter_word:list = []): + self.filter_word = filter_word + pass + def filter(self, record:logging.LogRecord) -> bool: + if self.filter_word is not None: + return record.funcname in self.filter_word + +class LevelNameFilter(logging.Filter): + def __init__(self, filter_word:list = []): + self.filter_word = filter_word + pass + def filter(self, record:logging.LogRecord) -> bool: + if self.filter_word is not None: + return record.levelname in self.filter_word + +class ColorFilter(logging.Filter): + def __init__(self,): + pass + def filter(self, record: logging.LogRecord) -> bool: + record.levelname = LOG_FORMAT_COLOR_DICT.get(record.levelname) + return True + +class Log(object): + _instance_lock = threading.Lock() + + def __new__(cls, *args, **kwargs): + if not hasattr(Log, "_instance"): + with Log._instance_lock: + if not hasattr(Log, "_instance"): + Log._instance = object.__new__(cls) + return Log._instance + + def __init__(self, loggername = "DefaultLog" ): + # 文件命名 os.path.join()： 将多个路径组合后返回 + self.logger_filepath = LOGGER_FILENAME + self.loggername = loggername + self.level = LOG_LEVEL + + # 日志输出格式 + fm = LOG_FORMAT + self.formatter = logging.Formatter( fm ) + + # 生成记录器对象 + self.logger = logging.getLogger( self.loggername ) + self.logger.setLevel(LOG_LEVEL) + + # 日志过滤 + self.__add_filter() + + def __console(self, level, message, extra={} ): + # 添加 handler + self.__add_handler() + + # 判断日志级别 + if level == logging.INFO: + self.logger.info( message, extra=extra) + elif level == logging.DEBUG: + self.logger.debug(message,extra=extra) + elif level == logging.WARNING: + self.logger.warning(message,extra=extra) + elif level == logging.ERROR: + self.logger.error(message,extra=extra) + + # removeHandler在记录日志之后移除句柄，避免日志输出重复问题 + self.__remove_handler() + # if LOG_TO_FILE and self.file_handler: + # self.logger.removeHandler(self.file_handler) + # # 关闭打开的文件 + # self.file_handler.close() + # if LOG_TO_CONSOLE and self.stream_handler: + # self.logger.removeHandler(self.stream_handler) + # # 关闭打开的文件 + # self.stream_handler.close() + pass + + # debug < info< warning< error< critical + # debug模式 + def debug(self, message, package="Unknown", classname="Unknown", funcname="Unknown" ): + self.__console(logging.DEBUG, message, extra={"package":package, "classname":classname, "funcname":funcname} ) + # self.__remove_handler() + # info模式 + def info(self, message, package="Unknown", classname="Unknown", funcname="Unknown" ): + self.__console(logging.INFO, message, extra={"package":package, "classname":classname, "funcname":funcname} ) + # self.__remove_handler() + # warning模式 + def warning(self, message, package="Unknown", classname="Unknown", funcname="Unknown"): + self.__console(logging.WARNING, message, extra={"package":package, "classname":classname, "funcname":funcname} ) + # self.__remove_handler() + + # error模式 + def error(self, message, package="Unknown", classname="Unknown", funcname="Unknown"): + self.__console(logging.ERROR, message, extra={"package":package, "classname":classname, "funcname":funcname} ) + # self.__remove_handler() + + def __add_filter(self ): + if len( LOGGER_FILTER_PACKAGE ) > 0 : + self.logger.addFilter( PackageFilter( filter_word=LOGGER_FILTER_PACKAGE ) ) + if len( LOGGER_FILTER_CLASS ) > 0 : + self.logger.addFilter( ClassFilter( filter_word=LOGGER_FILTER_CLASS ) ) + if len( LOGGER_FILTER_FUNCNAME ) > 0 : + self.logger.addFilter( FunctionFilter( filter_word=LOGGER_FILTER_FUNCNAME ) ) + if len(LOGGER_FILTER_LEVELNAME) > 0 : + self.logger.addFilter( LevelNameFilter( filter_word=LOGGER_FILTER_LEVELNAME ) ) + + def __add_handler(self ): + if LOG_ENABLED and LOG_TO_FILE: + # 考虑使用 RotatingFileHandler TimedRotatingFileHandler防止日志过大 + # RotatingFileHandler("test", "a", 4096, 2, "utf-8") + # TimedRotatingFileHandler(filename=LOG_PATH+"thread_", when="D", interval=1, backupCount=7) + self.file_handler = logging.handlers.TimedRotatingFileHandler(filename=self.logger_filepath, when='D', interval=1, backupCount=30, encoding='utf-8') + # self.file_handler = logging.FileHandler( self.logger_filepath, encoding='utf-8' ) + self.file_handler.setFormatter( self.formatter ) + # self.file_handler.setLevel( LOG_LEVEL ) + # if LOG_COLOR_ENABLE: # 文件日志无需加彩色 + # self.file_handler.addFilter( ColorFilter( ) ) + self.logger.addHandler(self.file_handler) + + if LOG_ENABLED and LOG_TO_CONSOLE: + # 创建一个StreamHandler,用于输出到控制台 + + self.stream_handler = logging.StreamHandler() + self.stream_handler.setFormatter(self.formatter) + # self.stream_handler.setLevel( LOG_LEVEL ) + if LOG_COLOR_ENABLE: + self.stream_handler.addFilter( ColorFilter( ) ) + self.logger.addHandler(self.stream_handler) + + def __remove_handler(self ): + if LOG_TO_FILE and self.file_handler: + self.logger.removeHandler(self.file_handler) + if len(self.logger.handlers)>0: + self.logger.handlers.pop() + # 关闭打开的文件 + self.file_handler.close() + if LOG_TO_CONSOLE and self.stream_handler: + self.logger.removeHandler(self.stream_handler) + if len(self.logger.handlers)>0: + self.logger.handlers.pop() + # 关闭控制台 + self.stream_handler.close() + + def __remove_handler2(self ): + if LOG_ENABLED and LOG_TO_CONSOLE: + self.logger.removeHandler(self.stream_handler) + self.logger.handlers.pop() + # 关闭控制台 + self.stream_handler.close() + if LOG_ENABLED and LOG_TO_FILE: + self.logger.removeHandler(self.file_handler) + self.logger.handlers.pop() + # 关闭打开的文件 + self.file_handler.close() + + +log = Log( loggername = "DefaultLog") + +""" +filename: 指定日志文件名 +filemode: 和file函数意义相同，指定日志文件的打开模式，’w’或’a’ +format: 指定输出的格式和内容，format可以输出很多有用信息。显示的条目可以是以下内容： +%(levelname)： 日志级别的名字格式 +%(levelno)s： 日志级别的数字表示 +%(name)s： 日志名字 loggername +%(funcName)s： 函数名字 +%(asctime)： 日志时间，可以使用datefmt去定义时间格式，如上图。 +%(pathname)： 脚本的绝对路径 +%(filename)： 脚本的名字 +%(module)： 模块的名字 +%(thread): thread id +%(threadName)： 线程的名字 +""" + +""" +文件名行号 函数名, 要在调用的时候想办法了 +# 绝对路径 +print( __file__ ) +print( sys.argv[0] ) + +# 文件名 +print( os.path.basename(__file__) ) +print( os.path.basename(sys.argv[0]) ) + +self.__class__.__name__ +self.__class__.__name__, get_current_function_name() + +logger名 __name__ +""" \ No newline at end of file diff --git a/tools/mypath.py b/tools/mypath.py new file mode 100644 index 0000000..b8f1064 --- /dev/null +++ b/tools/mypath.py @@ -0,0 +1,265 @@ +from pathlib import PurePath, Path + +# from myconfig import NEWLINE,TOKEN,SEPARATOR +""" +""" +class MyDir(object): + """ + 操作方法:设置base tuple_dir header + 设置的是路径, 文件名要 ifNotNewFile 传入 + """ + + def __init__(self) -> None: + self.base_dir = Path() + self.dir_tuple = () + self.header = [] + self.header_str = "" + self.content = [] + self.content_str = "" + self.current_dir = None + self.current_filepath = None + pass + + def getDir(self,): + return self.current_dir + pass + + def setBaseDir(self, dir: Path): + self.base_dir = dir + self.current_dir = self.base_dir + pass + + def setDir(self, t:tuple=()): + self.dir_tuple = t + if len(self.dir_tuple) == 0 : + self.current_dir = self.base_dir + else: + self.current_dir = self.base_dir.joinpath( *t ) + pass + + def getDirFromBaseAndTuple(self, base_dir:Path, dir_tuple: tuple): + '''外部调用返回路径''' + ret_path = base_dir + t = dir_tuple + if len(t) == 0 : + ret_path = ret_path + else: + ret_path = ret_path.joinpath( *t ) + return ret_path + pass + + def setHeader(self, headerlist:list, headerSeperator: str = ";", headerinfo: str = None): + header_str = "" + if len(headerlist) == 0: + return + if headerinfo != None: + header_str = headerinfo + headerSeperator + for hl in headerlist: + header_str = header_str + str(hl) + headerSeperator + self.header_str = header_str[:-1] + pass + + def setContent(self, contentlist: list, contentSeperator: str = ";", contentinfo: str = None): + content_str = "" + if len(contentlist) == 0: + return + if contentinfo != None: + content_str = contentinfo + contentSeperator + tmp_str = "" + for cl in contentlist: + tmp_str = tmp_str + str(cl) + contentSeperator + self.content_str = content_str + tmp_str[:-1] + pass + + def newDirIfNot(self,) -> None: + # self.current_path = self.base_path.joinpath(self.path_tuple) + self.current_dir.mkdir(parents=True, exist_ok=True) + pass + + def newFileIfNot(self, fname: str) -> None: + self.newDirIfNot() + self.current_filepath = self.current_dir.joinpath(fname) + if not self.current_filepath.exists(): + with open(self.current_filepath, 'a') as f: + pass + return + pass + + def getCurrentFileSize(self,): + return self.current_filepath.stat().st_size + + def getFirstline(self,): + first_line = "" + with open(self.current_filepath, 'r') as f: # 打开文件 + first_line = f.readline() # 取第一行 + return first_line.strip('\n').strip('\r') + + def checkHeader(self,) -> int: + ''' + 返回: + 0 : 文件为空,可以直接写header + 1 : header对应上 无需处理 + -1: 需要提醒用户保存数据后,删除文件后再处理 + ''' + if self.getCurrentFileSize() == 0: + return 0 + first_line = self.getFirstline() + # print(f"firstline: {first_line}" ) + # print(f"header_str: {self.header_str}" ) + if first_line == self.header_str: + return 1 + return -1 + pass + + def writeHeader(self,) -> None: + with open(self.current_filepath, "a") as f: + f.write(self.header_str) + return None + pass + + def writeContent(self,new_line="\n") -> None: + with open(self.current_filepath, "a") as f: + f.write(new_line+self.content_str) + return None + pass + + def is_dir_empty(self, ): + '''文件夹是否为空''' + has_next = next(self.current_dir.iterdir(), None) + if has_next is None: + return True + return False + + def is_file_empty(self,): + '''文件是否为空''' + if self.current_dir.stat().st_size ==0: + return True + return False + + def deleteDir(self,): + '''文件夹是否为空''' + try: + if self.current_dir.exists(): + self.current_dir.rmdir() + except OSError as e: + raise Exception(e) + return True + + ## 其他需求 + def get_child_dir(self,) -> list: + ret = [] + tmp_dir = self.current_dir.glob("**/") + for td in tmp_dir: + if td.is_dir(): + ret.append(td.relative_to(self.current_dir)) + return ret + pass + + def get_child_dir_only(self,) -> list: + ret = [] + for d in self.current_dir.iterdir(): + if d.is_dir(): + ret.append(d.relative_to(self.current_dir)) + return ret + pass + + def get_files_from_currentdir(self, fmt:str="*/*" ) -> list: + '''fmt: * */* */*/*''' + ret = [] + tmp_dir = self.current_dir.glob(fmt) + print(tmp_dir) + for td in tmp_dir: + if td.is_file(): + ret.append(td) + return ret + pass + + def sort_dir_and_check( self, dirs:list ): + '''相对目录排序,目录最后一级''' + ret = [] + if len(dirs) == 0: + return ret + tmp = {} + tmp_lst = [] + for d in dirs: + last:str = d.parts[-1] + if last.isdigit() : + tmp.update( {last:d} ) + tmp_lst.append(int(last)) + pass + + tmp_lst.sort() + for t in tmp: + ret.append(tmp.get(str(t))) + pass + return ret + + + def sort_filepath_and_check(self, path_files:list): + '''相对目录排序,目录最后一级''' + ret = [] + if len(path_files) == 0: + return ret + tmp = {} + tmp_lst = [] + for d in path_files: + last:str = d.stem + if last.isdigit() : + tmp.update( {last:d} ) + tmp_lst.append(int(last)) + pass + + tmp_lst.sort() + for t in tmp: + ret.append(tmp.get(str(t))) + pass + return ret + + def group_and_sort_filepath(self,path_files:list): + ret = {} + # dirs_lst = [] + # len_files = len(path_files) + # if len_files == 0: + # return False + # for pf in path_files: + # pf_dir:str = pf.parts[-2] + # if pf_dir.isdigit() and int(pf_dir) not in dirs_lst: + # dirs_lst.append( int(pf_dir) ) + # dirs_lst.sort() + + + + + def check_dirs(self, dirs:list, begin:int =0, step:int=1): + '''检查目录是否从begin开始递增''' + len_dirs = len(dirs) + if len_dirs == 0: + return False + for i in range(begin,len_dirs,step) : + if dirs[i].parts[-1] != str(i) : + return False + return True + + def check_path_files(self,path_files:list,begin:int =0, step:int=1): + '''检查文件名从begin开始递增''' + len_files = len(path_files) + if len_files == 0: + return False + for i in range(begin,len_files,step) : + if path_files[i].stem != str(i) : + return False + return True + +if __name__ == "__main__": + mp = MyDir() + mp.setBaseDir(Path()) + print(mp.current_dir) + # t = ("test_dir","1","11") + t = ("test_dir", ) + mp.setDir( t ) + print(mp.current_dir) + + cd = mp.get_child_dir_only() + c = mp.sort_dir_and_check(cd) + print(cd ) + print(c ) diff --git a/tools/mytime.py b/tools/mytime.py new file mode 100644 index 0000000..3481cf8 --- /dev/null +++ b/tools/mytime.py @@ -0,0 +1,66 @@ +from datetime import datetime, timedelta +# import locale +# locale.setlocale(locale.LC_ALL, '') + +TIME_STR = "2022-06-10 16:16:16" +STD_TIME_STR_FMT = "%Y-%m-%d %H:%M:%S" # 小写y是两位年份 +CUR_TIME_STR_FMT = "%Y-%m-%d %H:%M:%S" + + +class MyTime(object): + """ + 操作方法:设置base tuple_path header + """ + + def __init__(self) -> None: + self.cur_time_str_fmt = "%Y-%m-%d %H:%M:%S" + self.std_time_str_fmt = "%Y-%m-%d %H:%M:%S" + self.cur_time_str = "" + self.std_time_str = "" + self.cur_datetime = "" + pass + + def setCurrentTimeStrFmt(self, s): + self.cur_time_str_fmt = s + pass + + def setStdTimeStrFmt(self, s): + self.std_time_str_fmt = s + pass + + def setCurrentTimeStr(self, s): + self.cur_time_str = s + self.cur_datetime = datetime.strptime(self.cur_time_str, self.cur_time_str_fmt) + pass + + def setStdTimeStr(self, s): + self.std_time_str = s + self.cur_datetime = datetime.strptime( + self.std_time_str, STD_TIME_STR_FMT) + pass + + def Current2STD(self): + # self.cur_datetime = datetime.strptime(self.cur_time_str , CUR_TIME_STR_FMT) + self.std_time_str = datetime.strftime( + self.cur_datetime, STD_TIME_STR_FMT) + pass + + def STD2Current(self, format: str): + # self.cur_datetime = datetime.strptime(self.std_time_str , STD_TIME_STR_FMT) + self.cur_time_str = datetime.strftime( + self.cur_datetime, CUR_TIME_STR_FMT) + pass + + def timeDelta(self, days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0): + self.cur_datetime = self.cur_datetime + \ + timedelta(days=0, seconds=0, microseconds=0, + milliseconds=0, minutes=0, hours=0, weeks=0) + pass + + +if __name__ == "__main__": + s = "2023-02-07 14:02:46" + mt = MyTime() + mt.setCurrentTimeStr(s) + print(mt.cur_datetime) + pass