commit dbfb4ada50aac8afacd6cfd2a5f466b538b4df91 Author: tangchao0503 <735056338@qq.com> Date: Thu Sep 8 17:50:24 2022 +0800 第一次提交：完成数据读入 + 拼接 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2fe3d34 --- /dev/null +++ b/.gitignore @@ -0,0 +1,167 @@ +# 唐超添加 +/.idea +*.rar +*.docx +/2022neimengdata + + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ \ No newline at end of file diff --git a/merge_data.py b/merge_data.py new file mode 100644 index 0000000..0c2d693 --- /dev/null +++ b/merge_data.py @@ -0,0 +1,94 @@ +import numpy as np +import pandas as pd +import os +from sklearn import datasets, linear_model +import argparse + + +class GasAnalyzer(): + def __init__(self, folderPath): + self.folderPath = folderPath + + def read_data(self): + path_list = os.listdir(self.folderPath) + self.validFiles = [] + for filename in path_list: + if os.path.splitext(filename)[1] == '.txt': + self.validFiles.append(filename) + + # 这两个循环可以合并优化，但不确定是否可以提高性能 + names = locals() + self.rowCountsBackup = [] + for i, filename in enumerate(self.validFiles): + names[f'df_{i}'] = pd.read_csv(self.folderPath + "\\" + filename, skiprows=1, sep=',') + self.rowCountsBackup.append(names[f'df_{i}'].shape[0]) + + for x in range(len(self.rowCountsBackup)): + if(x==0): + self.df_total = names[f'df_{0}'] + else: + self.df_total = pd.concat([self.df_total, names[f'df_{x}']]) + + if(sum(self.rowCountsBackup) != self.df_total.shape[0]): + print("拼接气体分析仪数据失败！拼接前后行数不一致！") + return 1 + + return 0 + + def write_data(self): + a = 1 + + def merge_data(self, MeteorologicalStation): + a = 1 + + +class MeteorologicalStation(): + def __init__(self, folderPath): + self.folderPath = folderPath + + def read_data(self): + path_list = os.listdir(self.folderPath) + self.validFiles = [] + for filename in path_list: + if os.path.splitext(filename)[1] == '.dat': + self.validFiles.append(filename) + + # 这两个循环可以合并优化，但不确定是否可以提高性能 + names = locals() + self.rowCountsBackup = [] + for i, filename in enumerate(self.validFiles): + names[f'df_{i}'] = pd.read_csv(self.folderPath + "\\" + filename, header=None, sep=',') + self.rowCountsBackup.append(names[f'df_{i}'].shape[0]) + + for x in range(len(self.rowCountsBackup)): + if(x==0): + self.df_total = names[f'df_{0}'] + else: + self.df_total = pd.concat([self.df_total, names[f'df_{x}']]) + + if(sum(self.rowCountsBackup) != self.df_total.shape[0]): + print("拼接气体分析仪数据失败！拼接前后行数不一致！") + return 1 + + return 0 + + +if __name__ == "__main__": + # parser = argparse.ArgumentParser() + # parser.add_argument("csv_path", help="Path of csv file which contains wavelength.") + # parser.add_argument("start_row", help="Start row of coning 410 sensor.") + # args = parser.parse_args() + + # row_bin1, wave_bin1, row_bin2, wave_bin2 = read_data(args.csv_path, int(args.start_row)) + + GasAnalyzer_folderPath = r"D:\PycharmProjects\weatherInstrument\2022neimengdata\气体分析仪\2022-08-13" + + MeteorologicalStation_folderPath = r"D:\PycharmProjects\weatherInstrument\2022neimengdata\气象站\2022_08_13" + + tmp1 = GasAnalyzer(GasAnalyzer_folderPath) + tmp1.read_data() + + tmp2 = MeteorologicalStation(MeteorologicalStation_folderPath) + tmp2.read_data() + + print("completed!!")