Commit 97831c7c authored by LIBES Maurice's avatar LIBES Maurice 🏋🏾
Browse files

modifie : csv2NetCDF.py, modification des lbelles de colonnes traitement dans...

modifie : csv2NetCDF.py, modification des lbelles de colonnes traitement dans espace et avec majuscule au debut
parent 93a8660a
This diff is collapsed.
......@@ -27,6 +27,7 @@
import os
import sys
import glob
import re
try:
import netCDF4 as nc
import pandas as pd
......@@ -45,20 +46,20 @@ from datetime import datetime
def Convert_Datenum():
dateHeure = []
for dateiso in csvdata["DateTime"]:
#print("dateiso = ",dateiso)
try:
date,heure =dateiso.split('T')
yyyy,mm,jj=date.split('-')
hh, mn, ss = heure.split(':')
dateobs = datetime(int(yyyy), int(mm), int(jj), int(hh), int(mn), int(ss))
datenum = nc.date2num(dateobs, units=NC_TIME_FMT)
dateHeure.append(datenum)
except:
cprint("La date attendue n'est pas au format ISO yyymmddThh:mm:ss "+dateiso, color='red', attrs=['bold'])
sys.exit(0)
#print("tableau dateHeure ",dateHeure)
for dateiso in csvdata[libelle_Datetime]:
#print("dateiso = ",dateiso) # on cherche ce format 2018-07-07T10:08:59Z
pattern=re.compile('(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})Z?')
match=pattern.search(dateiso)
chainedate=match.groups()
annee,mois,jour,heure,minut,sec=chainedate
#print("annee,mois,jour,heure,minut,sec ", int(annee), int(mois), int(jour), int(heure), int(minut), int(sec))
dateobs = datetime(int(annee), int(mois), int(jour), int(heure), int(minut), int(sec))
datenum = nc.date2num(dateobs, units=NC_TIME_FMT)
dateHeure.append(datenum)
#print("tableau dateHeure ",dateHeure)
return dateHeure
......@@ -149,7 +150,7 @@ def Create_Dimensions(dimension):
ncFile.createDimension(dimension, None)
ncFile.createDimension('lenstation', len(stationname))
cprint('Dimension du fichier NetCDF : '+dimension, color='green', attrs=['bold'])
cprint('Dimension du fichier NetCDF : colonne '+dimension, color='green', attrs=['bold'])
return 0
......@@ -167,13 +168,13 @@ def Create_Write_Variables(dim, csvfile):
# convertit les dates ISO 2011-01-16T09:38:00 en format numérique pour NetCDF
dateHeure=Convert_Datenum()
#print(dateHeure)
#print("sortie dateHeure ", dateHeure)
i = 0 # index de colonnes
for col in cols:
print("Traitement col ",'-',col,'-')
cprint("\t Traitement col -"+col+'-', 'yellow', attrs=['bold'])
if (col == "DateTime"):
if (col == libelle_Datetime):
frmt[i]='i4' #on force la date en Int car on l'a convertie en Int avec la fonction Convert_Datenum()
units[i] = NC_TIME_FMT
......@@ -209,7 +210,7 @@ def Create_Write_Variables(dim, csvfile):
string_values = np.array(csvdata[col].values,dtype=frmt[i])
## ecriture des colonnes de valeurs CSV dans les tableaux tabcol[] des variables de Netcdf
if (col=='DateTime'):
if (col==libelle_Datetime):
tabcol[:] = dateHeure# cas particulier ecriture des valeurs de temps dans la variable NetCDF
tabcol.origin = NC_TIME_ORIGIN
else:
......@@ -285,13 +286,19 @@ def Control_Illegal_char(cols):
return 0
##
def Control_Header_Columns():
def Control_Header_Columns(cols):
#Controle si les columns date|latitude|longitude existe bien dans le fichier csv
colonnes_obligatoires=['DateTime','Latitude','Longitude']
#print("COLS ",cols)
for i in range(0, len(cols)): #on enleve les espace en debut et fin de chaine
cols[i]=cols[i].lstrip()
cols[i]=cols[i].rstrip()
cols[i]=cols[i].capitalize()
#print("cols",i,"."+cols[i]+".")
for col in colonnes_obligatoires:
if (col not in cols):
cprint(cols,color="yellow",attrs=['bold'])
cprint("Control failed!\nIl manque une des colonnes DateTime|Latitude|Longitude dans le fichier :"+fic, color='red', attrs=['bold'])
cprint("Control failed!\nIl manque une des colonnes "+str(colonnes_obligatoires)+" dans le fichier :"+fic, color='red', attrs=['bold'])
sys.exit(0)
cprint("Control Columns OK", color='green', attrs=['bold'])
......@@ -299,8 +306,7 @@ def Control_Header_Columns():
##
def Control_Attributes_Var(indexdrop):
print("index du fichier ",indexdrop)
attributs_obligatoires=['unites','dimension','standard_name','long_name']
#print("index du fichier ",indexdrop)
for att in attributs_obligatoires:
if att not in indexdrop:
......@@ -341,6 +347,7 @@ if __name__ == "__main__":
global glob_att,featuretype
global ncFile,fic
global cols,indexlist,indexdrop
global libelle_Datetime, colonnes_obligatoires, attributs_obligatoires
NC_TIME_FMT = 'seconds since 1970-01-01 00:00:00 UTC'
......@@ -351,7 +358,10 @@ if __name__ == "__main__":
InputPath="./CsvFiles/"
OutputPath="./NcFiles/"
fileMetaData = "global_attributes.csv"
libelle_Datetime="Datetime"
colonnes_obligatoires=['Datetime','Latitude','Longitude']
attributs_obligatoires=['unites','dimension','standard_name','long_name']
parser = argparse.ArgumentParser(description='-s delimiter of the csv file ')
parser.add_argument('-s', '--separator', help='Please enter a separator for you csvFile', required=True)
parser.add_argument('-n', '--name', help='Please enter the name of the station', required=False)
......@@ -390,7 +400,7 @@ if __name__ == "__main__":
## on extrait le nom des colonnes du fichier CSV
cols = csvdata.columns.values
print('cols',cols)
#print('cols',cols); print(type(cols))
if (nbfic > 1): ## verifie que les entetes des fichiers CSV dans le repertoire CsvFiles soient les memes
Compare_Files(cols,firstcols)
......@@ -398,7 +408,7 @@ if __name__ == "__main__":
firstcols = cols
Control_Attributes_Var(indexdrop)
Control_Header_Columns()
Control_Header_Columns(cols)
Control_Illegal_char(cols)
glob_att = pd.read_csv(fileMetaData, delimiter='=', header=None, index_col=0)
......
title= titre du projet (NetCDF files)
Institution=OSU Pytheas, UMS3470 CNRS
source= methodology The method of production of the original data. If it was model-generated, source should name the model and its version, as specifically as could be useful. If it is observational, source should characterize it (e.g., "surface observation" or "radiosonde").
history= history du projet, Provides an audit trail for modifications to the original data. Well-behaved generic netCDF filters will automatically append their name and the parameters with which they were invoked to the global history attribute of an input netCDF file. We recommend that each line begin with a timestamp indicating the date and time of day that the program was executed.
title= Maupiti Hoe 2018 (NetCDF files)
project= High-resolution experimental survey of the Maupiti Island reef-lagoon system
Institution=MIO UMR 7294 CNRS
source= The experimental strategy was to deploy a series of bottom-moored pressure sensors across the reef barrier to study the cross-reef evolution of wave energy and water level and a velocity profiler in the lagoon to quantify the cross-reef transport. The recovered data allowed to estimate the main terms of the wave-averaged depth-averaged momentum balance.
references=Published or web-based references that describe the data or methods used to produce it.
comment= commentaire Miscellaneous information about the data or methods used to produce it.
rfa=If you use these data in publications or presentation, please acknowledge the xxxxxxxxxxxxxxxxx of OSU Pytheas. Also, we would appreciate receiving a preprint and/or reprint of publications using the data for inclusion in our bibliography. Relevant publications should be sent to: xxxxxxxxx Rxxxxxx xxxx Project MIO UMR 7294 CNRS, Campus de Luminy 13288 Marseille cedex9
rfa=If you use these data in publications or presentation, please acknowledge the xxxxxxxxxxxxxxxxx of OSU Pytheas. Also, we would appreciate receiving a preprint and/or reprint of publications using the data for inclusion in our bibliography. Relevant publications should be sent to: damien.sous@mio.osupytheas.fr MIO UMR 7294 CNRS, Campus de Luminy 13288 Marseille cedex9
license=The data may be used and redistributed for free but is not intended for legal use. These are raw and uncorrected data and may contain inaccuracies. Neither the data Contributor, XXX, OSU Pytheas, CNRS, nor the French Government, nor any of their employees or contractors, makes any warranty, express or implied, including warranties of merchantability and fitness for a particular purpose, or assumes any legal liability for the accuracy, completeness, or usefulness, of this information.
summary= resume du projet
project= nom du projet
keywords=OSU Pytheas, CNRS, Observatory, France, Sea Regions, Mediterranean Sea, environmental data
summary= The scientific objectives of the project MAUPITI HOE are to understand the hydrodynamics of an archetypal reef-lagoon system of a high volcanic reef island. The physical functioning of the hydrosystem involves a fine coupling between water levels, waves (including wind, infragravity and VLF waves), currents and seabed structure (reef roughness). The present data focuses on the reef barrier dynamics.
keywords=OSU Pytheas, CNRS, Observatory, France, Sea Regions, Mediterranean Sea, environmental data, environment, geophysical environment, data acquisition, spatial distribution, monitoring data, geo-referenced data, environmental analysis, environmental research, environmental data, experiment, field experiment, research project, scientific research, hydrosphere, sea, geomorphic process , erosion
DOI=https://doi.org/10.34930/9db3bec4-0bbf-4531-8864-f100c4b8eced
featuretype=timeseries
cdm_data_type=timeseries
cdm_timeseries_variables=depth,latitude,longitude
cdm_timeseries_variables=datetime,latitude,longitude
Conventions=CF-1.6, ACDD-1.3
comment= -
history= -
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment