Commit 2b1689fd authored by Ayar Nour's avatar Ayar Nour
Browse files

Upload New File

parent cca52eac
#!/usr/bin/python3
import netCDF4 as nc
import pandas as pd
import datetime
from datetime import date, time, datetime, timedelta
import numpy as np
import argparse
import os
import shutil
def Convert_Datenum():
dateHeure = []
for dateiso in csvdata["Date"]:
#print("dateiso = ",dateiso)
date,heure =dateiso.split('T')
yyyy,mm,jj=date.split('-')
hh, mn, ss = heure.split(':')
dateobs = datetime(int(yyyy), int(mm), int(jj), int(hh), int(mn), int(ss))
datenum = nc.date2num(dateobs, units=NC_TIME_FMT)
dateHeure.append(datenum)
#print("tableau dateHeure ",dateHeure)
return dateHeure
# ========== NETCDF4 file ============
def Generate_NetCDF(csvfile):
# Generating empty NETCDF file
netCDFFileName = csvfile.replace('.csv', '.nc')
print("Generating NetCDF File : ", netCDFFileName)
ncFile = nc.Dataset(netCDFFileName, 'w', format='NETCDF4')
return ncFile
##
def Get_Formats(ligne):
liste_formats = [] #on recoit un tableau numpy a ce niveau, les types numpy sont np.float, np.int
for value in ligne:
if isinstance(value,str):
length=len(value)
frmt = 'S'+str(length)
liste_formats.append(frmt)
#print("value = ",value,type(value))
if isinstance(value,np.floating): # np.floating renvoie True si on est en float64 ou float32
frmt = 'f4'
liste_formats.append(frmt)
#print("value = ",value,type(value))
if isinstance(value,np.int64):
frmt = 'i4'
liste_formats.append(frmt)
#print("value = ",value,type(value))
#print('sortie liste_formats ', liste_formats)
return liste_formats
##
def Create_dimensions(dimension):
# defining the dimensions :
ncFile.createDimension(dimension, None)
ncFile.createDimension('station_name', len(station_name))
print('Dimension OK')
return 0
##
def Create_Write_Variables(dim, csvfile):
global csvdata
# recuperation du nom des colonnes
csvdata = pd.read_csv(InputPath+csvfile, sep=',', skiprows=[1, 2, 3]) # rendre parametrable cette entete
cols = csvdata.columns.values #recupere le nom des colonnes
units = datacsv.iloc[0]
standard_name = datacsv.iloc[1]
long_name = datacsv.iloc[2]
ligne=csvdata.iloc[0] # on prend une ligne de données sous forme de série pandas
l=ligne.tolist() # on la converti en tableau numpy avec la fonction tolist()
frmt=Get_Formats(l) #avec la ligne de données on va déterminer le type des varaibles (int, float, str...)
#print("sortie frmt ", frmt)
dateHeure=Convert_Datenum() # convertit les dates ISO 2011-01-16T09:38:00 en format numérique pour NetCDF
#print(dateHeure)
i = 0
for col in cols:
if (col == "Date"):
frmt[i]='i4' #on force la date en Int car on l'a convertie en Int avec la fonction Convert_Datenum()
units[i] = NC_TIME_FMT
## creation des variables NetCDf avec le nom des colonnes du fichier CSV
tabcol = ncFile.createVariable(col, frmt[i], dim) #on cree la variable NetCDF et on l'associe a un tableau tabcol
## rajout de certains attributs en fonction des variables
tabcol.units = units[i]
tabcol.long_name = long_name[i]
tabcol.standard_name = standard_name[i]
if (col=="Latitude"):
tabcol.axis="Y"
if (col=="Longitude"):
tabcol.axis="X"
if (col=="Depth" or col=="Profondeur"):
tabcol.axis="Z"
## ecriture des valeurs dans les tableaux
x = pd.Series(dateHeure)
if (col=='Date'):
tabcol[:] = dateHeure# cas particulier ecriture des valeurs de temps dans la variable NetCDF
tabcol.origin = NC_TIME_ORIGIN
else:
tabcol[:] = csvdata[col].tolist() # on ecrit les valeurs des colonnes du fichier CSV dans le tableau de la variable NetCDF
i += 1
print('Variables OK')
return 1
##
##
def Create_Global_Attributes(ncFile):
# with ncFile:
# metadata du fichier NC lues dans un fichier CSV externe
fileMetaData = "global_attributes.csv"
reader = pd.read_csv(fileMetaData, delimiter='=', header=None, index_col=None)
ncFile.rfa = reader.iloc[0, 1]
ncFile.license = reader.iloc[1, 1]
ncFile.summary = reader.iloc[2, 1]
ncFile.lineage = reader.iloc[3, 1]
ncFile.project = reader.iloc[4, 1]
ncFile.description = reader.iloc[5, 1]
ncFile.title = reader.iloc[6, 1]
ncFile.keywords = reader.iloc[7, 1]
ncFile.comments = reader.iloc[8, 1]
ncFile.history = reader.iloc[9, 1]
ncFile.featuretype = reader.iloc[10, 1]
ncFile.cdm_data_type = reader.iloc[11, 1]
ncFile.cdm_profile_variables = reader.iloc[12, 1]
ncFile.conventions = reader.iloc[13, 1]
print('Global Attributs OK')
return 0
###
### Programme Principal
###
if __name__ == "__main__":
global NC_TIME_FMT, NC_TIME_ORIGIN
global datacsv
global station_name, OutputPath
station_name='frioul'
var_dimension='Depth'
NC_TIME_FMT = 'seconds since 1970-01-01 00:00:00 UTC'
NC_TIME_ORIGIN = "01-JAN-1970 00:00:00"
# ========== Read CSV file =============
'''
csvfile = input("Please type the csv file's name (with extension .csv) " )
try:
test = open(csvfile)
test .close()
print("The file exists in your current directory")
except IOError:
print("The file that you are trying to reach is unavailable in your current directory")
'''
InputPath="./CsvFiles/"
OutputPath="./NcFiles/"
allfiles = os.listdir(InputPath)
for fic in allfiles:
if fic.endswith(".csv"):
print("traitement fichier ", fic)
parser = argparse.ArgumentParser(description='-s delimiter of the csv file ')
parser.add_argument('-s', '--separator', help='Please enter a separator for you csvFile', required=True)
args = parser.parse_args()
print("args ",args)
print("args.separator ",args.separator)
datacsv = pd.read_csv(InputPath+fic, sep=args.separator)
ncFile = Generate_NetCDF(OutputPath+fic)
Create_dimensions(var_dimension)
Create_Global_Attributes(ncFile)
Create_Write_Variables(var_dimension, fic)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment