Commit d0c2de90 authored by MALLARINO Didier's avatar MALLARINO Didier
Browse files

Initial commit

parents
Pipeline #65 failed with stages
rfa|If you use these data in publications or presentation, please acknowledge the HTMNET Project Office of MIO/OSU Pytheas. Also, we would appreciate receiving a preprint and/or reprint of publications utilizing the data for inclusion in our bibliography. Relevant publications should be sent to: HTMNET Project Office, Laboratoire MIO, Université de Toulon Batiment X CS 60584, 83041 TOULON CEDEX 9
license|The data may be used and redistributed for free but is not intended for legal use, since it may contain inaccuracies. Neither the data Contributor, MIO, OSU Pytheas, CNRS, nor the French Government, nor any of their employees or contractors, makes any warranty, express or implied, including warranties of merchantability and fitness for a particular purpose, or assumes any legal liability for the accuracy, completeness, or usefulness, of this information.
summary|This project consists in providing original measurement data for a better understanding and modelling of interactions and couplings between coastal dynamics and coastal dynamics at bay scale (currently the area of the Toulon conurbation with a project to extend it on a larger scale along the Mediterranean coast), by developing a long-term observation system (several years)
lineage|This dataset has data from the HTMNET Project. This dataset is a product of the HTMNET Project office at MIO/OSU Pytheas institute. This dataset is partially refreshed everyday by downloading and merging the latest version of the last days worth of data.
project|HTMNET
description|HTMNET project Data
title|HTMNET Data
keywords|meteorology, mio, OSU Pytheas, CNRS, station, time, HTMNET, Earth Science > Atmosphere > Atmospheric Temperature > Surface Temperature > Air Temperature, Earth Science > Atmosphere > Atmospheric Pressure > Sea Level Pressure, Earth Science > Oceans > Ocean Pressure > Water Pressure, Earth Science > Oceans > Ocean Temperature > Water Temerature
This diff is collapsed.
#!/usr/bin/python3
####### # -*- coding: utf-8 -*-
# ============= RCS Header =====================
#$Header: $
#$Rev$ Revision of last commit
#$Author: $ Author of last commit
#$Date: $ Date of last commit
#$Revision: $ Revision number
#$RCSfile: $ Name of the RCS file in the repository
# ============ End of RCS Header # =====================
# Exemple de fabrication d'un NetCDF
# Bibliotheques Python
import sys
from datetime import datetime
import os
import getopt
import numpy as np # Permet d'appliquer des fonctions mathematiques usuelles a des listes
# Support NetCDF4
import netCDF4 as nc4
# Reading CSV Files
import csv
import math
# ----------------------------------------------------------------------------------------------
# --------------------- Conversion des dates / From ML -------------------
# ----------------------------------------------------------------------------------------------
#
# # transforme une date de mesure en objet date et la convertir en format numerique
# # il faut que les dates soient au format objet datetime
#
def convert_datenum(date,heure):
# print(date,heure)
yyyy,mm,jj=date.split('/')
hh,mn,ss=heure.split(':')
# print(yyyy,mm,jj,hh,mn,ss)
dateobs=datetime(int(yyyy),int(mm),int(jj),int(hh),int(mn),int(ss))
return dateobs
# ----------------------------------------------------------------------------------------------
# --------------------- export as NetCDF4 pour une station ---------------
# ----------------------------------------------------------------------------------------------
def genereNetCDF4(file):
fName=sys._getframe().f_code.co_name
today=datetime.now()
print(fName," : Generate NetCDF Data for ",file," Today Date =",today)
dateHeure=[]
tempMer=[]
tempAir=[]
pressionMer=[]
pressionAir=[]
latitude=43.3297
longitude= 5.042020
nl=0
# stName=premiereColonne
# Extract data info from CSV for stName
# stName,date,heure,dateutc,heureutc,tz,peau,teau,pair,tair,x,y,diff,file,tel
stName=os.path.basename(file)
# Nbs secondes ecoulees depuis (tm_year=1970, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=0, tm_sec=0, tm_wday=3, tm_yday=1, tm_isdst=0)
# nbSecondesNouvelleref=int(calendar.timegm((2013,1,1,0,0,0)))
# nbSecRef=datetime(2013, 1, 1)
# nbSecondeEvenement=int(calendar.timegm((2013,1,1,1,0,0)))
# nouveauNbSecondes=nbSecondeEvenement-nbSecondesNouvelleref
# dateHeure=dateHeure+[row[1]]
# dt.datetime.strptime("2018-01-23 23:30:04", "%Y-%m-%d %H:%M:%S") - dt.datetime(2013, 1, 1)).total_seconds()
# dateHeure=dateHeure+[(datetime.strptime(row[1].strftime("%Y-%m-%d %H:%M:%S"), "%Y-%m-%d %H:%M:%S") -nbSecRef).total_seconds()]
# =========== Extraction des donnees du fichier CSV (ici, on pourrait aller chercher dans une base ou autre chose) ================
# Lecture "dictionnaire"; La premiere ligne contient les noms des colonnes
nl=0
nlS=0
longL=130
sizeF=os.path.getsize(file)
dispL=int(sizeF/longL/10)
print("Around 10 reading steps")
# dispL=1000
with open(file) as csvfile:
readCSV = csv.DictReader(csvfile,delimiter=',')
for row in readCSV:
nl=nl+1
if math.fmod(nl,dispL) == 0 :
nlS=nlS+1
# print("Read :",nl,math.fmod(nl,dispL),nl%dispL)
print("Step :",nlS," / ",nl," lines")
# print(row['dateutc'], row['heureutc'])
# la fonction date2num convertit la date objet de datetime en format numerique depuis la date indiquee
# dans NC_TIME_FMT cf http://unidata.github.io/netcdf4-python/#netCDF4.date2num
dateobs=convert_datenum(row['dateutc'],row['heureutc'])
datenum = nc4.date2num(dateobs, units=NC_TIME_FMT)
dateHeure.append(datenum)
pressionMer=pressionMer+[row['peau']]
pressionAir=pressionAir+[row['pair']]
tempMer=tempMer+[row['teau']]
tempAir=tempAir+[row['tair']]
print("Read ",nl," lines")
# ======= Create NetCDF File ==========
print("lat=",latitude,"lng=",longitude,"stName=",stName)
stName,ext=stName.split('.')
netCDFFileName="/tmp/"+stName+".nc"
print ("\n\tATTENTION : GENERATE UNDER /tmp\n")
print("Generate NetCDF File = "+netCDFFileName)
f = nc4.Dataset(netCDFFileName,'w', format='NETCDF4') #'w' stands for write
# tempgrp = f.createGroup(stName+'_station_data')
#==== Add global attributes =======
# Exemple 1 : Metedata Info in CSV file
# Lecture "normale", on indice les colonnes par des numeros
# On va chercher les donnees dans le csv et on indice le tableau avec les mots cles
# Le CSV contient des lignes sous la forme motCle|chaineDeMetaDonnee
ncMD={}
with open ("Metadata-"+file) as csvfile:
reader = csv.reader(csvfile, delimiter='|', quotechar='"')
for row in reader:
ncMD[row[0]]=row[1]
# Request For Acknowledgement:
f.rfa=ncMD["rfa"]
f.license=ncMD["license"]
f.summary=ncMD["summary"]
f.lineage=ncMD["lineage"]
f.description = ncMD["description"]
f.project=ncMD["project"]
f.title = ncMD["title"]
f.license=ncMD["license"]
f.Request_for_acknowledgement=ncMD["rfa"]
f.summary=ncMD["summary"]
f.keywords=ncMD["keywords"]
# Exemple 2 : Direct METADATA in code
f.history = "Created file : " + today.strftime("%d/%m/%y")
f.creator_name="HTMNET Project"
f.creator_type="group"
# f.title = "HTMNET Data for station "+stName
f.keywords_vocabulary = "GCMD Science Keywords";
f.production = "MIO UMR 7294 CNRS / OSU Pytheas"
f.contact = "Didier Mallarino (didier.mallarino@osupytheas.fr)"
f.source = "HTMNET Network"
f.institution = "MIO UMR 7294 CNRS / OSU Pytheas"
f.infoUrl = "https://htmnet.mio.osupytheas.fr/"
f.references = "https://htmnet.mio.osupytheas.fr/"
f.featureType = "TimeSeries"
f.cdm_data_type = "TimeSeries"
f.Conventions = "CF-1.6"
f.cdm_timeseries_variables = "station_name,latitude,longitude" ;
print("Header OK")
# ==== En commentaire, il reste des commandes de creation des groupes pour du NetCDF4 ====
# tempgrp.createDimension('time', None)
# tempgrp.createDimension('station', 1)
# tempgrp.createDimension('station_name_len', 40)
f.createDimension('time', None)
f.createDimension('station_name_len', 20)
# ==== Creating Variables ======
# Time = nom variable Netcdf qui depend de la dimension "time"
# time = tempgrp.createVariable('Time', 'i4', 'time')
# Variable associee à la dimension et proprietes de la variable
time = f.createVariable('time', 'i4', 'time')
#Add local attributes to variable instances
time.standard_name = "time"
time.units = NC_TIME_FMT
time.calendar = "standard"
time.axis = "T"
time.coverage_content_type = "coordinate"
# set variable value
time[:]=dateHeure
# ==== Creating Variables and attributes ======
# Station Name
# station_name = tempgrp.createVariable('station_name', 'f4', ('station','station_name_len'))
# station_name = tempgrp.createVariable('station_name', 'str', 'station_name_len')
station_name = f.createVariable('station_name', 'S1', 'station_name_len')
station_name.long_name="station_name"
station_name.cf_role = "timeseries_id"
# station_name=stName
dataName=np.array(stName,dtype='S20')
# dataName=numpy.array(["portCros"],dtype="S1')
station_name[:]=nc4.stringtochar(dataName)
# latitude Station
# lat = tempgrp.createVariable('lat', 'f4', 'station')
# lat = tempgrp.createVariable('lat', 'f4')
lat = f.createVariable('lat', 'f4')
lat.standard_name = "latitude"
lat.long_name = "station latitude"
lat.units = "degrees_north"
# lat.latitude = lat
lat[:]=float(latitude)
# longitude Station
# lon = tempgrp.createVariable('lon', 'f4', 'station')
# lon = tempgrp.createVariable('lon', 'f4')
lon = f.createVariable('lon', 'f4')
lon.standard_name = "longitude"
lon.long_name = "station longitude"
lon.units = "degrees_east"
# lon.longitude = lng
lon[:]=float(longitude)
# Definition variable Temperature de l'eau
# tmer = tempgrp.createVariable('tmer', 'f4', ('time','station'))
# tmer = tempgrp.createVariable('tmer', 'f4', 'time')
tmer = f.createVariable('tmer', 'f4', 'time')
tmer.standard_name = "sea_surface_temperature" ;
tmer.long_name = "SST"
tmer.units = "degC"
# set variable value
tmer[:]=tempMer
# Pression à 1 metre
# pmer = tempgrp.createVariable('pmer', 'f4', ('time','station'))
# pmer = tempgrp.createVariable('pmer', 'f4', 'time')
pmer = f.createVariable('pmer', 'f4', 'time')
pmer.standard_name="sea_water_pressure"
pmer.long_name = "1m pressure"
pmer.units = "hPa"
# set variable value
pmer[:]=pressionMer
# Temperature air
# tair = tempgrp.createVariable('tair', 'f4', ('time','station'))
# tair = tempgrp.createVariable('tair', 'f4', 'time')
tair = f.createVariable('tair', 'f4', 'time')
tair.standard_name = "air_temperature" ;
tair.long_name = "Air Temperature"
tair.units = "degC"
# set variable value
tair[:]=tempAir
# Pression air
# pair = tempgrp.createVariable('pair', 'f4', ('time','station'))
# pair = tempgrp.createVariable('pair', 'f4', 'time')
pair = f.createVariable('pair', 'f4', 'time')
pair.standard_name = "air_pressure" ;
pair.long_name = "Air pressure"
pair.units = "hPa"
#set variable value
pair[:]=pressionAir
print("Variables OK")
# Close and write to disk
f.close()
print("File Written & Closed / ncdump -h to see header")
# ----------------------------------------------------------------------------------------------
# ------------------------ Fonction pour l'usage du script -----------------
# ----------------------------------------------------------------------------------------------
def usage():
fName=sys._getframe().f_code.co_name
print(fName," : \n =============== USAGE ",nomScript," ======================\n")
print ("Ce programme permet de tracer des courbes ou de generer des fichiers NEetCDF a partir des donnees de la base HTMNET.")
print ("\t",nomScript," -h : Affiche l'aide")
print ("\t",nomScript," -n <station> : Genere un fichier NetCDF de la station; si <station> vaut 'all', on genere pour toutes les stations automatiques")
# ====================================
# ============== MAIN =================
# ====================================
def main(argv):
netcdf=0
single=0
if len(sys.argv) == 1:
usage()
sys.exit(2)
try:
opts, args = getopt.getopt(argv, "n:h", ["netcdf=","help"])
except getopt.GetoptError:
print("No options : Exiting...")
usage()
sys.exit(2)
for opt, arg in opts:
if opt in ("-n", "--netcdf"):
fileName=str(arg)
try:
with open(fileName): pass
except IOError:
print ("Error : File does not exist")
sys.exit(2)
if fileName == "all":
single=0
else:
single=1
netcdf=1
if opt in ("-h", "--help"):
usage()
sys.exit(0)
# Generation des fichiers netcdf
if netcdf==1:
if single==1:
print("Call genereNetCDF4(",fileName,")")
genereNetCDF4(fileName)
else:
print("Call generateNetCDF4 for all stations")
print("Non implemented")
sys.exit(0)
# netcdfAllStations(periode)
# genereAllNetCDF4(fileName)
else:
print("Nothing to do")
if __name__ == "__main__":
# Recuperation des arguments
nomScript=os.path.basename(sys.argv[0])
# unité de temps pour netcdf
global NC_TIME_FMT
NC_TIME_FMT = 'seconds since 1970-01-01 00:00:00 UTC'
# print ("MonScript=",nomScript)
# ---------------------- Verifie la presence d'un fichier de configuration ---------------------
main(sys.argv[1:])
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment