tuneBR and obstool

CMakeLists.txt

@@ -19,15 +19,21 @@ endif()
 set(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} ${bounds}")
 set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${dialect}")
 
+
 #
-# Compile.
+# .Compile 
 #
+# compilation of readgsa.F90 routine as a python extention 
+add_subdirectory(src)
+
 
 add_executable(jbdiagnose.x src/jbdiagnose.F90)
 add_executable(cv_header_list.x src/cv_header_list.F90)
 add_executable(prepdiacov.x src/prepdiacov.F90)
 add_executable(clddet_analyzer.x src/clddet_analyzer.F90)
 
+
+
 # install executables and scripts
 install (TARGETS jbdiagnose.x
          PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ
@@ -46,6 +52,8 @@ install (TARGETS clddet_analyzer.x
          DESTINATION libexec)
 
 
+
 add_subdirectory (scripts)
+add_subdirectory (modules)
 add_subdirectory (share)
 add_subdirectory (tests)

modules/CMakeLists.txt

@@ -0,0 +1,30 @@
+function (install_modules )
+  foreach (file ${ARGV})
+    get_filename_component (name_without_extension ${file} NAME_WE)
+    install (FILES ${file}
+             PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ
+             DESTINATION modules  )
+  endforeach ()
+endfunction()
+
+
+# __init__ file 
+install_modules  (__init__.py)
+# Py  modules For tuneBR 
+install_modules  (readgsa.so   
+	          gsacov.py  
+	          read_odb.py  
+		  config_env.py  
+		  sigma_bo.py )
+
+# Py modules for obstool 
+install_modules  (build_sql.py  
+	         conv_stats.py  
+		 handle_df.py  
+		 io_base.py  
+		 setting.py  
+		 utils.py
+		 obstype_info.py)
+
+
+

modules/__init__.py

@@ -0,0 +1,17 @@
+#-*- cvoding: utf-8 -*-
+
+# TuneBR modules 
+from  . import gsacov
+from  . import config_env
+from  . import sigma_bo
+from  . import read_odb
+
+
+# Obstool modules 
+from  . import conv_stats
+from  . import build_sql
+from  . import obstype_info
+from  . import io_base
+from  . import setting
+from  . import utils
+

modules/build_sql.py

@@ -0,0 +1,153 @@
+# -*- coding:utf-8 -*- 
+import os ,sys  
+from  odb4py.utils   import SqlParser 
+
+
+
+class SqlHandler:
+    """
+    Class : @ Parse the different observation dictionaries and build the 
+              SQL query according to obstype, codetype, varno , sensor and level range  
+            @ Checks the SQL statement before sending. 
+    Returns : The sql statement according to variable and varno  
+
+            Methods : BuildQuery  
+                      CheckQuery
+
+
+    """
+    def __init__(self ):
+
+        # For the moment !
+        self.obstool_select=None 
+
+        #self.derozier_select=None
+        #self.jarvinen_select=None
+        return None 
+
+
+    def BuildQuery(self , **kwarg ):
+        self.cols      =",".join(kwarg["columns" ])[1:]
+        self.tabs      =",".join(kwarg["tables"  ])
+        self.obs_      =kwarg["obs"]
+        self.varno     =kwarg["obsvano"]
+        self.ctype     =kwarg["codetype"]
+        self.obst      =kwarg["obstype"]
+        self.levels    =kwarg["lrange"]
+        self.vtype     =kwarg["vertco_type"]
+        self.vert_coord=kwarg["vertco"]
+        self.sensor    =kwarg["sensor"        ]
+        self.other     =kwarg["remaining_sql"]
+
+
+        # Obstool columns and tables 
+        self.obstool_select="SELECT  "+self.cols+" FROM " +self.tabs
+
+        # Check obstype  
+        type_list=[]
+        if self.obst is not None:
+           if isinstance (self.obst, list ):
+              for tp in self.obst:
+                  if tp is not None:
+                     type_list.append (" obstype =="+str( tp ) )
+           elif isinstance (self.obst, int  ):
+              type_list.append (" obstype =="+str( self.obst ) )                
+           else:
+              print(self.obs_, " : obstype  must be integer or a list" )
+              sys.exit()
+
+        # Check varno  
+        varno_list=[]
+        if self.varno is not None:
+           if isinstance (self.varno , list ) :
+              for vr in self.varno:
+                  if vr is not None:
+                     varno_list.append (" varno =="+str( vr ) )
+           elif  isinstance( self.varno , int  ):
+               varno_list.insert(0, " varno =="+ str(self.varno)  )
+
+           else:
+              print(self.obs_ , ": varno must be integer ot list" )
+              sys.exit()
+
+        # Check codetype 
+        """
+        Not needed for the moment !
+        ctype_list=[""]
+        if self.ctype is not None:
+            if isinstance( self.ctype,  list  ):
+               for ct  in self.ctype:
+                   if ct is not None:
+                      ctype_list.append (" codetype =="+str( ct ) )
+            elif isinstance( self.ctype , int  ):
+                   ctype_list.append(  " codetype =="+str(self.ctype ) )
+            else:
+               print(self.obs_, ": codetype  must be integer or list "  )
+               sys.exit()"""
+
+
+        # Check sensor 
+        sensor_list =[]
+        if self.sensor != None:
+            if isinstance   (self.sensor ,  list  ):
+               for s in   self.sensor:
+                   if s is not None:
+                      sensor_list.append( " sensor=="+str(s)   )
+            elif isinstance (self.sensor ,  int ):
+                     sensor_list.append(  " sensor =="+str( self.sensor ) )
+            else: 
+                print(self.obs_, ": sensor  must be integer or list"  )
+                sys.exit()
+
+
+        # Check level range 
+        level_list=[]
+        if self.levels != None :
+          if len (self.levels) <2  or  len( self.levels ) > 2:
+             print ( "Level range must have two limites   [l1 , l2], but got argument:", self.levels  )
+             sys.exit(0)
+          elif not isinstance (self.levels[0], int ) or not isinstance ( self.levels [1], int):
+             print( "One or the both level limites in the list  is/are not intger(s)" )
+             sys.exit(0)
+          elif self.levels[1] < self.levels[0]:
+             print( "Bottom level limit can't be greater than top level limit" )
+             sys.exit(0)
+          else:
+             l1 =str(self.levels[0] )
+             l2 =str(self.levels[1] )
+             level_cond="vertco_reference_1 >="+l1+" AND vertco_reference_1 <= "+l2 
+             level_list.append(level_cond)
+
+
+        where_cond_list=[]
+        for tp in type_list:            
+           if len( type_list) !=0 : 
+               where_cond_list.append(  "".join( ( tp )  ) )
+           for vr in varno_list:
+               if  len( varno_list) !=0:
+                   where_cond_list.append(  " AND ".join(  (tp, vr  ))   )
+               if len(level_list) !=0:
+                   for lv in level_list:
+                       where_cond_list.append(  " AND ".join(  (tp, vr ,  lv  ))   )
+
+        query=" " 
+        if   self.other !=None and len(where_cond_list) == 0:
+             query= self.obstool_select +" WHERE  "+self.other
+
+        elif self.other !=None and len(where_cond_list) > 0:             
+             for condition in where_cond_list:
+                 query=self.obstool_select +" WHERE  " +  condition + "  AND  "+self.other 
+
+        elif self.other ==None and len(where_cond) > 0:
+             for condition in where_cond_list:
+                 query= self.obstool_select  +" WHERE  " +  condition 
+
+        return  query
+
+
+
+    def CheckQuery(self  , query   ):
+        p      =SqlParser()
+        nfunc  =p.get_nfunc    ( query  )     # N Columns=N pure columns - N functions in the query 
+        sql    =p.clean_string ( query  )     # Remove any unprintable character ( \r\t ....etc)
+        return nfunc ,  sql 

modules/config_env.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+import os  
+import configparser  
+
+
+class TuneEnv:
+      """
+      Class:  Parse the sections and get the items  as a dict              
+      """
+      def __init__(self, config ):
+          self.BeginDate= config.get('DATES', 'DATESTART') 
+          self.EndDate  = config.get('DATES', 'DATEEND'  )
+          #self.ntaks    = config.get('DATES', 'NSLICE'     , fallback=2)
+          self.outfile  = config.get('OPTIONS', 'OUTFILE'  , fallback = "sigmabo_ratios"  )
+          self.dca_cpu  = config.get('OPTIONS' ,'DCA_CPU', fallback= 4 )
+          self.llverb   = config.getboolean('OPTIONS' ,'LLVERB' , fallback="false")
+          self.lwrite   = config.getboolean('OPTIONS' ,'LWRITE' , fallback="false")
+          self.lplot    = config.getboolean('OPTIONS' ,'LPLOT'  , fallback="false")
+          self.prog_bar = config.getboolean('OPTIONS' ,'PROGBAR', fallback="false")
+
+
+
+
+          pp = config.items("PATHS")
+          self.path_opt = { str(opt[0]):str(opt[1]) for opt in pp }
+
+          self.basedir =self.path_opt["BASEDIR"]
+          self.stabal  =self.path_opt["STATFILE"]
+          self.odbpath =self.path_opt["ODBPATH"]
+          self.odb_template=self.path_opt["ODB_TEMPLATE"]
+          self.rows_path=self.path_opt["ROWS_PATH"]
+          self.tmp_dir  =self.path_opt["WORKDIR"]
+
+          mm =  config.items("MODEL")
+          self.model_opt= { str(opt[0]):str(opt[1]) for opt in mm }
+
+          self.cycle_inc =self.model_opt["CYCLE_INC"]
+          self.deltax    =self.model_opt["DELTAX"]
+          self.nsmax     =self.model_opt["NSMAX"]
+          self.nflev     =self.model_opt["NFLEV"]
+          self.rednmc    =self.model_opt["REDNMC"] 
+          return None    
+
+      def __Dicts__(self ):
+          PathAttr = self.path_opt
+          ModelAttr= self.model_opt 
+          return PathAttr ,ModelAttr 
+
+