From f98f9d32371563206986b41c30aeee04bd678751 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Tue, 3 Mar 2026 23:35:58 +0000 Subject: [PATCH 01/80] Cmake file to compile readgsa.F90 fortran routine --- src/CMakeLists.txt | 67 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 src/CMakeLists.txt diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000..6c7daf0 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,67 @@ +cmake_minimum_required(VERSION 3.18) +project(readgsa LANGUAGES C Fortran) + +find_package(Python 3.10 REQUIRED + COMPONENTS Interpreter Development.Module NumPy +) + + + + +message(STATUS "Python executable : ${Python_EXECUTABLE}") +message(STATUS "NumPy include dirs : ${Python_NumPy_INCLUDE_DIRS}") + +message("------------------------listinig ----------------------") +execute_process( + COMMAND ls ${F2PY_INCLUDE_DIR} +) + + +message("------------------------listinig ----------------------") +execute_process( + COMMAND "${Python_EXECUTABLE}" + -c "import numpy.f2py; print(numpy.f2py.get_include())" + OUTPUT_VARIABLE F2PY_INCLUDE_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE +) + +message(STATUS "F2PY_INCLUDE_DIR = ${F2PY_INCLUDE_DIR}") + + +set(f2py_module_name readgsa) +set(fortran_src_file ${CMAKE_SOURCE_DIR}/src/readgsa.F90) +set(f2py_module_c ${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_name}module.c) + +# Generate wrapper +add_custom_command( + OUTPUT ${f2py_module_c} + COMMAND ${Python_EXECUTABLE} -m numpy.f2py + ${fortran_src_file} + -m ${f2py_module_name} + --lower + DEPENDS ${fortran_src_file} + COMMENT "Generating f2py wrapper" +) + +# Python_add_library( readgsa MODULE WITH_SOABI +# ${f2py_module_c} # Generated +# "${F2PY_INCLUDE_DIR}/fortranobject.c" # From NumPy +# "readgsa-f2pywrappers2.F90" # Fortran source(s) #"foo-f2pywrappers2.f90" +# ) + +# Build module +Python_add_library(${f2py_module_name} MODULE WITH_SOABI + ${F2PY_INCLUDE_DIR} + ${f2py_module_c} + ${fortran_src_file} +) + +target_include_directories(${f2py_module_name} PRIVATE + ${F2PY_INCLUDE_DIR} + ${Python_NumPy_INCLUDE_DIRS} +) + +target_link_libraries(${f2py_module_name} PRIVATE + Python::Module + Python::NumPy +) From 628ccf91d215c0281e6e2079d7acbae064cd1621 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Tue, 3 Mar 2026 23:37:56 +0000 Subject: [PATCH 02/80] Fortran routine to read background variances --- src/readgsa.F90 | 120 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 src/readgsa.F90 diff --git a/src/readgsa.F90 b/src/readgsa.F90 new file mode 100644 index 0000000..d326b8f --- /dev/null +++ b/src/readgsa.F90 @@ -0,0 +1,120 @@ + SUBROUTINE READCOV(pcov,kdim,nsmax,kpar,cfile ,llverb,kret) + IMPLICIT NONE +! read a set of nonsep matrices for the requested parameter +! from a GSA file +! F. Bouttier Oct 96 +! input : cfile = GSAfilename +! kpar = GSA parameter selector (IPAR coding) +! output : pcov = covariance matrices +! kret = ok if 0, not found if 1 + +! ROUTINE BASED ON +! THE MAIN PROGRAM stdav.F90 : G. Boloni based on "fediacov" 2010/09/09 +! : B. Strajnar 2010/10/26 +! USE AS PYTHON EXTENSION : I. Dehmous 2023/10/03 + + +! + INTEGER, PARAMETER :: jplun=67 + INTEGER, PARAMETER :: JPRB = SELECTED_REAL_KIND(13,300) + INTEGER, PARAMETER :: JPIM = SELECTED_INT_KIND(9) + INTEGER(KIND=JPIM),INTENT(IN ) :: kdim,nsmax ,kpar + INTEGER(KIND=JPIM),INTENT(OUT) :: kret + INTEGER(KIND=JPIM) :: idim1,idim2,ipar1,ipar2,icor1,icor2, icheck + REAL(KIND=JPRB) ,INTENT(OUT) :: pcov(kdim,kdim,0:nsmax) + CHARACTER(LEN=*) :: cfile + CHARACTER :: clid*8,clcom*70 + LOGICAL ,INTENT(IN) :: llverb + LOGICAL :: llfound + INTEGER(KIND=JPRB) :: iorig,idate,itime,inbset + INTEGER(KIND=JPRB) :: jset , jj , jn , jk + INTEGER(KIND=JPRB) :: inbmat,iweight,itypmat,isetdist,ilendef + + +! 1.OPEN FILE AND READ GSA HEADER +! THE FILE IS WRITTEN IN BIG ENDIAN MODE +! AVOID COMPILER OPTIONS ! + OPEN(jplun,FILE=cfile,FORM='unformatted',STATUS='old',& + & ACCESS='sequential',convert='BIG_ENDIAN') + + + + IF (llverb) THEN + WRITE(*,*) 'SEARCHING FOR PARAMETER:',kpar, & + & 'IN GSA FILE ',cfile,'...' + ENDIF + + READ(jplun) clid + READ(jplun) clcom +! This read statement below is not always valid!!! +! In some versions of festat it is not written to the stabal file!!! + READ(jplun) iorig,idate,itime,inbset + IF (llverb) THEN + WRITE(*,*) 'ID =',clid + WRITE(*,*) 'COMMENT=',clcom + WRITE(*,*) 'ORIGINE,DATE,TIME,INBSET',iorig,idate,itime,inbset + ENDIF + +! 2. SEARCH THE SET + llfound=.false. + DO jset=1,inbset + READ(jplun) inbmat,iweight,itypmat,isetdist,ilendef + !IF (llverb) THEN + WRITE(*,*) 'SCANNING SET No...',jset + !ENDIF + IF (llverb) THEN + WRITE(*,*) ' ',inbmat,iweight,itypmat,isetdist,ilendef + ENDIF + READ(jplun) idim1,idim2,ipar1,ipar2,icor1,icor2 + IF (llverb) THEN + WRITE(*,*) ' ',idim1,idim2,ipar1,ipar2,icor1,icor2 + ENDIF + IF ((ipar1==kpar).AND.(ipar2==kpar)) llfound=.true. +! skip coordinate records + READ(jplun) + READ(jplun) + IF (llfound) THEN + IF (llverb ) THEN + WRITE(*,*) 'FOUND PARAM :', kpar ,' IN THE SET :', jset + WRITE(*,*) 'READING DATA ...' + ENDIF + +! THE OLD ABORT ROUTINE CAUSES PYTHON CRASH + IF (inbmat/=nsmax+1)call abor1('Error - bad matrix number') + IF (itypmat/=1 )call abor1('Error - not vert cov matrices') + IF (isetdist/=2 )call abor1('Error - not distrib by wave n') + IF (ilendef/=1 )call abor1('Error - bad def list length') + IF (idim1/=kdim )call abor1('Error - bad 1st dimension') + IF (idim2/=kdim )call abor1('Error - bad 2nd dimension') + IF (icor1/=icor2 )call abor1('Error - coords differ') + DO jn=0,nsmax + READ(jplun) + READ(jplun) ((pcov(jj,jk,jn),jk=1,kdim),jj=1,kdim),icheck + IF (icheck/=3141592) call abor1('Error - wrong check word') + ENDDO + EXIT + ELSE +! SKIP THE SET + DO jn=1,inbmat + IF(ilendef>0) read(jplun) + READ(jplun) + ENDDO + ENDIF + ENDDO + CLOSE(jplun) + IF (llfound) THEN + kret=0 + ELSE + WRITE(*,*) 'WARNING : COULD NOT FIND MATRIX SET' + pcov(:,:,:)=0. + kret=1 + ENDIF + RETURN + END SUBROUTINE READCOV + +! ------------------------------------------------------------------ + SUBROUTINE abor1(cdmess) + character(len=*) :: cdmess + WRITE(*,*) cdmess + !call exit(1) + END From 611804ff0a98e5dec0dcc4297b341134d016a3fe Mon Sep 17 00:00:00 2001 From: rmibdev Date: Wed, 4 Mar 2026 10:46:49 +0000 Subject: [PATCH 03/80] Add subdirectory src and compile readgsa.F90 routine --- CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6b9eda9..5879692 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,10 @@ endif() set(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} ${bounds}") set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${dialect}") + +# Compile readgsa.F90 routine but as python module +add_subdirectory(src) + # # Compile. # @@ -28,6 +32,8 @@ add_executable(cv_header_list.x src/cv_header_list.F90) add_executable(prepdiacov.x src/prepdiacov.F90) add_executable(clddet_analyzer.x src/clddet_analyzer.F90) + + # install executables and scripts install (TARGETS jbdiagnose.x PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ @@ -46,6 +52,8 @@ install (TARGETS clddet_analyzer.x DESTINATION libexec) + add_subdirectory (scripts) +add_subdirectory (modules) add_subdirectory (share) add_subdirectory (tests) From 4232b78e5a7fa4ba906bcbf77476c18aea388261 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Wed, 4 Mar 2026 10:47:29 +0000 Subject: [PATCH 04/80] Install tune_br.py script --- scripts/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 7ce6714..0f2df6e 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -12,6 +12,7 @@ install_scripts (jbdiagnose.sh plotjbbal.py plotjbdiag.py) install_scripts (cv_header_list.sh) install_scripts (varbcdiagnose.sh plotvarbccoeff.py) install_scripts (rundiacov.sh diag_diacov.sh plotdiacov.py) -install_scripts(clddet_diagnosis.sh clddet_plotter.py) +install_scripts (clddet_diagnosis.sh clddet_plotter.py) install_scripts (datool_dfs.py) +install_scripts (tune_br.py) #install_scripts (diacov.sh festat.sh) From c890c3782d9c77f4f132a12d4fe21b4307033a9d Mon Sep 17 00:00:00 2001 From: rmibdev Date: Wed, 4 Mar 2026 10:54:20 +0000 Subject: [PATCH 05/80] Add comments for the different steps --- src/CMakeLists.txt | 87 ++++++++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 37 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6c7daf0..3d0abff 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,67 +1,80 @@ -cmake_minimum_required(VERSION 3.18) -project(readgsa LANGUAGES C Fortran) +# CmakeFile to compile +cmake_minimum_required (VERSION 3.18) +project (readgsa LANGUAGES C Fortran) -find_package(Python 3.10 REQUIRED - COMPONENTS Interpreter Development.Module NumPy -) +# Find system python +find_package(Python REQUIRED COMPONENTS Interpreter) +message(STATUS "System python: ${Python_EXECUTABLE}") +# Create venv inside build tree +set(VENV_DIR ${CMAKE_BINARY_DIR}/venv) +if(NOT EXISTS ${VENV_DIR}/bin/python) + message(STATUS "Creating virtualenv") + execute_process( + COMMAND ${Python_EXECUTABLE} -m venv ${VENV_DIR} + RESULT_VARIABLE venv_status + ) + if(NOT venv_status EQUAL 0) + message(FATAL_ERROR "Failed to create virtualenv") + endif() + execute_process( COMMAND ${VENV_DIR}/bin/pip install --upgrade pip ) + execute_process( COMMAND ${VENV_DIR}/bin/pip install numpy ) +endif() -message(STATUS "Python executable : ${Python_EXECUTABLE}") -message(STATUS "NumPy include dirs : ${Python_NumPy_INCLUDE_DIRS}") -message("------------------------listinig ----------------------") -execute_process( - COMMAND ls ${F2PY_INCLUDE_DIR} -) +# Switch python to venv +set(Python_EXECUTABLE ${VENV_DIR}/bin/python CACHE FILEPATH "" FORCE) +find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy) +message(STATUS "Python from venv: ${Python_EXECUTABLE}") -message("------------------------listinig ----------------------") +# Locate f2py include execute_process( - COMMAND "${Python_EXECUTABLE}" - -c "import numpy.f2py; print(numpy.f2py.get_include())" - OUTPUT_VARIABLE F2PY_INCLUDE_DIR + COMMAND ${Python_EXECUTABLE} + -c "import numpy.f2py;print(numpy.f2py.get_include())" + OUTPUT_VARIABLE F2PY_INCLUDE OUTPUT_STRIP_TRAILING_WHITESPACE ) -message(STATUS "F2PY_INCLUDE_DIR = ${F2PY_INCLUDE_DIR}") - - +# Generate f2py wrapper set(f2py_module_name readgsa) -set(fortran_src_file ${CMAKE_SOURCE_DIR}/src/readgsa.F90) -set(f2py_module_c ${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_name}module.c) +set(FSRC ${CMAKE_SOURCE_DIR}/src/readgsa.F90) + -# Generate wrapper add_custom_command( - OUTPUT ${f2py_module_c} + OUTPUT readgsa COMMAND ${Python_EXECUTABLE} -m numpy.f2py - ${fortran_src_file} + ${FSRC} -m ${f2py_module_name} --lower - DEPENDS ${fortran_src_file} - COMMENT "Generating f2py wrapper" -) + DEPENDS ${FSRC} ) -# Python_add_library( readgsa MODULE WITH_SOABI -# ${f2py_module_c} # Generated -# "${F2PY_INCLUDE_DIR}/fortranobject.c" # From NumPy -# "readgsa-f2pywrappers2.F90" # Fortran source(s) #"foo-f2pywrappers2.f90" -# ) - -# Build module -Python_add_library(${f2py_module_name} MODULE WITH_SOABI - ${F2PY_INCLUDE_DIR} - ${f2py_module_c} - ${fortran_src_file} +# Build the module +Python_add_library(${f2py_module_name} MODULE WITH_SOABI + ${F2PY_INCLUDE_DIR} + ${f2py_module_name} + ${FSRC} ) +# Include target_include_directories(${f2py_module_name} PRIVATE ${F2PY_INCLUDE_DIR} ${Python_NumPy_INCLUDE_DIRS} ) +# Libs target_link_libraries(${f2py_module_name} PRIVATE Python::Module Python::NumPy ) + +# Set the suffix to " .so " +set_target_properties(readgsa PROPERTIES + PREFIX "" + SUFFIX ".so" +) + +# Add it to the python modules +install(TARGETS readgsa LIBRARY DESTINATION ${CMAKE_SOURCE_DIR}/modules ) From ed7daa094295d6ce495892b73c1c732558f19b38 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Wed, 4 Mar 2026 10:54:49 +0000 Subject: [PATCH 06/80] Install tunebr_config.ini file --- share/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/CMakeLists.txt b/share/CMakeLists.txt index 5c8bb18..2171e73 100644 --- a/share/CMakeLists.txt +++ b/share/CMakeLists.txt @@ -1,2 +1,2 @@ install( DIRECTORY levdef DESTINATION share PATTERN "CMakeLists.txt" EXCLUDE ) - +install( DIRECTORY tunebr_conf DESTINATION share PATTERN "CMakeLists.txt" EXCLUDE ) From 1c55ece3468243988e996aebfb3a397e0a096961 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Wed, 4 Mar 2026 10:55:26 +0000 Subject: [PATCH 07/80] TuneBR pure python modules --- modules/CMakeLists.txt | 14 + modules/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 508 bytes modules/__pycache__/gsacov.cpython-312.pyc | Bin 0 -> 6405 bytes modules/__pycache__/odb.cpython-312.pyc | Bin 0 -> 9462 bytes .../__pycache__/setting_env.cpython-312.pyc | Bin 0 -> 2928 bytes modules/__pycache__/sigma_bo.cpython-312.pyc | Bin 0 -> 17208 bytes modules/config_env.py | 49 +++ modules/gsacov.py | 131 +++++++ modules/odb.py | 248 ++++++++++++ modules/readgsa.so | Bin 0 -> 17152 bytes modules/sigma_bo.py | 359 ++++++++++++++++++ 11 files changed, 801 insertions(+) create mode 100644 modules/CMakeLists.txt create mode 100644 modules/__pycache__/__init__.cpython-312.pyc create mode 100644 modules/__pycache__/gsacov.cpython-312.pyc create mode 100644 modules/__pycache__/odb.cpython-312.pyc create mode 100644 modules/__pycache__/setting_env.cpython-312.pyc create mode 100644 modules/__pycache__/sigma_bo.cpython-312.pyc create mode 100755 modules/config_env.py create mode 100755 modules/gsacov.py create mode 100755 modules/odb.py create mode 100755 modules/readgsa.so create mode 100755 modules/sigma_bo.py diff --git a/modules/CMakeLists.txt b/modules/CMakeLists.txt new file mode 100644 index 0000000..a33fefa --- /dev/null +++ b/modules/CMakeLists.txt @@ -0,0 +1,14 @@ +function (install_modules ) + foreach (file ${ARGV}) + get_filename_component (name_without_extension ${file} NAME_WE) + install (FILES ${file} + PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ + DESTINATION modules ) + endforeach () +endfunction() + +# Python modules For tuneBR +install_modules (gsacov.py odb.py setting_env.py sigma_bo.py ) +install_modules (readgsa.so) + + diff --git a/modules/__pycache__/__init__.cpython-312.pyc b/modules/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..199d168a262df9266ebb6d603ad80fa8c66c6b3b GIT binary patch literal 508 zcmYLFze^)Q7@f&(HtUagMT)d@RR~DdSo*6_I1nTU$AMroOm-)e(cPJKW;aKyVs*{+ zr}tFw*LG+?%i1YKWU0`-pTlf zB|kv~A{c`MPy!SjmI~7h?bMxh1^ZRHf>REsn_i*Jkh+EEpiIk$)tOST$SE8U3}MrK znh2dHH078MXneAd(AQkh?Jp>iQ65vYhC-R7c}5XIsni)&Tq=$tPMDCIMo4p(5Y&?j z$*!hqKr$``A|gUv5*{%nb4yXPVk)s@j0{WMIeN`i1e`a$*<8@F5S)=Dr9uLF#^ z;2FlPbZGpI>2ymi*;|vp)SLEsb|8df55NzPpf;*JfZ8pnoi#3&&zJ8(`z9`1+23<@LS>VOVY1A45(dQ4*j zHeLpovFUv@V8CXOMr;9T!d8%GTn5rIQSPyc`h)%6pc|aDGSbx%3vwLO&UAQ>9AUgn z*Rei-pSOqUJ9gaP-Q#1}a1al1!4@vTFp7q($1r_9ZztnFU`V@vM=zOh z2h76+M=|q81cWk7Ec1us$uJw_Q*1cx>`#V6k-C;A0o6>3Q^mh@}hmh_uS`Yo!~1oBWl9;-+vgZ!{) z2n|MJVNo9$}E22qxHS(ojPhj4?@rBTlLa{7-LdEXB1B?`Ff?+2?n)4ktsf zbn({EcyPFtjYrM|aCp3RD8dh?&a`?%p#+ONgMRRWYfX*}wYF%{)(Q|0CB|El6QVf~ zh>S+~KpS0>$Fo<(##l3E_>AXjNY+rTP<$RZ7&{9(Lth(3bx-P|`^|@7d zeQxii5qUs;Zl65%jgK8tqMkLB0i{ZfX$><(yxJ&v|~%@SPGrrMIKpASbNfz_y~@$%+4eunhtpTeVyHXX}UF$;?o8qTSY1$ zQfKdyHB8e4u_9%Kn9(IT(a243d^paMnMFF043CP|2p1XU_#osGk&Z<;UbKWqQ*n}r z!fY)Wr{{PUVl9^7AZ%E&#-eFF$c_f%VF(}jF&>Piphn^`Aez7nDKe>uNJSqB9CkfX zx4uDQaCaAsPTL9Ci}fwg&ynwR4t--Szu=kjWZ%p;UG>a)=DQ1<_7$r4EsPhcU%qAi z>59WOW0|((JOxKn{*8iT$2^EV3;hMhON-S7$19UtzNvI)Q+fSzWz*!gRa;GN%g0T( zY};0A8}oZ_*LtR_vd-+T6=&_t`swu_SLJ*2V{`q#s-53DxA8Ys3%#GZ3ym)>(uKzT zUpimDr$=?$@7P=yPEVcwWk4xhej%6osQpXZHg+TU8!C+yf>0@*^4}w#){c&~TL*P2 z8Zx>el+hPs#X3>rJ_T_O zt=b(IhNp&e^cBmjCBNrt+g#gH&CX?e%jD)2yJK?mYE^B{ncH)vZMN-p)t1R1$^70N zz7m;@+;%r-ovZch^E5&Aubuhhb7jl*Em?imn+;|=E}HM_QH=*c?b`aS8M*3mt}Au3 zb+@ZFPXzAzSo(uzRTX_I)L|AV|Cs3FgX@(Ev08l{d;}@2bzizaA?f#68*VxEfn-JkM zQH^gDtrON0)p$lpJBn&sNU`<0!h@s@Ll4lTv3}%5@Y+>`Bc^$cn+uc_3`z=+(z_mj zW$0mvakhyF(wW}Y}}l$ohH$O?re4^Bj3435GHVjR(U#q~5fZJ zuN_`=E#lure;fV6_2>FO)-PB4rPh!Qys;zR#_44oOoBq$OU)qk`an(!?JP8-F zJW|&D2lxs^6-6~30zJxwDK)wQOlyxQ=1BchMrhdpmUe9$8okXJG{_!>u^RYPe}H9Z z^J)BIZGL2sfWH;2Xw0T)1q0^*QmojQIMZ6A%rMGSGRi2JNDc!Zb&~a``KZ0I}m;rmyZ^~Tr&IGZ(_z4!BR5HELapD_0RNC`{t65)=b%t^wBC8`RK~u^0DLOx~RwoF2QdP2$5!GVDYbDnh_jLb}s{w@iglF|9jqq*2C^ zsVK2U(mgqUJ$99_mn_l#K^iI0a6YE7hQB3q41h+Z;1DXw7s*udF4Y~ODh)YFVoYmT zkou&J3*N%5>J*<+znw_JJ}{MrOjm|7E}=qj60~vD`Bwx3sR>mwuL7o1QX}6JR|qcE zf?$_=D+l!w-+Bx9)>*hO0;gF!L)q1wDWSGHmog$2mf*Fcl|8w9XK1TMTY^-j+Fi+6z#R8)PR${mF_Y}$IY zb*^<`&+?|Wf~|4#WX?j)Y}RRO4*$imiIgj*E0#9xUg%hS>6W8QX~^${dK;;`8gdg$ zuI43Mv&uw%m_^_J)1q(j&?7$j7Q6pPAHUP$w|3;LnlVqCb4^#a&u-86&hDJ2Ke1l3 zF6_EyTfm=1Z$xi8Z^W1CI+q>3$t~X+(fUpK#(d}1!*hohsJWxJ8g?%`_7)m;XU8r& zml}3YHZ3{!e)C<&hPn;8vApYQ{apR+x*b{XN(~$Gz;=<`uz?6El}MRI$}z*x zb`41cSBf;2Od<)A;W#q-eRN-EGFk6cqKbwUd*h0|ZpB`I-=r_!d=5V_TTJc;?lM#J zgAG`j&x5$9rzq-cWcdnN|B4#ELX~%r{sZ%S<_{{~ WtH_q!B2D&xnNt+?+yg|UWbwayA3$yZ literal 0 HcmV?d00001 diff --git a/modules/__pycache__/odb.cpython-312.pyc b/modules/__pycache__/odb.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9c25148e1bc5b17892012f8ff870cc14e083580b GIT binary patch literal 9462 zcmbtZTTmQVdhVY46&Pk+m-4|XM~xGJgCj0dk~+qmphRF(1@mYjIweaZiy zo}L+GWFB$`&gDPwkZw_7L(o7VN{$U%zwPb{cGm(M&|h0F@YQY;;#0!m5; zXeeu9nz%Ngr4^bkpaYsVrjHu}hPW|cjOzpXe4Qy^ChaW&3#7W3HEs*o;`V?&UKS`L zJo;F9ydqE$cLW^q%0Ok@8E_K1A?Au#1*&LDLk&@^@g0gaeV|ch7O0-5J?5{lFzTUY z-NleN<|HY{2SB2nICzXA>NB7Ez|7#QC{VPo9VJ3dK>l zQvnT22eho_9SZhH3$dQnL2O|45F1%DYj}qam~eFg^SsGpk+oxNIP^MAQ5J@T{#Zy5 zm>vemOe-_w^)vo+Ugl!2|J*n;=o=eh@>5~P`d;$(`K_~R_YRMa!{|OP#KH`G$?F32Il_rkNtW@CF+Oka zz|KRC^^LcAeZDat%sLqs+9qc?SjT`DCg8v1^TJ$42S`OI5sJ+VQDNt9N*e4qM;|B7 z@(HGAEGz^$p3e_9F`3VV5)8v~Ga(+hIGB3AkBIuM@XP7pUkm?F{sy8IN}?qy2stIN zSc0#SFKM7uXe%03`m{;S71Iy$*#YXdCZd_7BAPdVe^Dc8SZyuTEo*L>bwC9Q6~sA^ zAllWOM6){N`yf9$a23Xzhjuh+d!6#=M`c}q|48q`*tqus9s$^HaejttoeS}aWNRo9 zBs;(96v7`gsE9E8k3!qoEQx8k*Vk$7o14)@sKFUChmq9j70^J$BmZF zJe;E-c7>N#;9?W99!KnT7mx{#*>^o1Q!|knj*oXl=0Z~)d^{S4CCqh9M#ZVw za7S+>lH}QekUyD>2_5kyI~(JK4!ET4GxM@37>p*OVlcR1gM-_ymg=EvVH~2js9bGh zYTvCxx!UI-Ih?DVhNOM7{F=DBZ;gKMV7lXcre*|q+BWMN z*Ys;wGtAL!-LcfZKWhKq$6Bg(los$S-m~?(blc9C0jT7B^}&~1q0E$6I8W1svjjZ( zAQsMoM62JPrB&oR4kyS=Nt)@xSy0lq06xP{9f_vK zslAkFQ~CB$yp^V|5S~)5dChg|hV}||o!(1zQUZM)PUaPghTnXFPE%r;+7IL%YM#|j zJJozKEv{Q#KlZ5h=FyGHG(Wh|bh+0zIy^cA=p=@CW+EEnnBQMy5=oJnNX{nMb_USY z2vt}k>yk5ELe|0wl+AEL<6Jxpr@RUVmGx19jq!El70qc zhA1191MSiAc#cT#c#35js+2ATl$~&P8NLxYZ2ePQ!8ZZsqjy>0fBqhJJ2a7dD(-_pfbfbd^oHhQ_$45rOM|)k zrn{%_oPO`@w?@i!EaU9@?KdW>w&^iN19v_1tre(gXV;bq%HMr!R!YCyvT3c3&utBx zHb=Vh_`}N6naa}-E@vyf8@9pU*ebpg&ckHhey+EN`b)aEUHkJxy>87%b~BVea+`Y( z>px9Zz`@<&BCvER z{4HvV*X9}gnr-~rUHJ(Nt4C~6w=4fPI9}9CdWl|u3$lcbeQWbS6z z%kma>2C>h%>7I@SZ?K%s*uTO>IEclv|Z?oqqHYiGF$sk!jKoEz3z&1~L z%0^{lU+=hgVAv-cL7PGeDcc|u^m|7xUg-6EW#i>B-+8F_do;2Uln^eV$U+c=!Q@rY zBkj@2M3UzeIV^Z+{uuNxTdqJD85tNjcW!W@%|FEvWi2mLnfRKlCsWy51p+UXX|?GL0{JSLh4}7Rlhow zakZol>mU-oqN)aBU$HI+CByYxvE#Xm%AT2_hy`X z*M_pt)5{rW$9?*~2BZUYVt2uc-?;Ne#(s#j9R9ZxS*Jg3_dl|i-@bhF^6fWozPUQI zVc-AA?tEx(&e)sRy4MHqU;fm7W=l_%wLF2{qayjMfq4pbVJts+`)QchsGtG5$PYmN zqK?(<1bz}&P!q*I^ujh?Z8u31l(Kyp##X>3(za#bthrlp4%kTd(_VgX9M-cTfbAZ zQcw9Pe*{{{bcd`3O!*4^9^bW&gklOdB1!^!@FIiGf_7#E@iojbMG&^|C!oo~o{Ip3 z0OACVdzb;>X4>1^8Pqy0JPJX;tWX$ikOgzw0@F6jv|VF7dg3-fTqVk;E)kiTl`ROA zAwb1Rfk$Ixp=l&42#CmWP7JldL_$QN6&S#HwA6bTfGkK6^{`OJfWaKkw`sC?DrGI` zrwWYgrjyZxY>7gDu&xBaHUV)h;3eE}ALRY2NwI9J@u?v;k+ zhP9Rr$Du8Qwelr;v+=-sb++-yQeUo-xf{C^doQ_kF4x$yb|VACbDQ-|>l3-dow?Q{ zxsIOP;jVAYx;@>?ecx15mF`tL^xoOG;popfs=hu2OZn|qs|Gs#+_itxRhw=&`(W66J#bbZDb-eS`w3h>f_R%&PKyBGO86eAQqwxMu9zws z`(g<&?_`ilB>;=yyhOk8htGs(&G+D0TLjQu;8|C~S%hZ*Pjxi_u!bGMTF)9wpwsva zE+gpdrisFQK&yryxb!TniM5i{0BCP|K~b`^F8Eck)%V=0Fr(p2--?xpvsx5SY~9ZaXN7PqmT=VX z$nnPovDd1OyiM%YOn0dCVp>!n)On*ggc<#9ULfvK1S0=3wBdaiDe{eYcZp<*pmyI% zBoD$oNF06)`7UBaBprQ3raNW&n5^r&;`e&2as^sGHiE3^6i;gk zV81!tD~Fa3tzOAg9srBbV!M6t=E0@z4NE;3SVb+OX4Uu8#%F2~UHAEoveT-<(7oG}1(`_<{@6Iu7^)WnvFYU|3i9?cy%lxsVZYd)Aeco_eh_ve_sxu$)& z{fEDHf999$9?>W?1gHef8$m z)nj+h-Z}d~_i5P~SW!(q(OY`f`E)awuqRSe>NHNS-$*x~%({D0lhC%(RnT0jHyj?I z)YjiM-Z9=SyHmDq+^9LSrEyj5dDQH=KbY%0{`ZZ)YRq*W&2@G`f&87!b)NX9+_?AD z()>3rs-|v5T9(#^v+nkX?&BHv@%z_vXZ!K*)Y;q%eOqR#=K24jbTwWYw4Cbtbi=@h zuY7p!W8Z>ky|Rl3u4!ymVNT$6s;pRT#|>&jQY{xMC|pBSyJ+Vi=4@1}cC zy5Y>jhJg&U4rd!)+HhZx-SyuQJ@cO%dksSt&A(WhhdT6;QWXXTE+|0MlT8r9@a9QR z!)bUrz?%%b-z)&RMC|E`YSKn_ z77;ohSp%`F!1>4;NzMpv$by;qm+^DAK$OU$$@pM~^i-j&Zv_$<%hvpqGgbyq%R zT-&aE%DDDt@D-GT{CdlF;U^AB!N321z zNEMP@a!%;RDWF#5sb;BMvPqSBsv}P|NoC+zbs&{EE-OcgFqh4Ur{pD$|1lgFuvCrt zmqS@ko(^S``gSOr)W<`)d}90Sp-00bc!CH8$QSrOg*?9k(Sn8a!JIk69OYM$dIzIF z+g{bZU@Ne7cAi|QE|j_$@K}=4nKQ>^!we5k`V+Dd=MfD}f};$EC;>B5-sOR-8nGormWj@K+{?7}2m3#U;o(c*#s)JTWQjMfB3j zx@&}VjgYPpl3oP6HUSJ_BpM(=M*Zv@p+bUGNPsG!pYZYxNbM*%!)DLX{By z9hwR&L_`68S5^@d^~!qveSgMtGF^8n>o}ci*(`C|O?+xU@Cfkx@bcl+x%K`}PKq0q zb1C1KRqj>u@*AnkImj3xa|yZwFCiMtv8=OgeLCYjneykHRX;>Zx4)2azWD!cTjgFk zw|s8(M_E_LeM`o5Ds?FbQ~3c>`bcla)t7n&=B%_$Ki8dg^*rdwxCRT2pG`}jx{z^= zkme5es_ByoCRft{yQ5#9OS_*>mp_ktQMY>X6X(8M&7RfSHF2FwyN{*IkA3N^Rmp4m zb=~@G+I>7-etb(yx%T5^YWWfPK%*Yb;2^v^!s|2MhhR{)1cS=UG~{hTcq|Xa@-@a_ zkWEH{K^`4w#I>$?VmK_mi_xzjdY>X4lLNw|6Ou%zgA3@qd58L1V>DVGSLlt$A6Hn7 zj>nCR@$BOZbc?b2$t(0Z+IX3M@`~n=QJEiU>~SmC1+_kWmk_67;Nd~xM|}T0ib5V` zBaeOpxO<@l8xqK!QoQZxs6RF{-;NLd0*^1zJlax<)5w~ygtvK4(G9IwhYoq-MUt%o zCyLXJKVoe}_5TL**`*PAl7Xmj4H!p#&37LbRo&Y5KpY^3SQ7&nWvBR9A-T n`iydYMwNYQqUh5Z>eLrhUzY0oZzwf=tDDf$wT~%`N!R}e9L8k+ literal 0 HcmV?d00001 diff --git a/modules/__pycache__/setting_env.cpython-312.pyc b/modules/__pycache__/setting_env.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cdae8a3c05d498c778cf7ef198561cf5170ddb77 GIT binary patch literal 2928 zcmb_eOH3PA6rGRlv5iR}2?P(&SSeI75Ysk_s-#sLW1B$ZUu`EORLN*OGr*9s$DJ7< zq(~{RE6J8&>6y8G?9@6=%3?VfzdS3xr6Bq(Y2ZoE#-7;xZvp_Xv?*qb%eo zHA{wFFR+ppER#h^TiNaaVS$i{QBovEDUlqdafVTsNCS1xy25V66sF2jqC9;!?+=+H zrtrFcF%-)3nl6Qm2`QvYf+4GAJ;ayA(70rnM%j=`dMLz$H6+S{VFf$X&L!BDn)P^r z((v0Zz?Uu%RdUj)({x)zBt^>kzAC}h5eK5{-2ilbwgb>E(Jgu&Gi$gC7!YAgAn#v| zOZ1Aq*CL#M2}d+od3ICLCHlpN*CITQ|GiII<_gw=1H> zk$1Rwbj|Ml995;_!~MLay))8s0rI}q)_>DAj$D@Tr9{EgGh~^52!D1pEM@%ML7|x{gAY3gB4kr`nh%S$p z_@c^bJRER(m_*(&UQvp?a2vv6Q7)Ff`T}owQc08HnZ+@!=fu<_l44QYI50z%Ud5fh#0Z*(4N*e7Vd> zo--ZHNG5j$2f0(3cp`~>u^X{uf*Vf9kO%Za^g42<^C^=JCKK0?Czpt)Q?YOm(QrH? z-+(kOm*c!4Atq54EkT_!#$*L7qQG-PWeWKLT2xg<;>*aRDASTw1U{p|!H(QYMNtjp zt!U~vSL8M1HjjHm>xKqa-k9Ljih-CSuS=q=A&+kGMP5OSDi*O91wi27hNvjG3ZJIV z=$z?bj8cWeA-<9-N{X4rC(H_p#K~m=dBA4ycaXcRmrORMNYlupNn*Jqz-{HyC1nhG z1hqUSk0Yv5K{PIeDwT9M^xM3C{u$_>s0bBFEA#>7;yKPm{S z23;w@$Lsx2#*`xI{kmira(SGS%F~g`Eb?+tk8E(<-6s2jj5rp2gX@37-}?pej14XK zEcL96tlr*WFV1(}kL*}v%hI=LV{Sb0tq)ya?Ac&H1z?{81oID{VC#@={f+IdvAwIq zPwxEu-a6a6!REoy3-H4oGrJ;sc1O(aa-4nXCz@IxRKKb&Ufwv`HGlf|kD?oF?0fAg z{cUW8UWtC^`R2-FdNuZhd^EV$Q0qANRI9NuDB|p9TgP(aQsatTYkS+Q*48I$ZSR;x z?)7(smQzcq)kv-5!hF}S{`P0V&gIUf&Xvz=!E+E0arD_*5GUx{Y-(KyJPa(3Y&3QM zhr%M8>>JCGrO0Y~jXiJXwe_|d`>t7-t)Hy1cpm7Dh2N{!@y#u-w%#P#oV+a=UrCg8yP-F(?x1OZ+)#3b;&V~zxS{e`mQh5MDPQe#w}2$ShB zrA5G{od*)8wfBJ`Zx(h`)PSAmOW{3(MIBEEt6jT<5k8KK!xK!0)!MwXT5VGwAoNo} zV7ayg7j8bhIe!KVw;tY_4?k-TE?!t~?ppVEZ3Bczhdp@dm-&*!amdec*7g87z;T~V z@rvzXI8Iarj?)Uc590XWo-oxoyR?o?{UuMV}n6YK^W48-P}%!`Yk4;pcW(d&Ov03qA6N2 zLiGwyMX!RSmAxu>Do50#nqG|p^Hd|+QC+WYG^00zq}3ytqxxPwiEBo(Mzedf6_k=X zNYUDtDO&foQmCcZKyVp=%am~&2u=^UEE$(GrLbjxj!Ydk1+Qku8P2BUH7rNFCVBO! z%i#gE>RH+e4_)_!$92Fn*(cN?K!upNR{@bZ%0*#UdX=;Sx}toU>Q&P!fErp2P&=iu zX?g8Y#znjOeFyXw684WcSk}^N>2z@ujEA+1k!-ieMO(U_I`~Z6vzC4@W8o|f7AMzW z8Min*v}M?35xM&`39Z52tKfH!A&v1XXhku^{FEJ_5CEqXaZ=MMzX~YCHwV;eNMR`? z3?R2Oc9dTUDY-@(`Lq({6t$@C;C1`jx(>8I)yZdeKHJk}KibxP==T(L?N5LD(`Riq z9j|jbSQqVP_)NxohP975xItbsGU;NRymsUaG??ciK2uVEZc*1M>I z!EufB^BQDfnF6Sl$wOcSz+#ig%lxvdX=H+J8r;UX*fY&rn+C`F#$3#3Q{SXxu!$LU zJMA#AO#^OjaKhQt*4O7{=ypf9*E_;CjSUYpH8wVldg+M~7u!S{VRw2P$ENrUyWQ<^ zb9TEgM;N?D34!fk_W<}l6)V~pJP|3XnX#U0h?Q&(WkpIFAl?Xgc+|y;%HUDIW-{X= zW-28}Q^kz+)w+970M#eIWk~4YM@I7RVU*@5P9Ccvd7O|(2!3iPkypgh3LnI1&;DB>2^|Rw`#&!)3~N6-U-QrlowQi!;RY-xk~6PMzLC zQKz8&)7ojBUo(^=(Zrt|4k-zKEv>^gNZfP=rV{dXQV!zj%tTrRb#9lMf*SHkQAyx! zF{PlUkYfEdsH8fiCvq+7JHAvmc_+AT*b_2bFSuFG?j8P8gQzB6VdoWRKF2P9sm2+} zEA)$cUg>4|j8VrhPJNcaN_n++%;n+rZr1H#Iftju#j8i4C_dBWnHY63F!y7>M#oVWAYp$B;QQ2VefDehXX^v>+{o-mXjjB!Tk8H)-x|v;n9H8c z4)#QHtswvn;jTz-^HO0Xch^kSZKEkL5mH5s)|r}BLs78ml6A#UvuZ96?YwEW&E^O4 z0$XEw=DD)jvP=1)?}Ww|x-Oc-)eD>7&0qS?&rFf>maFPW`D35v?YX0(itE-41+Sex zd-_$o7OHr-mA)6q?wVxEyd&l11~+upH*70cu)_Ti|dlY!k2>PpdqSKw2fH z)sWr_X^oiHLfQstotVymba{N?(V38^e_Pxb6%;3Lk?EFHe44?Mleb7{pBUPyfZ}vF zSz_{18j=^1M1@=TP2} z*-Rb0f@5$u-S9xA7Gm^9*~>cr0TM1~z}r#aOm%dE8gmU>WnW ztlK%_vao|*hO?kdK$TKZ8bNcEv_?>_h<5oo%7o8xzu^^@FSV`B&G5%qA^oKag_gR* z>v47Sg{;?(40cq6S^FewN1cq%PDI;GAcPpVhvQZ36w9kYAaa6whT9&W)#-B3pg(tl zx<>>blM8634)ZkJ#282^n zi2SJV5!H{`O$gYDQCqek8$m;ax5H~ln|YO+et}og95xb~z-N;Fkf_$&>Hx{>DNowL#kerzZU0fHcr$>FcqTrM<+kk^Dfe`~;FxNO!wU$#5=zGN% z%A4*zvsE)xyD?7E^0bHGqGyi7<657tr)kg7H$qzg;fi6m!0pp7u_p`yJHnK zn7&o9JyNkfTCww1#hysTo@m7rffKQc>bL3_>cdl!irsni_W*b7v(5{8d%axb8rS|s@UpX9Y+8eEFi&)yORYfcZKT%zO^5d+CrR%2g*jnjE zD7~mSSRQN(oPp9wbYI{FD9cnFtehPPI%kJMW>7y;N)J~@EX_-%h^6I2)zv3Ifb#a= zG`8QVp-h%L4b%qXYbVd147T4eSYn2}*X(EQ%bRvCOy_4k4XZ;#yEkJ>XUpK#?QX|_Z&fcW6UmMhlLE3*Hfu#&wymRG zFl+Z&XyEpES~w=(I;GfW4s;)SwjK7k z6g*UWXy2jdyDfVxbgi?&VYAech}QP6H?Q9yZwvInZ`r&i# zz|Y`pid)vge8Z{eGvK((Ch#zlfhvpJUZM$HY6+D_O1FTS+ZZ#W?OPW!Z$jJmj+QD$ z>(>@Dm&6}igwz(vGA^n_%UDqIMWq(%_tt}8muy~%)}fY>>{Y+Ppi(As3}g~Q~YULhS(c`8_D7m})R6X&(W zkLF}?=MqGMs7V?2dq>A6I2X~ToIWF&6iK+h0~Yp00I;!S8)nK^L1deLa4NX<6+1W% zj$bSdkH5Kj)l~Y%{*dLySC}yXHZ2)bhCu|2fI~pWQy57v zi?d!BNkJt}8c1)Nf%N_mIM`7DNdtK@*mlEEcBwMdf3bck??%NoX($ihG}XdT<`)FD z!SVUbP<1$O!4@&sNAkDK=E2|;*>0Okf^Bc~g{sdF&p#72)g=yHQEAf9CHjZZ-XZI? z0A%GHI6Ot)kP>Wb^b`r6AtkLQDZwwKq_rd^c!iX7#c8_@bnD1aF?k>z31gPOLT?pI=(Ebrz&LbEjZs=i*5zic2 zSPXhW7}O$cIf5R23=us5cnws>x|ruMj<1e*pz+m_h?4H-mHh*}(&6EitQUSx<`nXt zM({iU7JDNaP|_pCCEf1Z&<^jeQF2JP{}lQD4Kzm3yA8&`{#SZ~_rv6T}iZ=G;PPB=e(f{s~gfXEQ& zkt0xAY1swaye-rdKC^6Xx$0Oc*&FBti%meEuMc#_N;Y1onXh@X91fMO^IL9}G(}39 zq9t3`%ULVgbPLSFl7_HzsUljk3u*7em`j56>-rEA?tar3GjF_*IiGpUTpKahhK(y` zaBkg?DZ7w8pM9w-Y`oY56l*|{HJ=sgUNPIia~@K^{z%vurvIrpW~m5mh1Tj|R7kWl zlo8w(+guUaK7S^}%=^N%k5VjdS4zjfjbd(=j<3aA9cGrw zm-dBU2v=NXuWf(-%r)kH-}Tza6P;JnwfSmwqd;*0jB@f%gUC4YbSjofmK=gj7WHY{mARA24>=*0UcKFC?gJ@g>_ zD!3J_I8%RQzr=;x-+5~Bsf#bc>BNB#^vi|&mJR#X#KVM?!mp}7(!Z}?$=#n&a2!`q zNjwo9woJx>TY?io9|9Ud2?DYYk)4a|VXq?2g#c{^!F@Y`F$BoZS^(g2ffkMna_R~44wI&`B9x!U`7_KO%9@Ki@H|_sKkQ6aV2ctA^Z$Ty1;d=Uw-WBH#SLmWP!?+k7r0VtkspjOkq1j70dVA^0??@w zaDwYB;T{{%X+&PxEJqbq#G&ho1iwbc$%Z=6Np;9Q0y+tYE(d{EO-uD=$vC(cDAtiJ zES{ei)r*vx{EM!CtW$2czshrw`= zJm$p6I7EiwQWFS&ZcKQv{x<-S(dM$C8>9-Hi8QR1lm+SYHgY^tvI*Q^c&)GoWRn=J zjYpHP#TGbY##5ELrIxE(ue64^OTL(lte255#*q)BT$9=BmJVKRyYh5ceOaH#o(5Sj zQ)aE1@cJX%dxV>ga0G;@SL+40s=O$n`?$z(;ob|7EgVp2fs-NYTMY{hk;2CCWTbHCioONK>BH|T zR{b^ZDx!@b1&ovVJ_r#a@&u1u9S9>KRt2I(xVxwWhrfc{T_Wy!6iwo-E5OHBieI)b z$>|S>8SsdLn|}&#Dv=l%OUxA1$4mkMdri|n;&pJM(;qfq*a7j_2KKN^K)`D`2Q%P; zt21yL6+|}5Mqx`!cKi$D;wuoxM?Aw3Qd(He>~Ua36C^Lt{mQ8z7vf&`$8t@(>ES{~u6?tWYGCCaVug*!AlGZ@oBwK|O=B6=^nO z3J$pv^58r(A*F$~$!bWvK3YvY*`J{j=YzM+$qES2qv%cv(vFF1B#Aan*)lDzB`vh= z{u)s8P6;yC*R28Xcivw^nwzrUxP}aPqxaW<`*}+E#x-Pe4rN(4)|ciF#sRPkd4XWW-Wcf zVKQxTPFc>l+yjFg7V5_kaba`X*aK&x{xleoWmfF2TDB0b}EOzy#t zf@&G>cwODLI`0A1g|4Fj{q+}G5?5Q7Qtl%KSb~&1V2Xb69l*mGfyGFU4jUCxQ*%Ih@C24 z)@0s+EM5igk5}VM!RRHh}HlEJSN6Tg5Lpq)u12T%B_gxR)pH$dV1mMCF^^2SL&{*f0^}*tn2;J+~YHq zi4=zy4g*E~mHI>qYa+!rEn^LCH-LBy8H4*{iBTdn0oyLhPmHd?%|`cGcd+dhFJ6#r zn;m##0`4XDEcBdzDPn4TR|7Xh-B;Yv=DiWq-WfWE$K@i;a|_R1kLcWB3a zZ@4N_*0cnmC2D#E+fW>A4{Z+@gt_Iy9m^Yb+}@DCT(Ey-Lp$7YT}O=5AddfEjKeBP z2q4(j7qRLT5t}SjlLao7?r2^RssRNvD2Y|5Grh1+qZpv7isBWoBnjdsz32w)dfZw` z90Hx}V~D?ohj8mSbghtp$Nj|IOZuV=4&d8u`NvdZZr6sAJ?Bp__9`$|okmK-pTaxNY z^Zupsr6=hR`G6)~2Go;rUp?f*nmA2TJ!$5HRKD~i^^i|j;FF!S7Zm6`+6Y%7Pyj&y z7l5E@Wb~_(6pehTM51UEI1VH$8bn%ybEl4Om98p-PGJ%5v*~47E<})StTpl5-Sl*f zv(ADaW=weSzO$f;;OD&pwS|1@OZ1ZXCmyL)F;2%M6d`LlqB4}GutY_HG?Q`eVV!hS zE$bYs6TDrvbft?LkCSLTW8+Qen5$<5m1T^Rl7fx`9y)?ZEQpww9n8<*<;NA0ZeJz? zUN~q+tHr0TYTVK9%d=SgkVem%l%dxH(YQdAm@T46HhlbyWh7>cs4SB2m>fvWhYkRW zxF*2QN32=q95m0@kg6rQJsz*14yI9(jDnJc=W5Ia1Vl*+VQc}xn+PrSp?7J=d#zVmuUUUt|BL$NqsOCp&w&E; z^+--GoP<&yz4B;MDONbEgHxAvRh%{WO5?=Y6T$KyJRi$91r@Wqg6$U$&L4!&UZxhN zA{(0{`P*i?*Uc&l=FPSM*P;1CNnFQP3-BVSb-n=<6F3$u*gDfKoy6d09Wis|rTt;; zQvPCg)Vecb-nqQ{aKwE0ll?14yOz73yVZR%(tR>|^i=f7>4^FC9ZKO;SihhY#nx48 zjm-9iN>RKkw)-$Xk0@4;JbNGW^O$8<+))yKm9p4+oA4Jb8lu)+UucxYEo;CJ&m)T4 zasE@0?o-jDr=v%n$1LDiE3tl!y4d=M463RYS|FB?sV;5-56L4(6kkB}hyt>oQ5;oB z4LKd@J{>*!eDsJNv%jkVQm3My(Do|=nmc+buaF#fnZrAls#bCzSp&Uof5;k!&yyOL z4UM-A#^r*2D~9%1fthG+9kz_r6R8q;ViS|%BTW7ZfeS(WFzPi-BKR-(v1sPLOx;!H z>&osnz-KFWEA+aOyCpTc-S>JG)zTL%NN&qzo`#3$VH|`1hu~nz!9M)M2KeUy#PHzt z@KH4fACSSvB@C#L!WUz_%1b*L)SdV&u=F_CgY7QQB!kXp;ZvFcjFE;VHdpLwY&N>(k|?- z_;?Z{S8T@X?Is!~)Qp?N%+%9R;u$k(a0xCQS$r1iHdfoj#hoS*IS7Y|DH>sUyb2lwqun z)v$T+dna4QWt~OVwA{p}1&JVI3MXJ0bK&p&8jb=~c!uDK5Y2GF7*g9 zl9#Ydm|=4O;@^2$RWn>6R5q)Y8F{_tbso-k-;=J)1jlyu$N~?6>c)S&2m;AIqZnKkZrhDL~7R zb~F58I{DUtuhx5idbR99|Hl`1ZoBi%Z?(QFIMaV`1w*8#=c0W@PmuY5jr=6=t?Zn& zp`*VF{F&2gASh-c;7vB=uC}579q3s36SJW=0)09BnbTz;SlKyXL;sQu{Q~$|&TeB* zS1BM-OfPf#t!&UlKMEo@vxV%t3xw~|r2Kp-e^7`s76^3r$D)BmG8j(= z0xaNbZw`bb@kn=XA{mLdH`n*YqLKDsXJ16{tT}HyZ5geImkwow4{@u+`la ziznkjv>wS0o+?^#tR=5R^4dgtP)-aDl}m!Eh+H%VajTBNDZ;+8a$q zx~*-ZAuAQl5gij;Kyl%Lh4Zo)orYE~(CgFD!BVI%mWZ&v-cAaFI}vkNBcUI24{%l} z6i6Vp7O|ZJ!Ek>t*@J{CB!q-uAQBEHgV1)JiG*l67KlW{tjX`&R3BL9UI%8I8e7}i z+dM6S8f@wkm=Q|&KZlkpf!U%+L%R6)BJV?tDWEs7v=^QfrR>K*!-NuX$w$2B;{)^1 z`hQA3&F?fXGuhw9PD_!SYMMuZEl+sEaot^>6tJ2Z+&&e7co$8G8bvn8X0g{=jqsx$S`W791A*YC+ zZ92NXUfXr_n{@I$I{IQAeTR;Y&lgI2baWRH0lx40^e5`j)iO2XeCl3>si~1<$<&xS zbf)Yymzt_R0OYC9ehYqGchx}-@gC$A#-|{C_8G)6h}<=x}m;$qf|{#sITT9Q61;hf4z~s1q^tlfwHObF4tWRMSYC-VC`{Wvw_cbs6(~O zF{`HEPA&q|*Y8$X{tj>}ABLZE&MYt;k3-p7`KVvm4a^SEQJ=c1K%~|5sr>JtD(U0n z2r?-S(xX2@$Dw8lJkZp1Y7&^a6uU7swF~IE)CAyzJof?Q=5kq1RMWZqPe4q~925q# z+mppIRNbS!1#q=D1uP|0J|qJ#sp)fU{nFXehQ3H z!FXCQhNXsRGLV4A8-X&JtHhB@oq=3ESK~x%oWDnhGAWE{hNVVa``-{|Zx?2ZRa&jt z8ss^&B|8EZfVN&knB}HVR&X=bZf+)*-;S&dZ%C5Bajm9#4=K?p;&6i@bCXoOmn(Ku zfHIG!Ffa8M==4MYt1rQfg}z+-jphrY|2a;|TWxLzKC$tdN4*$?`}GceZ~qEV|w z18rGOY>S0s71o{c*hf4dxkXgPY0a1Hvbp=5PqEO>?~UA#W_3<~S?3HqG%p!L@0QH0Nru zc!#6TWw1mvGzToo=sBuY-SixNUX-(1*WGjvIFI12wC3I|xO1$zPQhi?++TgbV^e3% zy&$*`TXVn9xmxZ0i0XU79b!ga1XH{r9Qarg%ZfoQ9H!nmQZ!9HVLvyM%XbK$Y$Em1 z`@B9jk$O{bZ6fuG;Mzp$JA!KyseOWL6R9BQYLOa6G?O|EVo=eDf#97M$fU6IbAek$LC>SWCm`iKddTV}j9b;I@;a;B$;$U8TrYZC+0UN`n~>7UPY_V{v7T(isdZ;Ay;fj}jf| z?~KHW5sf94U6FWFf%g+aA{0sJnPGUD67TI)I`=BOgS#S1l$dBT9O+W}KtSn>M7xtc zde&N?hI{)X(L`^|ptCLtdS*r_20>2%y$jx}keS`_ShQOS^+ZBnQFa3#8X40UJZ(H8 z8``?+G!dJxqa%E}?ZFF9pn%WPL z&Ws&Tv@yq?a>ueo^U8*yC#?Y9o$xnNn3@^~$QGA3F0R<@TDZGxkUg^Wq5IaaSdL_r zX@I{V*yZn}Jd4Y}UQ&P4+>&9i#MnyUVR>%=pTFZG{+4f+eq&D7`4@AC=9PYP36}-2 z*)$u0*$B)=U^W7?5txm@Yy@T_FdKo{2+T%cHUhH|`2QUNc!h?^{M{TL+F-(O@-PuT zWWuv1{I5(ne)ES3-WBt-1@_*U@CYAM18&qI;lVa0`pyrJqcKTwwp5^uzW0-A8vnx^ zQ!x~okmbSYGEd4J0r71Drj>9ghDlbNy)EVOgBMJAq=^a7CU}zHt4TFhBjxFQVxAPc zUBU`UAo>njFxBITl%sF-s603`<^ESmc{p<9IF$>h&m5mC`S3cPq+?*A3gzMD4{ z_;p#)zm)KABz#%Ia}vHG;bjT0OE^~s<~9lMk?;Wt8zpS2uivPwgfG@tDK+kDca^fP zYTeqZ>b3VPE88MrMGYneea*U>Rp=s`$5$`dk()d#9P5K?wYv&*XTefN`qd?&l??S^ zzwDG&GE@eACv7FON$L~)LuJUHvjYlKh$*5l&>2n#42P1tRqj6=VmoXZDM*4h4^Nx|efbEbmGSU}neZxp!q|H}G z`b}(`MI1}nV(tEiQQpP2%X=Y4dbu|L8R<*3dCy3{nbCL6M*64Vo{OR<@q8(qs?zt* zdS)qucXD!`GtzIHc0U05rHp1fy)@D*;GV&D0}-doIqcmj0}&8>9+bd60~!~^$8U&{ zJ}AdI(LW>U^qe9(e!Ap<`wfHAe>pv~Moym|w|;3ydjLp7d%67d_#XngmH&42dV$+< zv8xvCYh~vLy!`3?1mEYCK$K{1;bQRf8vHpd+Ut^}QyfVC*PK4RpM1jU)B8y&RIG$e zA20KPj$Z%}-52W&_65SpSUeF34(wr}SpSZ`2)tu+ zKe)bXy|obD-{}nmKQNGc(X>kqf=L>yY^zf+CyCgTG3P!8VIM%>-e0e9y>Z(n##Zi_jB+Lc>3kmmk zxJQC%iYHoM5?mqa1mrf3Q09gleAgKxeOiYStdWwmKQb@#kdJ#l(x-hP!AW?IV8U`S zOZv25s{jeHoo|r5 z+5cyxKD8h19|+=w1%0CU6YT(GP=xgD?w6Cix%|T>ecF!@9418x&E+2jI0D=Q4GIWA=S6@EutHvuu35o#PFX",param , ", possible short names : t , bt , q and ke" ) + sys.exit () + for dt in self.dates: + prows= Odb.ReadOdbRows (self.basedir, self.rows_path , dt , "predef") + if isinstance( prows , tuple ): + if isinstance( prows[idx], pd.Series): + pred_rows = prows[idx].dropna().to_list() # Clean NaN and convert to list + self.psigma= self.psigma +pred_rows + + # Get mean for each datetime + if len( self.psigma) != 0: + nobs_p = len( self.psigma ) + dte = dt[0:8] + hh = dt[8:10] + dh = dte+" "+hh+":00:00" + means.append( dh+" "+str( mean(self.psigma))+" "+str(nobs_p) ) + else: + nobs_p = 0 + dte= dt[0:8] + hh = dt[8:10] + dh =dte+" "+hh+":00:00" + means.append( dh+" "+"None"+" "+str(nobs_p)) + + + if self.lwrite == True: + self.Write2File ("so_pred_means_vs_date" ,param , means ) + + if len(self.psigma) != 0: + so_pred=sum( self.psigma)/len(self.psigma) + if self.lwrite == True: self.Write2File ("so_pred_mean" ,param ,so_pred ) + else: + so_pred=None + if self.lwrite == True: self.Write2File ("so_pred_mean" ,param ,so_pred ) + return so_pred + + + + + + +class Diag: + """ + Class : + # Returns SIGMAO AND SIGMAB diagnostics using the "obs differences" method + # METHOD : + # HBH^T = d(b,a) * d(b,o)^T + # R = d(o,a) * d(b,o)^T + """ + + def __init__(self ,paths , dates, lverb , lwrite): + self.basedir=paths["BASEDIR"] + self.rows_path = paths["ROWS_PATH"] + self.lverb =lverb + self.lwrite =lwrite + + # INIT LISTS + self.sigb =[] ; + self.sigo =[] ; + self.sigb_out=[] ; + self.sigo_out=[] ; + self.dates =dates ; + self.rabso = 2147483647.0 ; # Missing data + + def Write2File (self , varname , pname , value ): + # OUT PATH ( BASEDIR/out) + os.makedirs(self.basedir+"/out" , exist_ok=True) + file_=self.basedir+"/out/"+varname+"_"+pname + outfile=open(file_ , "w") + if isinstance ( value , list): + for i,j in enumerate (value): + outfile.write( str(i+1)+" "+str(j)+"\n") + outfile.close() + else: + outfile.write(str(value) ) + + + def ComputeSigmab(self,fg_dep , an_dep): + s=0 + if len(fg_dep )!=0 and len(an_dep) !=0: + for i in range(len(fg_dep )): + diff=fg_dep[i] - an_dep[i] + s =s+ diff * fg_dep[i] + sigmab=sqrt(s/len(fg_dep)) + else: + sigmab=None + return sigmab + + + def ComputeSigmao(self,fg_dep , an_dep): + s=0 + if len(fg_dep) != 0 and len(an_dep) != 0: + for i in range(len(fg_dep)): + s=s + (fg_dep[i]*an_dep[i] ) + sigmao = sqrt( s/len(fg_dep)) + else: + sigmao = None + return sigmao + + + # 0 = t, 1=bt, 2=q, 3=ke + def GetSigmaD(self, param): + if param=="t" : idx=0 ; + if param=="bt": idx=1 ; + if param=="q" : idx=2 ; + if param=="ke": idx=3 ; + ncase=[] + + for dt in self.dates: + # Get the pd.Series according to the idx ==> param + fg_rows =Odb.ReadOdbRows (self.basedir ,self.rows_path , dt, "fg_diag") + an_rows =Odb.ReadOdbRows (self.basedir ,self.rows_path , dt, "an_diag") + + # Check if it's pandas Series inside a tuple (as expected ) + if isinstance ( fg_rows , tuple ) and isinstance ( an_rows , tuple ): + if isinstance (fg_rows[idx], pd.Series ) and isinstance (an_rows[idx], pd.Series ) : + fg_rows = fg_rows [idx] + an_rows = an_rows [idx] + + # Concat an and fg + df = pd.concat([fg_rows , an_rows], axis=1) + + # Clean the df ( align the available values without NaN ) + df = df.dropna() + fg = df.iloc[:, 0].to_list() + an = df.iloc[:, 1].to_list() + + so = self.ComputeSigmao( fg, an ) # Observations + sb = self.ComputeSigmab( fg, an ) # Background + if so != None and sb != None: + self.sigo.append(so ) + self.sigb.append(sb ) + ncase.append(len(fg) ) # len(fg)=len(an) + + if self.lwrite==True: + if so !=None and sb !=None : # Available values + dte= dt[0:8] + hh = dt[8:10] + dh =dte+" "+hh+":00:00" + self.sigo_out.append(dh+" "+str(self.ComputeSigmao( fg, an ))+" "+str(len(fg) ) ) + self.sigb_out.append(dh+" "+str(self.ComputeSigmab( fg, an ))+" "+str(len(fg) ) ) + else: + dte= dt[0:8]; hh = dt[8:10] ; dh =dte+" "+hh+":00:00" + self.sigo_out.append(dh+" "+"None"+" "+str(len(fg) ) ) + self.sigb_out.append(dh+" "+"None"+" "+str(len(fg) ) ) + + if self.lwrite==True: + self.Write2File("so_diag_means_vs_date", param , self.sigo_out) + self.Write2File("sb_diag_means_vs_date", param , self.sigb_out) + + # Mean values + if len( self.sigo) !=0 and len(self.sigb ) !=0: + if self.lwrite==True: + self.Write2File("so_diag_mean", param , mean(self.sigo)) + self.Write2File("sb_diag_mean", param , mean(self.sigb)) + return mean(self.sigb) , mean(self.sigo), sum(ncase) + else: + if self.lwrite==True: + self.Write2File("so_diag_mean", param , self.rabso) + self.Write2File("sb_diag_mean", param , self.rabso) + return None , None , 0 + + + + +class Ratios: + def __init__(self,paths, Nobs, rednmc ,so_pred, so_diag , sb_pred, sb_diag, lwrite ): + self.basedir= paths["BASEDIR"] + self.ptot = sum(Nobs) + self.pt = Nobs[0] + self.pbt = Nobs[1] + self.pq = Nobs[2] + self.pke = Nobs[3] + self.so_tp = so_pred[0] ; self.so_td =so_diag[0] + self.so_btp = so_pred[1] ; self.so_btd=so_diag[1] + self.so_qp = so_pred[2] ; self.so_qd =so_diag[2] + self.so_kep = so_pred[3] ; self.so_ked=so_diag[3] + + self.sb_tp = sb_pred[0] ; self.sb_td =sb_diag[0] + self.sb_qp = sb_pred[1] ; self.sb_qd =sb_diag[2] + self.sb_kep = sb_pred[2] ; self.sb_ked=sb_diag[3] + + self.rednmc=rednmc + + self.rabso= 2147483647.0 + self.lwrite=lwrite + + + def Write2File (self , varname , pname , value ): + # OUT PATH ( BASEDIR/out) + os.system( "mkdir -p "+ self.basedir+"/out" ) + file_=self.basedir+"/out/"+varname+"_"+pname + + outfile=open(file_ , "w" ) + if isinstance ( value , list): + for i,j in enumerate (value): + outfile.write( str(i+1)+" "+str(j)+"\n") + outfile.close() + else: + outfile.write(str(value) ) + + + + + + def ComputeRatios(self, pred , diag , rednmc, target ): + if pred != None and diag != None : + if target == "sigmao": + ratio =diag/pred + return ratio + elif target == "sigmab": + ratio =diag/(pred * float(rednmc)) + return ratio + else: + ratio =None + return ratio + + + def RatioSo(self): + """ + The averaged ratio is computed by weighted sum of + the ratio of each observation subset + roav=sqrt(roq**2*float(pq)/float(ptot)+rot**2*float(pt)/float(ptot) + robt**2*float(pbt)/float(ptot)+roke**2*float(pke)/float(ptot)) + + """ + + target="sigmao" + rot = self.ComputeRatios( self.so_tp , self.so_td ,self.rednmc , target ) + robt= self.ComputeRatios( self.so_btp , self.so_btd ,self.rednmc , target ) + roq = self.ComputeRatios( self.so_qp , self.so_qd ,self.rednmc , target ) + roke= self.ComputeRatios( self.so_kep , self.so_ked ,self.rednmc , target ) + + if rot == None : rot =0. + if roq == None : roq =0. + if robt == None : robt=0. + if roke == None : roke=0. + + if self.ptot != 0: + rrot = rot **2* float(self.pt ) /float(self.ptot) + rrobt = robt**2* float(self.pbt) /float(self.ptot) + rroq = roq **2* float(self.pq ) /float(self.ptot) + rroke = roke**2* float(self.pke) /float(self.ptot) + + roav = sqrt( rroq +rrot + rrobt + rroke ) + return (rot , robt , roq , roke ,roav) + else: + print("Number of total observations is equal to zero") + return None + + if self.lwrite==True: + lines= "ro_t : "+str("%.4f" % rot) +" | Nobs_t : "+str(int(self.pt)) +"\n" \ + "ro_bt : "+str("%.4f" % robt) +" | Nobs_bt : "+str(int(self.pbt)) +"\n" \ + "ro_q : "+str("%.4f" % roq) +" | Nobs_q : "+str(int(self.pq) ) +"\n" \ + "ro_ke : "+str("%.4f" % roke) +" | Nobs_ke : "+str(int(self.pke)) +"\n" \ + "\n" \ + "ro_avg : "+str("%.4f" % roav) +" | Nobs_tot : "+str(int(self.ptot)) +"\n" + self.Write2File( "ratios" ,"so" , lines ) + #return rot , robt , roq , roke ,roav + + + + + def RatioSb(self): + """ + sb_*d: diagnosed sigmab + sb_*p: predefined sigmab + rb* : tuning ratio for sigmab for each parameter + rbav : weighted average ratio for sigmab + + rb=sbd/(sbp*rednmc) + rbav=sqrt(rbq**2*pq/ptot+rbt**2*pt/ptot+rbke**2*pke/ptot) + """ + + target="sigmab" + rbt = self.ComputeRatios( self.sb_tp , self.sb_td ,self.rednmc , target ) + rbq = self.ComputeRatios( self.sb_qp , self.sb_qd ,self.rednmc , target ) + rbke = self.ComputeRatios( self.sb_kep , self.sb_ked ,self.rednmc , target ) + + # AVERAGE + if rbt == None : rbt =0. + if rbq == None : rbq =0. + if rbke == None : rbke=0. + + if self.ptot != 0: + rrbt =rbt**2 *(self.pt / self.ptot ) + rrbq =rbq**2 *(self.pq / self.ptot ) + rrbke=rbke**2*(self.pke / self.ptot ) + rbav = sqrt( rrbt + rrbq + rrbke ) + + if self.lwrite == True: + lines= "rb_t : "+str("%.4f" % rbt) +" | Nobs_t : "+str(int(self.pt)) +"\n" \ + "rb_q : "+str("%.4f" % rbq) +" | Nobs_q : "+str(int(self.pq) ) +"\n" \ + "rb_ke : "+str("%.4f" % rbke) +" | Nobs_ke : "+str(int(self.pke)) +"\n" \ + "\n" \ + "rb_avg : "+str("%.4f" % rbav) +" | Nobs_tot : "+str(int(self.ptot)) +"\n" + self.Write2File( "ratios" ,"sb" , lines ) + + return (rbt , rbq , rbke , rbav) + else: + print( "Number of total innovations is equal to zero" ) + return None + From 9413daea5909d24fdba7481613365affd51a614f Mon Sep 17 00:00:00 2001 From: rmibdev Date: Wed, 4 Mar 2026 10:55:51 +0000 Subject: [PATCH 08/80] tuneBR main script --- scripts/tune_br.py | 171 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100755 scripts/tune_br.py diff --git a/scripts/tune_br.py b/scripts/tune_br.py new file mode 100755 index 0000000..0d4ebd2 --- /dev/null +++ b/scripts/tune_br.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 + +# -*- coding: utf-8 -*- +""" +observation and background error tuning +re-implementation in python of the RC-LACE version. +""" + +import os +import sys +sys.path.append("./modules") +import configparser +from datetime import datetime ,timedelta +from statistics import mean +from modules.sigma_bo import Predef, Diag , Ratios +from modules.setting_env import TuneEnv +from modules.odb import Odb +import gsacov + + +StartTime = datetime.now() + +# GET CONFIG FILE AS ARGUMENT +nargv = len(sys.argv) +if nargv > 1 : + ini_file = sys.argv[1] + if not os.path.exists(ini_file) : + print("File " + ini_file + " not found.") + exit(1) +else : + print("You need to provide the config.ini file!\n") + print("Usage:") + print("> python tuneBR.py config.ini\n") + exit(1) + + + +# Parse config file +config=configparser.ConfigParser( +interpolation=configparser.ExtendedInterpolation() +) + +# All items in upper case +config.optionxform = str +config.read(ini_file) + +# Init TuneBR env +env = TuneEnv ( config ) +PathDict, ModelDict = env.__Dicts__() +bdate =env.BeginDate +edate =env.EndDate +cycle_inc=env.cycle_inc + + +# Init GSA fortran routine +statfile =env.stabal +nlev =env.nflev +nsmax =env.nsmax +deltax =env.deltax +lverb =env.llverb +lwrite =env.lwrite +outfile =env.outfile +prog_bar =env.prog_bar +ndca_cpu =env.dca_cpu +rows_path=env.rows_path + + +# Get background standard deviations (Profiles + means ) +g=gsacov.GSA(PathDict , statfile ,nsmax ,nlev , deltax , lverb ,lwrite ) +tsig_ver , sb_pred_t = g.GetSigmaB (2) # Temperature KPAR=2 +qsig_ver , sb_pred_q = g.GetSigmaB (3) # Specific q KPAR=3 +vsig_ver , sb_pred_v = g.GetSigmaB (4) # Vorticity KPAR=4 +dsig_ver , sb_pred_d = g.GetSigmaB (5) # Divergence KPAR=5 +kesig_ver, sb_pred_ke = g.GetSigmaB (999) # UV Components not in stabal file --> set arbitrary unique number 999 +print("Extraction of SIGMA_B values, done !\n") + +# Create datetime list +cdtg=[] +bdate =datetime.strptime( bdate , "%Y%m%d%H") +edate =datetime.strptime( edate , "%Y%m%d%H") +delta =timedelta(hours=int(cycle_inc)) +while bdate <= edate: + strdate=bdate.strftime("%Y%m%d%H") + cdtg.append( strdate ) + bdate += delta + + +# ODB extraction +print( "Proceed to ODB extraction ...!\n") +db=Odb ( PathDict ) +#nslice=env.ntaks # If in parallel (Later !!) + +# Get ODB rows +db.CreateDca ( cdtg , ndca_cpu ) +db.OdbExtract ( cdtg ,rows_path , prog_bar ) + +# Predefined SIGMA_O +print("Compute predefined SIGMA_O ...!\n") +so_pred_t =Predef ( PathDict , cdtg , lverb , lwrite).GetSigmaP ("t" ) +so_pred_bt=Predef ( PathDict , cdtg , lverb , lwrite).GetSigmaP ("bt") +so_pred_q =Predef ( PathDict , cdtg , lverb , lwrite).GetSigmaP ("q" ) +so_pred_ke=Predef ( PathDict , cdtg , lverb , lwrite).GetSigmaP ("ke") + + +# Compute SIGMA_O AND SIGMA_B diagnostics +print("Compute SIGMA_O, SIGMA_B diagnostics ...!\n") +sb_diag_t , so_diag_t , pt =Diag(PathDict ,cdtg, lverb , lwrite).GetSigmaD("t" ) +sb_diag_bt , so_diag_bt, pbt=Diag(PathDict ,cdtg, lverb , lwrite).GetSigmaD("bt") +sb_diag_q , so_diag_q , pq =Diag(PathDict ,cdtg, lverb , lwrite).GetSigmaD("q" ) +sb_diag_ke , so_diag_ke, pke=Diag(PathDict ,cdtg, lverb , lwrite).GetSigmaD("ke") + + +# Use the same notation as in the RC-LACE version +sb_pred=[sb_pred_t,sb_pred_q,sb_pred_ke] # PREDEFINED Sb (sb_bt PREDEFINED DOESN'T EXIST FOR BRIGHTNESS T) +so_pred=[so_pred_t,so_pred_bt,so_pred_q,so_pred_ke] # // So +sb_diag=[sb_diag_t,sb_diag_bt,sb_diag_q,sb_diag_ke] # DIAGNOSED Sb +so_diag=[so_diag_t,so_diag_bt,so_diag_q,so_diag_ke] # // So + +# Total wind obs is : nobs ke/2 +nobs =[ pt , pbt , pq , pke/2. ] + +# OBS Mean +nobs_mean =int(sum(nobs)/len(nobs)) + +# INIT RATIO OBJECT WITH CORRESPONDING PREDEF AND DIAG LISTS +rednmc=env.rednmc +r=Ratios(PathDict, nobs, rednmc , so_pred , so_diag , sb_pred , sb_diag ,lwrite) + +# GET RATIOS +sigo = r.RatioSo() +sigb = r.RatioSb() + +rot , robt , roq , roke,roav = sigo[0],sigo[1],sigo[2],sigo[3],sigo[4] +rbt , rbq , rbke , rbav = sigb[0],sigb[1],sigb[2],sigb[3] + +# Output +text = ( + 60*"-" + "\n" + + "Var | cases | Ratio_o | Ratio_b".center(50, " ") + "| \n" + + 60*"-" + "\n" + + f"t |{str(pt).center(15)}{str(round(rot,5)).center(15)}{str(round(rbt,5)).center(15)} \n" + + f"bt |{str(pbt).center(15)}{str(round(robt,5)).center(15)}{'None'.center(15)} \n" + + f"q |{str(pq).center(15)}{str(round(roq,5)).center(15)}{str(round(rbq,5)).center(15)} \n" + + f"ke |{str(pke).center(15)}{str(round(roke,5)).center(15)}{str(round(rbke,5)).center(15)} \n" + + 60*"-" + "\n" + + f"Mean |{str(nobs_mean).center(15)}{str(round(roav,5)).center(15)}{str(round(rbav,5)).center(15)}\n" + + 60*"-" +) + +# Simple print on screen +print( text) + +# Write in file given in config + date1_date2 +if lwrite == True : + period = str(cdtg[0]) +"_"+ str(cdtg[-1] ) + outfile= "_".join( ( outfile, period)) + dir_ = os.getcwd() + print( f"SIGMA B/O written in file: {dir_}/{outfile} " ) + with open( outfile , "w", encoding="utf-8") as f: + f.write(text) + +# +if lverb == True: + print("\n"+"The different I/O files are written in "+os.getenv("PWD")+"/out") + print( " ") + EndTime = datetime.now() + Duration=EndTime - StartTime + print( " EXECUTION TIME : \n" , Duration ) +# END +quit() + From f12f042f985ef88226447229c80706ec762f27d1 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Wed, 4 Mar 2026 10:56:22 +0000 Subject: [PATCH 09/80] tuneBR config.ini file --- share/tunebr_conf/tunebr_config.ini | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100755 share/tunebr_conf/tunebr_config.ini diff --git a/share/tunebr_conf/tunebr_config.ini b/share/tunebr_conf/tunebr_config.ini new file mode 100755 index 0000000..8a07bd0 --- /dev/null +++ b/share/tunebr_conf/tunebr_config.ini @@ -0,0 +1,48 @@ +# Start and end dates +[DATES] +DATESTART=2024011003 +DATEEND =2024011009 + +[PATHS] +# Basedir +BASEDIR=/scratch/cvah/tunebr + +# ODB path +ODBPATH=${BASEDIR}/odb + +# Workdir +WORKDIR=${BASEDIR}/tmp + +# Temporarly extracted ODB files +ROWS_PATH=${BASEDIR}/odb_rows + +# B matrix file +STATFILE =/hpcperm/cvah/tuneBR_1.1.0/stab/stabcvt + +# The filename where the CCMA is assumed archived in +ODB_TEMPLATE=CCMA_minim_YYYYMMDDHH.tar + + +# Some model setting +[MODEL] +CYCLE_INC=3 +DELTAX =1.3 +NSMAX =287 +NFLEV =87 +REDNMC =1.0 + +[OPTIONS] +# Number of CPUs for DCA files creation +DCA_CPU= 16 +# Progress bar when the ODB rows are fetched ! +PROGBAR=True + +# Sigma B/O will be written in file +OUTFILE=sigmabo_ratios + +# Enable verbosity +LLVERB=False + +# Enable file writing +# The output files of the diffrent steps will be written in $BASEDIR/out +LWRITE=True From d04b5ae9aebabba97eac8f4346fb38386481b1b7 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Wed, 4 Mar 2026 12:20:42 +0000 Subject: [PATCH 10/80] Add __init__.py file and expose clases --- modules/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/CMakeLists.txt b/modules/CMakeLists.txt index a33fefa..f25ec2e 100644 --- a/modules/CMakeLists.txt +++ b/modules/CMakeLists.txt @@ -8,7 +8,7 @@ function (install_modules ) endfunction() # Python modules For tuneBR -install_modules (gsacov.py odb.py setting_env.py sigma_bo.py ) +install_modules (gsacov.py odb.py config_env.py sigma_bo.py __init__.py) install_modules (readgsa.so) From f96cb0a60516549806f6a5cebc3d4fee4a7f7f07 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Wed, 4 Mar 2026 12:21:14 +0000 Subject: [PATCH 11/80] use import readcov rather than import readgsa --- modules/gsacov.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/gsacov.py b/modules/gsacov.py index 0130df0..482f5a6 100755 --- a/modules/gsacov.py +++ b/modules/gsacov.py @@ -4,8 +4,8 @@ from math import sqrt, pi import numpy as np -# Py/Fortran module -import readgsa +# Py/Fortran module (Reading B matrix ) +from .readgsa import readcov @@ -68,7 +68,7 @@ def GetGSA( self, kpar ): if kpar == 2: nlev=self.nlev + 1 else: nlev=self.nlev - cov,kret = readgsa.readcov( nlev, self.nsmax , kpar,self.cfile,self.lverb) + cov,kret = readcov( nlev, self.nsmax , kpar,self.cfile,self.lverb) if kret != 0: print( "FAILED TO READ MATRICES FOR PARAMETER :" , dpar[kpar] ) else: From 543f270ecff98152cdcd0140dd4e7c0e5a02b785 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Wed, 4 Mar 2026 12:22:08 +0000 Subject: [PATCH 12/80] replace ''.join() by Path --- modules/odb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/odb.py b/modules/odb.py index bed1228..ff333a7 100755 --- a/modules/odb.py +++ b/modules/odb.py @@ -13,7 +13,7 @@ # TuneBR modules -from config_env import TuneEnv +from .config_env import TuneEnv class Odb: """ From 2d13951feff38ac5da8a4e3308aa016c1e434245 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Wed, 4 Mar 2026 12:22:47 +0000 Subject: [PATCH 13/80] Fix the import statement for Odb.py and config_env.py --- modules/sigma_bo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/sigma_bo.py b/modules/sigma_bo.py index a4de653..ec3cfbc 100755 --- a/modules/sigma_bo.py +++ b/modules/sigma_bo.py @@ -5,8 +5,8 @@ from math import sqrt from statistics import stdev , mean -from odb import Odb -from config_env import TuneEnv +from .odb import Odb +from .config_env import TuneEnv # BASED ON THE PROGRAM ratio.F90 From f28bfe9ceacda971eb97c7a31486f2ee3ab430cd Mon Sep 17 00:00:00 2001 From: rmibdev Date: Wed, 4 Mar 2026 12:23:34 +0000 Subject: [PATCH 14/80] Locate the main script tunebr.py set the import path --- scripts/tune_br.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/scripts/tune_br.py b/scripts/tune_br.py index 0d4ebd2..778ea58 100755 --- a/scripts/tune_br.py +++ b/scripts/tune_br.py @@ -1,21 +1,19 @@ #!/usr/bin/env python3 - # -*- coding: utf-8 -*- -""" -observation and background error tuning -re-implementation in python of the RC-LACE version. -""" - -import os -import sys -sys.path.append("./modules") + +import os , sys import configparser from datetime import datetime ,timedelta from statistics import mean -from modules.sigma_bo import Predef, Diag , Ratios -from modules.setting_env import TuneEnv -from modules.odb import Odb -import gsacov +from pathlib import Path + +# TuneBR modules +# Insert parent directory to get "modules" directory +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from modules import Predef, Diag , Ratios +from modules import Odb ,TuneEnv +from modules import GSA StartTime = datetime.now() @@ -66,14 +64,14 @@ # Get background standard deviations (Profiles + means ) -g=gsacov.GSA(PathDict , statfile ,nsmax ,nlev , deltax , lverb ,lwrite ) +g=GSA(PathDict , statfile ,nsmax ,nlev , deltax , lverb ,lwrite ) tsig_ver , sb_pred_t = g.GetSigmaB (2) # Temperature KPAR=2 qsig_ver , sb_pred_q = g.GetSigmaB (3) # Specific q KPAR=3 vsig_ver , sb_pred_v = g.GetSigmaB (4) # Vorticity KPAR=4 dsig_ver , sb_pred_d = g.GetSigmaB (5) # Divergence KPAR=5 kesig_ver, sb_pred_ke = g.GetSigmaB (999) # UV Components not in stabal file --> set arbitrary unique number 999 print("Extraction of SIGMA_B values, done !\n") - +quit() # Create datetime list cdtg=[] bdate =datetime.strptime( bdate , "%Y%m%d%H") From cc0f46c5f7fc82a067fd7024a2c77aee03f9d50a Mon Sep 17 00:00:00 2001 From: rmibdev Date: Wed, 4 Mar 2026 12:24:32 +0000 Subject: [PATCH 15/80] Replace path and filename by variables --- src/CMakeLists.txt | 59 +++++++++++++++++----------------------------- 1 file changed, 22 insertions(+), 37 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3d0abff..6176ff1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -30,51 +30,36 @@ set(Python_EXECUTABLE ${VENV_DIR}/bin/python CACHE FILEPATH "" FORCE) find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy) message(STATUS "Python from venv: ${Python_EXECUTABLE}") -# Locate f2py include -execute_process( - COMMAND ${Python_EXECUTABLE} - -c "import numpy.f2py;print(numpy.f2py.get_include())" - OUTPUT_VARIABLE F2PY_INCLUDE - OUTPUT_STRIP_TRAILING_WHITESPACE +# Locate f2py includes +execute_process( COMMAND ${Python_EXECUTABLE} -c "import numpy.f2py;print(numpy.f2py.get_include())" OUTPUT_VARIABLE F2PY_INCLUDE_DIR OUTPUT_STRIP_TRAILING_WHITESPACE) + +# Common variables +set(f2py_module_name "readgsa") +set(fsrc_path "${CMAKE_SOURCE_DIR}/src/readgsa.F90") +set(fortran_src_file "${fsrc_path}" ) +set(f2py_module_c "${f2py_module_name}module.c") + +add_custom_command( OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_c}" COMMAND ${Python_EXECUTABLE} -m "numpy.f2py" "${fortran_src_file}" + -m ${f2py_module_name} + --lower + DEPENDS readgsa.F90 # Fortran source ) -# Generate f2py wrapper -set(f2py_module_name readgsa) -set(FSRC ${CMAKE_SOURCE_DIR}/src/readgsa.F90) - - -add_custom_command( - OUTPUT readgsa - COMMAND ${Python_EXECUTABLE} -m numpy.f2py - ${FSRC} - -m ${f2py_module_name} - --lower - DEPENDS ${FSRC} ) - -# Build the module -Python_add_library(${f2py_module_name} MODULE WITH_SOABI - ${F2PY_INCLUDE_DIR} - ${f2py_module_name} - ${FSRC} +# Set up target +Python_add_library( readgsa MODULE WITH_SOABI + "${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_c}" # Generated + "${F2PY_INCLUDE_DIR}/fortranobject.c" # From NumPy + "${fortran_src_file}" # Fortran source(s) ) # Include -target_include_directories(${f2py_module_name} PRIVATE - ${F2PY_INCLUDE_DIR} - ${Python_NumPy_INCLUDE_DIRS} -) +target_include_directories(${f2py_module_name} PRIVATE ${F2PY_INCLUDE_DIR} ${Python_NumPy_INCLUDE_DIRS}) # Libs -target_link_libraries(${f2py_module_name} PRIVATE - Python::Module - Python::NumPy -) +target_link_libraries(${f2py_module_name} PRIVATE Python::Module Python::NumPy) -# Set the suffix to " .so " -set_target_properties(readgsa PROPERTIES - PREFIX "" - SUFFIX ".so" -) +# Set the suffix to " .so " without machine/system architecture/realease ..etc +set_target_properties(readgsa PROPERTIES PREFIX "" SUFFIX ".so") # Add it to the python modules install(TARGETS readgsa LIBRARY DESTINATION ${CMAKE_SOURCE_DIR}/modules ) From 7661ce6377095f8af9574faf6c1ef805a085eb5c Mon Sep 17 00:00:00 2001 From: rmibdev Date: Wed, 4 Mar 2026 12:25:10 +0000 Subject: [PATCH 16/80] Expose all available classes. To be imported by tunebr main script --- modules/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100755 modules/__init__.py diff --git a/modules/__init__.py b/modules/__init__.py new file mode 100755 index 0000000..928eafb --- /dev/null +++ b/modules/__init__.py @@ -0,0 +1,14 @@ +# -*- coding :utf-8 -*- + + +from modules.sigma_bo import Predef, Diag , Ratios +from modules.config_env import TuneEnv +from modules.odb import Odb +from modules.gsacov import GSA + +__all__ = [ "Predef" , + "Diag" , + "Ratios" , + "TuneEnv", + "Odb" , + "GSA" ] From 2d2119733cf6ec69fe91a288d67e28a49b9fbf16 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Wed, 4 Mar 2026 14:22:43 +0000 Subject: [PATCH 17/80] Clean and replace .join() by Path --- modules/gsacov.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/gsacov.py b/modules/gsacov.py index 482f5a6..cdb988d 100755 --- a/modules/gsacov.py +++ b/modules/gsacov.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -import os -import sys -from math import sqrt, pi -import numpy as np +import os, sys +from pathlib import Path +from math import sqrt, pi +import numpy as np # Py/Fortran module (Reading B matrix ) from .readgsa import readcov @@ -23,6 +23,7 @@ class GSA: 3:"Specific humidity", 4:"Vorticity" , 5:"Divergence" } + def __init__ (self,paths , cfile , nsmax , nflev , deltax , lverb ,lwrite): self.path =paths self.cfile =cfile From 2bb2664e1869567a282454e8531b03885aa1446e Mon Sep 17 00:00:00 2001 From: rmibdev Date: Wed, 4 Mar 2026 14:26:09 +0000 Subject: [PATCH 18/80] Cleaning and comment. Add install statement --- src/CMakeLists.txt | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6176ff1..8aaf107 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,13 +1,18 @@ -# CmakeFile to compile +# CmakeFile to compile readgsa.F90 +# Creates a virtualenv , where f2py compiles the routine +# When the venv is created , the numpy module is installed. +# ==> Eveything is done inside the build directory ! cmake_minimum_required (VERSION 3.18) + project (readgsa LANGUAGES C Fortran) + # Find system python find_package(Python REQUIRED COMPONENTS Interpreter) message(STATUS "System python: ${Python_EXECUTABLE}") -# Create venv inside build tree +# Create venv inside build set(VENV_DIR ${CMAKE_BINARY_DIR}/venv) if(NOT EXISTS ${VENV_DIR}/bin/python) message(STATUS "Creating virtualenv") @@ -23,10 +28,10 @@ if(NOT EXISTS ${VENV_DIR}/bin/python) endif() - # Switch python to venv set(Python_EXECUTABLE ${VENV_DIR}/bin/python CACHE FILEPATH "" FORCE) +# Look for numpy inside venv find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy) message(STATUS "Python from venv: ${Python_EXECUTABLE}") @@ -47,9 +52,9 @@ add_custom_command( OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_c}" COMMAN # Set up target Python_add_library( readgsa MODULE WITH_SOABI - "${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_c}" # Generated - "${F2PY_INCLUDE_DIR}/fortranobject.c" # From NumPy - "${fortran_src_file}" # Fortran source(s) + "${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_c}" # Generated .c module + "${F2PY_INCLUDE_DIR}/fortranobject.c" # From NumPy + "${fortran_src_file}" # Fortran source ) # Include @@ -58,7 +63,7 @@ target_include_directories(${f2py_module_name} PRIVATE ${F2PY_INCLUDE_DIR} # Libs target_link_libraries(${f2py_module_name} PRIVATE Python::Module Python::NumPy) -# Set the suffix to " .so " without machine/system architecture/realease ..etc +# Set the suffix to " .so " without machine_system architecture/realease info ..etc set_target_properties(readgsa PROPERTIES PREFIX "" SUFFIX ".so") # Add it to the python modules From b722eef4a70c42897b3277d8c467751638f6b8a8 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Wed, 4 Mar 2026 14:26:33 +0000 Subject: [PATCH 19/80] simple file rename --- share/tunebr_conf/config.ini | 48 ++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100755 share/tunebr_conf/config.ini diff --git a/share/tunebr_conf/config.ini b/share/tunebr_conf/config.ini new file mode 100755 index 0000000..8a07bd0 --- /dev/null +++ b/share/tunebr_conf/config.ini @@ -0,0 +1,48 @@ +# Start and end dates +[DATES] +DATESTART=2024011003 +DATEEND =2024011009 + +[PATHS] +# Basedir +BASEDIR=/scratch/cvah/tunebr + +# ODB path +ODBPATH=${BASEDIR}/odb + +# Workdir +WORKDIR=${BASEDIR}/tmp + +# Temporarly extracted ODB files +ROWS_PATH=${BASEDIR}/odb_rows + +# B matrix file +STATFILE =/hpcperm/cvah/tuneBR_1.1.0/stab/stabcvt + +# The filename where the CCMA is assumed archived in +ODB_TEMPLATE=CCMA_minim_YYYYMMDDHH.tar + + +# Some model setting +[MODEL] +CYCLE_INC=3 +DELTAX =1.3 +NSMAX =287 +NFLEV =87 +REDNMC =1.0 + +[OPTIONS] +# Number of CPUs for DCA files creation +DCA_CPU= 16 +# Progress bar when the ODB rows are fetched ! +PROGBAR=True + +# Sigma B/O will be written in file +OUTFILE=sigmabo_ratios + +# Enable verbosity +LLVERB=False + +# Enable file writing +# The output files of the diffrent steps will be written in $BASEDIR/out +LWRITE=True From 8f3564e8fe9e6e375480a2ec5c7edebf07fa7f1d Mon Sep 17 00:00:00 2001 From: rmibdev Date: Wed, 4 Mar 2026 14:27:04 +0000 Subject: [PATCH 20/80] Copy the tuneBR config.ini file --- share/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/CMakeLists.txt b/share/CMakeLists.txt index 2171e73..4ca9fbb 100644 --- a/share/CMakeLists.txt +++ b/share/CMakeLists.txt @@ -1,2 +1,2 @@ -install( DIRECTORY levdef DESTINATION share PATTERN "CMakeLists.txt" EXCLUDE ) +install( DIRECTORY levdef DESTINATION share PATTERN "CMakeLists.txt" EXCLUDE ) install( DIRECTORY tunebr_conf DESTINATION share PATTERN "CMakeLists.txt" EXCLUDE ) From 1691e3f4a8296002a9019c57345478e39b306a2f Mon Sep 17 00:00:00 2001 From: rmibdev Date: Wed, 4 Mar 2026 14:31:03 +0000 Subject: [PATCH 21/80] Fix end of file bug. Use JPIM for integers --- src/readgsa.F90 | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/readgsa.F90 b/src/readgsa.F90 index d326b8f..26a00eb 100644 --- a/src/readgsa.F90 +++ b/src/readgsa.F90 @@ -1,7 +1,7 @@ SUBROUTINE READCOV(pcov,kdim,nsmax,kpar,cfile ,llverb,kret) IMPLICIT NONE -! read a set of nonsep matrices for the requested parameter -! from a GSA file +! read a set of nonsep matrices for the requested parameter +! from a GSA file ! F. Bouttier Oct 96 ! input : cfile = GSAfilename ! kpar = GSA parameter selector (IPAR coding) @@ -14,34 +14,33 @@ SUBROUTINE READCOV(pcov,kdim,nsmax,kpar,cfile ,llverb,kret) ! USE AS PYTHON EXTENSION : I. Dehmous 2023/10/03 -! - INTEGER, PARAMETER :: jplun=67 - INTEGER, PARAMETER :: JPRB = SELECTED_REAL_KIND(13,300) + INTEGER, PARAMETER :: jplun=67 + INTEGER, PARAMETER :: JPRB = SELECTED_REAL_KIND(13,300) INTEGER, PARAMETER :: JPIM = SELECTED_INT_KIND(9) INTEGER(KIND=JPIM),INTENT(IN ) :: kdim,nsmax ,kpar INTEGER(KIND=JPIM),INTENT(OUT) :: kret INTEGER(KIND=JPIM) :: idim1,idim2,ipar1,ipar2,icor1,icor2, icheck REAL(KIND=JPRB) ,INTENT(OUT) :: pcov(kdim,kdim,0:nsmax) CHARACTER(LEN=*) :: cfile - CHARACTER :: clid*8,clcom*70 + CHARACTER(LEN=8) :: clid + CHARACTER(LEN=70) :: clcom LOGICAL ,INTENT(IN) :: llverb LOGICAL :: llfound - INTEGER(KIND=JPRB) :: iorig,idate,itime,inbset - INTEGER(KIND=JPRB) :: jset , jj , jn , jk - INTEGER(KIND=JPRB) :: inbmat,iweight,itypmat,isetdist,ilendef + INTEGER(KIND=JPIM) :: iorig,idate,itime,inbset + INTEGER(KIND=JPIM) :: jset , jj , jn , jk + INTEGER(KIND=JPIM) :: inbmat,iweight,itypmat,isetdist,ilendef ! 1.OPEN FILE AND READ GSA HEADER ! THE FILE IS WRITTEN IN BIG ENDIAN MODE -! AVOID COMPILER OPTIONS ! - OPEN(jplun,FILE=cfile,FORM='unformatted',STATUS='old',& - & ACCESS='sequential',convert='BIG_ENDIAN') - +! This option is added also in compiler options anyway + OPEN(jplun,FILE=cfile,FORM='unformatted',STATUS='old',& + & ACCESS='sequential' ,convert='BIG_ENDIAN') IF (llverb) THEN - WRITE(*,*) 'SEARCHING FOR PARAMETER:',kpar, & - & 'IN GSA FILE ',cfile,'...' + WRITE(*,*) 'Searching for parameter :',kpar, & + & 'in GSA file: ',cfile,'...' ENDIF READ(jplun) clid @@ -60,7 +59,7 @@ SUBROUTINE READCOV(pcov,kdim,nsmax,kpar,cfile ,llverb,kret) DO jset=1,inbset READ(jplun) inbmat,iweight,itypmat,isetdist,ilendef !IF (llverb) THEN - WRITE(*,*) 'SCANNING SET No...',jset + WRITE(*,*) 'Scanning set No...',jset !ENDIF IF (llverb) THEN WRITE(*,*) ' ',inbmat,iweight,itypmat,isetdist,ilendef @@ -70,13 +69,14 @@ SUBROUTINE READCOV(pcov,kdim,nsmax,kpar,cfile ,llverb,kret) WRITE(*,*) ' ',idim1,idim2,ipar1,ipar2,icor1,icor2 ENDIF IF ((ipar1==kpar).AND.(ipar2==kpar)) llfound=.true. + ! skip coordinate records READ(jplun) READ(jplun) IF (llfound) THEN IF (llverb ) THEN - WRITE(*,*) 'FOUND PARAM :', kpar ,' IN THE SET :', jset - WRITE(*,*) 'READING DATA ...' + WRITE(*,*) 'Found param :', kpar ,' In the set No :', jset + WRITE(*,*) 'Reading data ...' ENDIF ! THE OLD ABORT ROUTINE CAUSES PYTHON CRASH @@ -87,6 +87,7 @@ SUBROUTINE READCOV(pcov,kdim,nsmax,kpar,cfile ,llverb,kret) IF (idim1/=kdim )call abor1('Error - bad 1st dimension') IF (idim2/=kdim )call abor1('Error - bad 2nd dimension') IF (icor1/=icor2 )call abor1('Error - coords differ') + DO jn=0,nsmax READ(jplun) READ(jplun) ((pcov(jj,jk,jn),jk=1,kdim),jj=1,kdim),icheck @@ -105,16 +106,17 @@ SUBROUTINE READCOV(pcov,kdim,nsmax,kpar,cfile ,llverb,kret) IF (llfound) THEN kret=0 ELSE - WRITE(*,*) 'WARNING : COULD NOT FIND MATRIX SET' + WRITE(*,*) 'WARNING : Could not find matrix set' pcov(:,:,:)=0. kret=1 ENDIF RETURN END SUBROUTINE READCOV -! ------------------------------------------------------------------ +!-------------------------abor1 ------------------------- SUBROUTINE abor1(cdmess) character(len=*) :: cdmess WRITE(*,*) cdmess !call exit(1) END +! Finish From b06d0a0dd91c9c8c29c185628d277eefe742ac56 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Wed, 11 Mar 2026 21:40:57 +0000 Subject: [PATCH 22/80] Privide .ini file --- scripts/tune_br.py | 2 +- share/tunebr_conf/tunebr_config.ini | 48 ----------------------------- 2 files changed, 1 insertion(+), 49 deletions(-) delete mode 100755 share/tunebr_conf/tunebr_config.ini diff --git a/scripts/tune_br.py b/scripts/tune_br.py index 778ea58..b200954 100755 --- a/scripts/tune_br.py +++ b/scripts/tune_br.py @@ -28,7 +28,7 @@ else : print("You need to provide the config.ini file!\n") print("Usage:") - print("> python tuneBR.py config.ini\n") + print("> python tune_br.py config.ini\n") exit(1) diff --git a/share/tunebr_conf/tunebr_config.ini b/share/tunebr_conf/tunebr_config.ini deleted file mode 100755 index 8a07bd0..0000000 --- a/share/tunebr_conf/tunebr_config.ini +++ /dev/null @@ -1,48 +0,0 @@ -# Start and end dates -[DATES] -DATESTART=2024011003 -DATEEND =2024011009 - -[PATHS] -# Basedir -BASEDIR=/scratch/cvah/tunebr - -# ODB path -ODBPATH=${BASEDIR}/odb - -# Workdir -WORKDIR=${BASEDIR}/tmp - -# Temporarly extracted ODB files -ROWS_PATH=${BASEDIR}/odb_rows - -# B matrix file -STATFILE =/hpcperm/cvah/tuneBR_1.1.0/stab/stabcvt - -# The filename where the CCMA is assumed archived in -ODB_TEMPLATE=CCMA_minim_YYYYMMDDHH.tar - - -# Some model setting -[MODEL] -CYCLE_INC=3 -DELTAX =1.3 -NSMAX =287 -NFLEV =87 -REDNMC =1.0 - -[OPTIONS] -# Number of CPUs for DCA files creation -DCA_CPU= 16 -# Progress bar when the ODB rows are fetched ! -PROGBAR=True - -# Sigma B/O will be written in file -OUTFILE=sigmabo_ratios - -# Enable verbosity -LLVERB=False - -# Enable file writing -# The output files of the diffrent steps will be written in $BASEDIR/out -LWRITE=True From 121a8782434711c755c794fde8337d1285039666 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Sun, 15 Mar 2026 17:03:28 +0000 Subject: [PATCH 23/80] Adapt to odb4py --- scripts/tune_br.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/tune_br.py b/scripts/tune_br.py index b200954..8cb273a 100755 --- a/scripts/tune_br.py +++ b/scripts/tune_br.py @@ -3,9 +3,9 @@ import os , sys import configparser -from datetime import datetime ,timedelta -from statistics import mean -from pathlib import Path +from datetime import datetime ,timedelta +from statistics import mean +from pathlib import Path # TuneBR modules # Insert parent directory to get "modules" directory @@ -15,7 +15,7 @@ from modules import Odb ,TuneEnv from modules import GSA - +# Start StartTime = datetime.now() # GET CONFIG FILE AS ARGUMENT From 76be2ccc233b63ad52bf2e18263f496cdfc5e295 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Sun, 15 Mar 2026 17:03:41 +0000 Subject: [PATCH 24/80] Adapt to odb4py --- modules/odb.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/modules/odb.py b/modules/odb.py index ff333a7..07a8a33 100755 --- a/modules/odb.py +++ b/modules/odb.py @@ -8,8 +8,8 @@ # odb4py -from odb4py.utils import OdbEnv , StringParser , OdbObject -from odb4py.core import odbDict , odbConnect , odbClose , odbDca +from odb4py.utils import SqlParser , OdbObject +from odb4py.core import odb_open , odb_dca ,odb_dict # TuneBR modules @@ -107,15 +107,18 @@ def CreateDca ( self, dates , ncpu ): if os.path.isdir ( dbpath ): # It can hapen to have CCMA directory but a missing CCMA.dd , .sch or .desc # Better use try + conn = odb_open ( dbpath ) try: db = OdbObject (dbpath) - db_attrs= db.GetAttrib () + db_attrs= db.get_attrib () tab_list= db_attrs["tables"] - ic = odbDca (odbdir =dbpath , - dbtype ="CCMA" , - ncpu = NCPU , + ic = conn.odb_dca(database =dbpath , + dbtype ="CCMA" , + ncpu = NCPU , extra_args ="-z -u -q ", # Means Update if existe,remove empty files and run in quite mode tables =tab_list ) # Only the tables found in the CCMA + conn.odb_close() + except: FileNotFoundError print( f"Missing meta-data files in ODB : {dbpath}" ) @@ -175,8 +178,9 @@ def OdbExtract (self, dates, rows_path, progress=False , verbose =False ): # Skip the datetime if the odb rows are already there ! if not os.path.isfile (outfile): try: - # odbDict method throws an exception if no rows returned - data_dict =odbDict (database=dbpath , + # odb_dict method throws an exception if no rows returned + conn =odb_open( dbpath ) + data_dict =conn.odb_dict (database=dbpath, sql_query =self.sql_query , nfunc =0 , fmt_float =10 , @@ -191,6 +195,7 @@ def OdbExtract (self, dates, rows_path, progress=False , verbose =False ): index= False , sep="," , decimal='.' ) + conn.odb_close() except: RuntimeError print("No data returned for ODB {} \n".format( dt ) ) From 6b3ddbfdd1782855848114199e9e391f04255273 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Sun, 15 Mar 2026 17:46:16 +0000 Subject: [PATCH 25/80] Adapte to odb4py --- modules/odb.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/odb.py b/modules/odb.py index 07a8a33..1648e45 100755 --- a/modules/odb.py +++ b/modules/odb.py @@ -78,6 +78,7 @@ def CopyOdb (self , dt ): raise Exception ("Supported formats for archive unpacking :", ext_list) # Supports: zip, tar, gztar, bztar, xztar or zstdtar. if os.path.isfile(tarfile ): + print( "Decompress the ODB archive ...", tarfile ) # Unpack in tmp dir shutil.unpack_archive(tarfile, dtedir , arc_ext) @@ -85,6 +86,7 @@ def CopyOdb (self , dt ): + def CreateDca ( self, dates , ncpu ): NCPU = int(ncpu ) basedir =self.paths["BASEDIR"] From 6ad4416a1b3a1eee0d3d13189205c679ae2bc8b2 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Sun, 15 Mar 2026 17:46:33 +0000 Subject: [PATCH 26/80] Use argparse --- scripts/tune_br.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/scripts/tune_br.py b/scripts/tune_br.py index 8cb273a..5d360ed 100755 --- a/scripts/tune_br.py +++ b/scripts/tune_br.py @@ -3,6 +3,7 @@ import os , sys import configparser +import argparse from datetime import datetime ,timedelta from statistics import mean from pathlib import Path @@ -15,11 +16,30 @@ from modules import Odb ,TuneEnv from modules import GSA + + +# Get args +parser = argparse.ArgumentParser( description=" ") +parser.add_argument("--config_file" ,required=True ,type=str ,default="../share/tunebr_conf/config.ini",choices=None ,help="Path to tuneBR config file" ) + +if len(sys.argv) == 1: + parser.print_help() + sys.exit(1) + +# Check config filename +args=parser.parse_args() +ini_file=args.config_file +if not os.path.isfile(ini_file): + print("The given .ini file {} not found !\n".format(ini_file )) + print("Usage:") + print("> python tune_br.py config.ini\n") + sys.exit(1) + # Start StartTime = datetime.now() # GET CONFIG FILE AS ARGUMENT -nargv = len(sys.argv) +"""nargv = len(sys.argv) if nargv > 1 : ini_file = sys.argv[1] if not os.path.exists(ini_file) : @@ -29,14 +49,12 @@ print("You need to provide the config.ini file!\n") print("Usage:") print("> python tune_br.py config.ini\n") - exit(1) - + exit(1)""" # Parse config file config=configparser.ConfigParser( -interpolation=configparser.ExtendedInterpolation() -) +interpolation=configparser.ExtendedInterpolation() ) # All items in upper case config.optionxform = str @@ -71,7 +89,8 @@ dsig_ver , sb_pred_d = g.GetSigmaB (5) # Divergence KPAR=5 kesig_ver, sb_pred_ke = g.GetSigmaB (999) # UV Components not in stabal file --> set arbitrary unique number 999 print("Extraction of SIGMA_B values, done !\n") -quit() + + # Create datetime list cdtg=[] bdate =datetime.strptime( bdate , "%Y%m%d%H") From e7844691144b3984827790802610b5e66de44c61 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Sun, 15 Mar 2026 17:55:38 +0000 Subject: [PATCH 27/80] Add comment for the DCA file --- modules/odb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/odb.py b/modules/odb.py index 1648e45..45c635a 100755 --- a/modules/odb.py +++ b/modules/odb.py @@ -88,6 +88,7 @@ def CopyOdb (self , dt ): def CreateDca ( self, dates , ncpu ): + print("Prepare DCA file for all ODBs ...\n") NCPU = int(ncpu ) basedir =self.paths["BASEDIR"] tmpdir= "/".join((basedir, "tmp" )) @@ -101,7 +102,7 @@ def CreateDca ( self, dates , ncpu ): print("Malformatted start date. Must be YYYYMMDDHH\n") sys.exit(1) - print( "Prepare ODB : Date ... {} \n".format( dt ) ) + #print( "Prepare ODB : Date ... {} \n".format( dt ) ) self.CopyOdb ( dt ) dtedir= "/".join((tmpdir , dt )) # Get attributes and create DCA From b8e0b892865b71ff2a40ff42bc8770d0fc1d2628 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Sun, 15 Mar 2026 17:55:53 +0000 Subject: [PATCH 28/80] Add comment for the DCA files --- modules/odb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/odb.py b/modules/odb.py index 45c635a..033298e 100755 --- a/modules/odb.py +++ b/modules/odb.py @@ -88,7 +88,7 @@ def CopyOdb (self , dt ): def CreateDca ( self, dates , ncpu ): - print("Prepare DCA file for all ODBs ...\n") + print("Prepare DCA files for all ODBs ...\n") NCPU = int(ncpu ) basedir =self.paths["BASEDIR"] tmpdir= "/".join((basedir, "tmp" )) From 26a72a8787bf26e1012deff5e83c23cc1c68db54 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Sun, 15 Mar 2026 17:56:17 +0000 Subject: [PATCH 29/80] Add a check for getargs --- scripts/tune_br.py | 41 +++++++++++++---------------------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/scripts/tune_br.py b/scripts/tune_br.py index 5d360ed..fe0d6c7 100755 --- a/scripts/tune_br.py +++ b/scripts/tune_br.py @@ -18,39 +18,26 @@ -# Get args -parser = argparse.ArgumentParser( description=" ") -parser.add_argument("--config_file" ,required=True ,type=str ,default="../share/tunebr_conf/config.ini",choices=None ,help="Path to tuneBR config file" ) -if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) +# Get the config file +parser = argparse.ArgumentParser() +parser.add_argument( + "config_file", + type=Path, + help="Path to tuneBR config file (.ini file format )" +) + +args = parser.parse_args() + +if not args.config_file.is_file(): + parser.error(f"Config file '{args.config_file}' not found") + -# Check config filename -args=parser.parse_args() ini_file=args.config_file -if not os.path.isfile(ini_file): - print("The given .ini file {} not found !\n".format(ini_file )) - print("Usage:") - print("> python tune_br.py config.ini\n") - sys.exit(1) # Start StartTime = datetime.now() -# GET CONFIG FILE AS ARGUMENT -"""nargv = len(sys.argv) -if nargv > 1 : - ini_file = sys.argv[1] - if not os.path.exists(ini_file) : - print("File " + ini_file + " not found.") - exit(1) -else : - print("You need to provide the config.ini file!\n") - print("Usage:") - print("> python tune_br.py config.ini\n") - exit(1)""" - # Parse config file config=configparser.ConfigParser( @@ -103,9 +90,7 @@ # ODB extraction -print( "Proceed to ODB extraction ...!\n") db=Odb ( PathDict ) -#nslice=env.ntaks # If in parallel (Later !!) # Get ODB rows db.CreateDca ( cdtg , ndca_cpu ) From 03827623c477dd86d458ec0eb31ad2d0d3e45af3 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Sun, 15 Mar 2026 18:01:55 +0000 Subject: [PATCH 30/80] Add tune_br test script --- tests/tools/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tools/CMakeLists.txt b/tests/tools/CMakeLists.txt index 212890c..89da080 100644 --- a/tests/tools/CMakeLists.txt +++ b/tests/tools/CMakeLists.txt @@ -8,6 +8,7 @@ list( APPEND tools_tests_scripts test_plotdiacov_help.sh test_varbcdiagnose_help.sh test_plotvarbcpred_help.sh + test_tune_br_help.sh ) From ba36c702c4a8ff4cdb7b56c295f99ca551c0d127 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Sun, 15 Mar 2026 18:02:16 +0000 Subject: [PATCH 31/80] tune_br bash script test --- tests/tools/test_tune_br_help.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100755 tests/tools/test_tune_br_help.sh diff --git a/tests/tools/test_tune_br_help.sh b/tests/tools/test_tune_br_help.sh new file mode 100755 index 0000000..7de9bb2 --- /dev/null +++ b/tests/tools/test_tune_br_help.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -uex + +# A unique working directory +wd=$(pwd) +test_wd=$(pwd)/test_tune_br_help + +mkdir -p ${test_wd} +cd ${test_wd} + +tune_br.py -h > /dev/null && echo "tune_br help/usage works" + + +# Clean up +cd ${wd} +rm -rf ${test_wd} From 066abe18cebc68c65bc140897dc1433057fa0de2 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Sun, 15 Mar 2026 18:06:24 +0000 Subject: [PATCH 32/80] Move odb to read_odb --- modules/sigma_bo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/sigma_bo.py b/modules/sigma_bo.py index ec3cfbc..7de72f1 100755 --- a/modules/sigma_bo.py +++ b/modules/sigma_bo.py @@ -5,7 +5,7 @@ from math import sqrt from statistics import stdev , mean -from .odb import Odb +from .read_odb import Odb from .config_env import TuneEnv From 95c681ce44b1a24677e928527fed7efe09b85af5 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Sun, 15 Mar 2026 18:06:52 +0000 Subject: [PATCH 33/80] rename odb.py to read_odb.py --- modules/read_odb.py | 256 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 256 insertions(+) create mode 100755 modules/read_odb.py diff --git a/modules/read_odb.py b/modules/read_odb.py new file mode 100755 index 0000000..033298e --- /dev/null +++ b/modules/read_odb.py @@ -0,0 +1,256 @@ +# -*- coding: utf-8 -*- +import os , sys +from pathlib import Path +import shutil +from datetime import date, timedelta , datetime +import pandas as pd +import tarfile + + +# odb4py +from odb4py.utils import SqlParser , OdbObject +from odb4py.core import odb_open , odb_dca ,odb_dict + + +# TuneBR modules +from .config_env import TuneEnv + +class Odb: + """ + Class : Odb , GET THE PATHS FROM TuneEnv OBJECT + PRFORMS A PARALLEL ODB EXRACTION. + CONTAINS : ReadOdbRows Method TO READ + OBS-ERROR , obs-guess DEPARTURES AND obs-analysis + DEPARTURES + Return :Obs_err , obs-fg , obs-an departures + """ + def __init__(self, Paths ): + self.paths =Paths + self.dbtype ="CCMA" # Can be hard coded since it's NOT possible to use ECMA in TuneBR + self.sql_query="SELECT obstype,varno,an_depar,fg_depar,final_obs_error FROM hdr, body, errstat" + self.ext_list=[ item [0] for item in shutil.get_archive_formats() ] + self.rows_path =Path (Paths["ROWS_PATH"] ) + + + def CheckTarball (self, tarpath ): + try: + with tarfile.open(str(tarpath) , 'r') as t: + try: + member = t.getmember(self.dbtype ) + return member.isdir() + except KeyError: + return False + except: + FileNotFoundError + print("WARNING : tar file {} not found. ".format(tarpath)) + pass + + + + def CopyOdb (self , dt ): + basedir =Path (self.paths["BASEDIR"] ) + odbpath =Path (self.paths["ODBPATH"] ) + odb_template=Path (self.paths["ODB_TEMPLATE"] ) + tmpdir =Path (self.paths["WORKDIR"] ) + dtedir = tmpdir / dt + + # rreate tmpdir if it doesn't exist + tmpdir.mkdir(parents=True, exist_ok=True) + + # Delete and recreate the date dir + shutil.rmtree(dtedir, ignore_errors=True) + dtedir.mkdir(parents=True) + + # Decompress ODB archive + # (We assume that the CCMA is the first dir inside the tar file , + # if not, an Exception is raised ! ) + FileTemplate=odb_template + filename=str(FileTemplate).replace( "YYYYMMDDHH", dt) + tarfile = odbpath / filename + status = self.CheckTarball( tarfile ) + + if status == False: + raise Exception ("The ODB archive must contain a CCMA directory" ) + + # Which extension .? It is often ".tar" But never know ! + arc_ext =os.path.splitext(tarfile)[1][1:] + if arc_ext not in self.ext_list: + raise Exception ("Supported formats for archive unpacking :", ext_list) # Supports: zip, tar, gztar, bztar, xztar or zstdtar. + + if os.path.isfile(tarfile ): + print( "Decompress the ODB archive ...", tarfile ) + # Unpack in tmp dir + shutil.unpack_archive(tarfile, dtedir , arc_ext) + + + + + + + def CreateDca ( self, dates , ncpu ): + print("Prepare DCA files for all ODBs ...\n") + NCPU = int(ncpu ) + basedir =self.paths["BASEDIR"] + tmpdir= "/".join((basedir, "tmp" )) + # It is possible to have a string , be sure it s a list + if isinstance (dates , str ): + dates = [ dates ] + + # Loop over dates list + for dt in dates: + if len( dt ) != 10: + print("Malformatted start date. Must be YYYYMMDDHH\n") + sys.exit(1) + + #print( "Prepare ODB : Date ... {} \n".format( dt ) ) + self.CopyOdb ( dt ) + dtedir= "/".join((tmpdir , dt )) + # Get attributes and create DCA + dbpath="/".join( (dtedir ,"CCMA" )) + if os.path.isdir ( dbpath ): + # It can hapen to have CCMA directory but a missing CCMA.dd , .sch or .desc + # Better use try + conn = odb_open ( dbpath ) + try: + db = OdbObject (dbpath) + db_attrs= db.get_attrib () + tab_list= db_attrs["tables"] + ic = conn.odb_dca(database =dbpath , + dbtype ="CCMA" , + ncpu = NCPU , + extra_args ="-z -u -q ", # Means Update if existe,remove empty files and run in quite mode + tables =tab_list ) # Only the tables found in the CCMA + conn.odb_close() + + except: + FileNotFoundError + print( f"Missing meta-data files in ODB : {dbpath}" ) + else: + + # Missing odbs + print ("ODB file : " , dbpath , " is missing !\n") + pass + + + + + def OdbExtract (self, dates, rows_path, progress=False , verbose =False ): + if isinstance (dates , str ): + dates = [ dates ] + + # Get Basedir + basedir=self.paths["BASEDIR"] + os.chdir(basedir) + + # Loop over dates list + for dt in dates: + if len( dt) != 10: + print("Malformatted start date. Must be YYYYMMDDHH\n") + sys.exit(1) + + print( "Rows extraction, ODB date : " , dt ) + tmpdir= "/".join( (basedir, "tmp" ) ) + dbpath= "/".join( (tmpdir , dt , "CCMA") ) + + # ODB ENV VARIBALES + # Update the variables. They are visible only inside this scope 'for each datetime' + odb_env={ "ODB_SRCPATH_CCMA" :dbpath , + "ODB_DATAPATH_CCMA":dbpath , + "TO_ODB_ECMWF":"0" , + "ODB_STATIC_LINKING":"1" , + "ODB_CMA":"CCMA" , + "ODB_IO_METHOD":"4" , + "ODB_CTX_DEBUG":"0" , + "VERSION":"1" , + "DEGRE":"1" , + "DIRECT":"0" , + "F_RECLUNIT":"BYTE" } + # Export variables + for k , v in odb_env.items(): os.environ[k] =v + + # Save extracted rows + os.makedirs( self.rows_path , exist_ok=True) + + # prepre a csv file name + filename="_".join( ("odb_rows", dt )) + outfile ="/".join( (rows_path , filename )) + + # Fetch ODB rows + if os.path.isdir ( dbpath ): + # We don't need to extract if we rerun for the same period + # Skip the datetime if the odb rows are already there ! + if not os.path.isfile (outfile): + try: + # odb_dict method throws an exception if no rows returned + conn =odb_open( dbpath ) + data_dict =conn.odb_dict (database=dbpath, + sql_query =self.sql_query , + nfunc =0 , + fmt_float =10 , + queryfile =None , + poolmask =None , + pbar =progress , + verbose =verbose ) + + # Save as csv file ( flush ) + pd.DataFrame(data_dict).to_csv( outfile , + header=False , + index= False , + sep="," , + decimal='.' ) + conn.odb_close() + except: + RuntimeError + print("No data returned for ODB {} \n".format( dt ) ) + pass + else: + + print(f"Rows are already extracted for the ODB {dbpath} \n" ) + continue + else: + print(f"WARNING : CCMA directory not found for the ODB {dbpath} \n") + + + def ReadOdbRows ( basedir, rows_path, cdtg, target ): + # If file exists else skip datetime (in case of a rerun ) + + infile="/".join(( rows_path , "odb_rows_"+cdtg )) + if os.path.isfile ( infile ): + # Load dataset from file + df = pd.read_csv( infile , sep="," , names=["obstype","varno","an_depar","fg_depar","final_obs_error"] ) + # Get column Series + obstype=df["obstype"] + varno =df["varno" ] + + # Temperature + t_an =df.query( "varno == 2" )["an_depar"] + t_fg =df.query( "varno == 2" )["fg_depar"] + t_err=df.query( "varno == 2" )["final_obs_error"] + + # Specific Q + q_an =df.query( "varno == 7" )["an_depar"] + q_fg =df.query( "varno == 7" )["fg_depar"] + q_err=df.query( "varno == 7" )["final_obs_error"] + + # Brightness Tb + tb_an =df.query( "varno == 119" )["an_depar"] + tb_fg =df.query( "varno == 119" )["fg_depar"] + tb_err=df.query( "varno == 119" )["final_obs_error"] + + # Wind speed U,V + uv_an =df.query( "varno == 3 or varno==4" )["an_depar"] + uv_fg =df.query( "varno == 3 or varno==4" )["fg_depar"] + uv_err=df.query( "varno == 3 or varno==4" )["final_obs_error"] + + # Obs errors + if target=="predef": # Returns predefined errors + return (t_err, tb_err , q_err, uv_err) + + # Returns diagnosed fg departures & analysis departures + elif target=="fg_diag": # fg_dep + return (t_fg , tb_fg , q_fg , uv_fg) + + elif target=="an_diag": # an_dep + return (t_an , tb_an , q_an , uv_an) + else: + return None From f35e23490fdd199ef882b53998f46b000153b23d Mon Sep 17 00:00:00 2001 From: rmibdev Date: Sun, 15 Mar 2026 18:07:13 +0000 Subject: [PATCH 34/80] Add read_odb.py --- modules/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/CMakeLists.txt b/modules/CMakeLists.txt index f25ec2e..8f2d623 100644 --- a/modules/CMakeLists.txt +++ b/modules/CMakeLists.txt @@ -8,7 +8,7 @@ function (install_modules ) endfunction() # Python modules For tuneBR -install_modules (gsacov.py odb.py config_env.py sigma_bo.py __init__.py) +install_modules (gsacov.py read_odb.py config_env.py sigma_bo.py __init__.py) install_modules (readgsa.so) From 61f5b349a2e96e6e6d2b54284fbd1a7426116ef1 Mon Sep 17 00:00:00 2001 From: rmibdev Date: Sun, 15 Mar 2026 18:08:21 +0000 Subject: [PATCH 35/80] replace import odb by import read_odb --- modules/__init__.py | 2 +- modules/__pycache__/__init__.cpython-312.pyc | Bin 508 -> 0 bytes modules/__pycache__/gsacov.cpython-312.pyc | Bin 6405 -> 0 bytes modules/__pycache__/odb.cpython-312.pyc | Bin 9462 -> 0 bytes .../__pycache__/setting_env.cpython-312.pyc | Bin 2928 -> 0 bytes modules/__pycache__/sigma_bo.cpython-312.pyc | Bin 17208 -> 0 bytes modules/odb.py | 256 ------------------ 7 files changed, 1 insertion(+), 257 deletions(-) delete mode 100644 modules/__pycache__/__init__.cpython-312.pyc delete mode 100644 modules/__pycache__/gsacov.cpython-312.pyc delete mode 100644 modules/__pycache__/odb.cpython-312.pyc delete mode 100644 modules/__pycache__/setting_env.cpython-312.pyc delete mode 100644 modules/__pycache__/sigma_bo.cpython-312.pyc delete mode 100755 modules/odb.py diff --git a/modules/__init__.py b/modules/__init__.py index 928eafb..f327247 100755 --- a/modules/__init__.py +++ b/modules/__init__.py @@ -3,7 +3,7 @@ from modules.sigma_bo import Predef, Diag , Ratios from modules.config_env import TuneEnv -from modules.odb import Odb +from modules.read_odb import Odb from modules.gsacov import GSA __all__ = [ "Predef" , diff --git a/modules/__pycache__/__init__.cpython-312.pyc b/modules/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 199d168a262df9266ebb6d603ad80fa8c66c6b3b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 508 zcmYLFze^)Q7@f&(HtUagMT)d@RR~DdSo*6_I1nTU$AMroOm-)e(cPJKW;aKyVs*{+ zr}tFw*LG+?%i1YKWU0`-pTlf zB|kv~A{c`MPy!SjmI~7h?bMxh1^ZRHf>REsn_i*Jkh+EEpiIk$)tOST$SE8U3}MrK znh2dHH078MXneAd(AQkh?Jp>iQ65vYhC-R7c}5XIsni)&Tq=$tPMDCIMo4p(5Y&?j z$*!hqKr$``A|gUv5*{%nb4yXPVk)s@j0{WMIeN`i1e`a$*<8@F5S)=Dr9uLF#^ z;2FlPbZGpI>2ymi*;|vp)SLEsb|8df55NzPpf;*JfZ8pnoi#3&&zJ8(`z9`1+23<@LS>VOVY1A45(dQ4*j zHeLpovFUv@V8CXOMr;9T!d8%GTn5rIQSPyc`h)%6pc|aDGSbx%3vwLO&UAQ>9AUgn z*Rei-pSOqUJ9gaP-Q#1}a1al1!4@vTFp7q($1r_9ZztnFU`V@vM=zOh z2h76+M=|q81cWk7Ec1us$uJw_Q*1cx>`#V6k-C;A0o6>3Q^mh@}hmh_uS`Yo!~1oBWl9;-+vgZ!{) z2n|MJVNo9$}E22qxHS(ojPhj4?@rBTlLa{7-LdEXB1B?`Ff?+2?n)4ktsf zbn({EcyPFtjYrM|aCp3RD8dh?&a`?%p#+ONgMRRWYfX*}wYF%{)(Q|0CB|El6QVf~ zh>S+~KpS0>$Fo<(##l3E_>AXjNY+rTP<$RZ7&{9(Lth(3bx-P|`^|@7d zeQxii5qUs;Zl65%jgK8tqMkLB0i{ZfX$><(yxJ&v|~%@SPGrrMIKpASbNfz_y~@$%+4eunhtpTeVyHXX}UF$;?o8qTSY1$ zQfKdyHB8e4u_9%Kn9(IT(a243d^paMnMFF043CP|2p1XU_#osGk&Z<;UbKWqQ*n}r z!fY)Wr{{PUVl9^7AZ%E&#-eFF$c_f%VF(}jF&>Piphn^`Aez7nDKe>uNJSqB9CkfX zx4uDQaCaAsPTL9Ci}fwg&ynwR4t--Szu=kjWZ%p;UG>a)=DQ1<_7$r4EsPhcU%qAi z>59WOW0|((JOxKn{*8iT$2^EV3;hMhON-S7$19UtzNvI)Q+fSzWz*!gRa;GN%g0T( zY};0A8}oZ_*LtR_vd-+T6=&_t`swu_SLJ*2V{`q#s-53DxA8Ys3%#GZ3ym)>(uKzT zUpimDr$=?$@7P=yPEVcwWk4xhej%6osQpXZHg+TU8!C+yf>0@*^4}w#){c&~TL*P2 z8Zx>el+hPs#X3>rJ_T_O zt=b(IhNp&e^cBmjCBNrt+g#gH&CX?e%jD)2yJK?mYE^B{ncH)vZMN-p)t1R1$^70N zz7m;@+;%r-ovZch^E5&Aubuhhb7jl*Em?imn+;|=E}HM_QH=*c?b`aS8M*3mt}Au3 zb+@ZFPXzAzSo(uzRTX_I)L|AV|Cs3FgX@(Ev08l{d;}@2bzizaA?f#68*VxEfn-JkM zQH^gDtrON0)p$lpJBn&sNU`<0!h@s@Ll4lTv3}%5@Y+>`Bc^$cn+uc_3`z=+(z_mj zW$0mvakhyF(wW}Y}}l$ohH$O?re4^Bj3435GHVjR(U#q~5fZJ zuN_`=E#lure;fV6_2>FO)-PB4rPh!Qys;zR#_44oOoBq$OU)qk`an(!?JP8-F zJW|&D2lxs^6-6~30zJxwDK)wQOlyxQ=1BchMrhdpmUe9$8okXJG{_!>u^RYPe}H9Z z^J)BIZGL2sfWH;2Xw0T)1q0^*QmojQIMZ6A%rMGSGRi2JNDc!Zb&~a``KZ0I}m;rmyZ^~Tr&IGZ(_z4!BR5HELapD_0RNC`{t65)=b%t^wBC8`RK~u^0DLOx~RwoF2QdP2$5!GVDYbDnh_jLb}s{w@iglF|9jqq*2C^ zsVK2U(mgqUJ$99_mn_l#K^iI0a6YE7hQB3q41h+Z;1DXw7s*udF4Y~ODh)YFVoYmT zkou&J3*N%5>J*<+znw_JJ}{MrOjm|7E}=qj60~vD`Bwx3sR>mwuL7o1QX}6JR|qcE zf?$_=D+l!w-+Bx9)>*hO0;gF!L)q1wDWSGHmog$2mf*Fcl|8w9XK1TMTY^-j+Fi+6z#R8)PR${mF_Y}$IY zb*^<`&+?|Wf~|4#WX?j)Y}RRO4*$imiIgj*E0#9xUg%hS>6W8QX~^${dK;;`8gdg$ zuI43Mv&uw%m_^_J)1q(j&?7$j7Q6pPAHUP$w|3;LnlVqCb4^#a&u-86&hDJ2Ke1l3 zF6_EyTfm=1Z$xi8Z^W1CI+q>3$t~X+(fUpK#(d}1!*hohsJWxJ8g?%`_7)m;XU8r& zml}3YHZ3{!e)C<&hPn;8vApYQ{apR+x*b{XN(~$Gz;=<`uz?6El}MRI$}z*x zb`41cSBf;2Od<)A;W#q-eRN-EGFk6cqKbwUd*h0|ZpB`I-=r_!d=5V_TTJc;?lM#J zgAG`j&x5$9rzq-cWcdnN|B4#ELX~%r{sZ%S<_{{~ WtH_q!B2D&xnNt+?+yg|UWbwayA3$yZ diff --git a/modules/__pycache__/odb.cpython-312.pyc b/modules/__pycache__/odb.cpython-312.pyc deleted file mode 100644 index 9c25148e1bc5b17892012f8ff870cc14e083580b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9462 zcmbtZTTmQVdhVY46&Pk+m-4|XM~xGJgCj0dk~+qmphRF(1@mYjIweaZiy zo}L+GWFB$`&gDPwkZw_7L(o7VN{$U%zwPb{cGm(M&|h0F@YQY;;#0!m5; zXeeu9nz%Ngr4^bkpaYsVrjHu}hPW|cjOzpXe4Qy^ChaW&3#7W3HEs*o;`V?&UKS`L zJo;F9ydqE$cLW^q%0Ok@8E_K1A?Au#1*&LDLk&@^@g0gaeV|ch7O0-5J?5{lFzTUY z-NleN<|HY{2SB2nICzXA>NB7Ez|7#QC{VPo9VJ3dK>l zQvnT22eho_9SZhH3$dQnL2O|45F1%DYj}qam~eFg^SsGpk+oxNIP^MAQ5J@T{#Zy5 zm>vemOe-_w^)vo+Ugl!2|J*n;=o=eh@>5~P`d;$(`K_~R_YRMa!{|OP#KH`G$?F32Il_rkNtW@CF+Oka zz|KRC^^LcAeZDat%sLqs+9qc?SjT`DCg8v1^TJ$42S`OI5sJ+VQDNt9N*e4qM;|B7 z@(HGAEGz^$p3e_9F`3VV5)8v~Ga(+hIGB3AkBIuM@XP7pUkm?F{sy8IN}?qy2stIN zSc0#SFKM7uXe%03`m{;S71Iy$*#YXdCZd_7BAPdVe^Dc8SZyuTEo*L>bwC9Q6~sA^ zAllWOM6){N`yf9$a23Xzhjuh+d!6#=M`c}q|48q`*tqus9s$^HaejttoeS}aWNRo9 zBs;(96v7`gsE9E8k3!qoEQx8k*Vk$7o14)@sKFUChmq9j70^J$BmZF zJe;E-c7>N#;9?W99!KnT7mx{#*>^o1Q!|knj*oXl=0Z~)d^{S4CCqh9M#ZVw za7S+>lH}QekUyD>2_5kyI~(JK4!ET4GxM@37>p*OVlcR1gM-_ymg=EvVH~2js9bGh zYTvCxx!UI-Ih?DVhNOM7{F=DBZ;gKMV7lXcre*|q+BWMN z*Ys;wGtAL!-LcfZKWhKq$6Bg(los$S-m~?(blc9C0jT7B^}&~1q0E$6I8W1svjjZ( zAQsMoM62JPrB&oR4kyS=Nt)@xSy0lq06xP{9f_vK zslAkFQ~CB$yp^V|5S~)5dChg|hV}||o!(1zQUZM)PUaPghTnXFPE%r;+7IL%YM#|j zJJozKEv{Q#KlZ5h=FyGHG(Wh|bh+0zIy^cA=p=@CW+EEnnBQMy5=oJnNX{nMb_USY z2vt}k>yk5ELe|0wl+AEL<6Jxpr@RUVmGx19jq!El70qc zhA1191MSiAc#cT#c#35js+2ATl$~&P8NLxYZ2ePQ!8ZZsqjy>0fBqhJJ2a7dD(-_pfbfbd^oHhQ_$45rOM|)k zrn{%_oPO`@w?@i!EaU9@?KdW>w&^iN19v_1tre(gXV;bq%HMr!R!YCyvT3c3&utBx zHb=Vh_`}N6naa}-E@vyf8@9pU*ebpg&ckHhey+EN`b)aEUHkJxy>87%b~BVea+`Y( z>px9Zz`@<&BCvER z{4HvV*X9}gnr-~rUHJ(Nt4C~6w=4fPI9}9CdWl|u3$lcbeQWbS6z z%kma>2C>h%>7I@SZ?K%s*uTO>IEclv|Z?oqqHYiGF$sk!jKoEz3z&1~L z%0^{lU+=hgVAv-cL7PGeDcc|u^m|7xUg-6EW#i>B-+8F_do;2Uln^eV$U+c=!Q@rY zBkj@2M3UzeIV^Z+{uuNxTdqJD85tNjcW!W@%|FEvWi2mLnfRKlCsWy51p+UXX|?GL0{JSLh4}7Rlhow zakZol>mU-oqN)aBU$HI+CByYxvE#Xm%AT2_hy`X z*M_pt)5{rW$9?*~2BZUYVt2uc-?;Ne#(s#j9R9ZxS*Jg3_dl|i-@bhF^6fWozPUQI zVc-AA?tEx(&e)sRy4MHqU;fm7W=l_%wLF2{qayjMfq4pbVJts+`)QchsGtG5$PYmN zqK?(<1bz}&P!q*I^ujh?Z8u31l(Kyp##X>3(za#bthrlp4%kTd(_VgX9M-cTfbAZ zQcw9Pe*{{{bcd`3O!*4^9^bW&gklOdB1!^!@FIiGf_7#E@iojbMG&^|C!oo~o{Ip3 z0OACVdzb;>X4>1^8Pqy0JPJX;tWX$ikOgzw0@F6jv|VF7dg3-fTqVk;E)kiTl`ROA zAwb1Rfk$Ixp=l&42#CmWP7JldL_$QN6&S#HwA6bTfGkK6^{`OJfWaKkw`sC?DrGI` zrwWYgrjyZxY>7gDu&xBaHUV)h;3eE}ALRY2NwI9J@u?v;k+ zhP9Rr$Du8Qwelr;v+=-sb++-yQeUo-xf{C^doQ_kF4x$yb|VACbDQ-|>l3-dow?Q{ zxsIOP;jVAYx;@>?ecx15mF`tL^xoOG;popfs=hu2OZn|qs|Gs#+_itxRhw=&`(W66J#bbZDb-eS`w3h>f_R%&PKyBGO86eAQqwxMu9zws z`(g<&?_`ilB>;=yyhOk8htGs(&G+D0TLjQu;8|C~S%hZ*Pjxi_u!bGMTF)9wpwsva zE+gpdrisFQK&yryxb!TniM5i{0BCP|K~b`^F8Eck)%V=0Fr(p2--?xpvsx5SY~9ZaXN7PqmT=VX z$nnPovDd1OyiM%YOn0dCVp>!n)On*ggc<#9ULfvK1S0=3wBdaiDe{eYcZp<*pmyI% zBoD$oNF06)`7UBaBprQ3raNW&n5^r&;`e&2as^sGHiE3^6i;gk zV81!tD~Fa3tzOAg9srBbV!M6t=E0@z4NE;3SVb+OX4Uu8#%F2~UHAEoveT-<(7oG}1(`_<{@6Iu7^)WnvFYU|3i9?cy%lxsVZYd)Aeco_eh_ve_sxu$)& z{fEDHf999$9?>W?1gHef8$m z)nj+h-Z}d~_i5P~SW!(q(OY`f`E)awuqRSe>NHNS-$*x~%({D0lhC%(RnT0jHyj?I z)YjiM-Z9=SyHmDq+^9LSrEyj5dDQH=KbY%0{`ZZ)YRq*W&2@G`f&87!b)NX9+_?AD z()>3rs-|v5T9(#^v+nkX?&BHv@%z_vXZ!K*)Y;q%eOqR#=K24jbTwWYw4Cbtbi=@h zuY7p!W8Z>ky|Rl3u4!ymVNT$6s;pRT#|>&jQY{xMC|pBSyJ+Vi=4@1}cC zy5Y>jhJg&U4rd!)+HhZx-SyuQJ@cO%dksSt&A(WhhdT6;QWXXTE+|0MlT8r9@a9QR z!)bUrz?%%b-z)&RMC|E`YSKn_ z77;ohSp%`F!1>4;NzMpv$by;qm+^DAK$OU$$@pM~^i-j&Zv_$<%hvpqGgbyq%R zT-&aE%DDDt@D-GT{CdlF;U^AB!N321z zNEMP@a!%;RDWF#5sb;BMvPqSBsv}P|NoC+zbs&{EE-OcgFqh4Ur{pD$|1lgFuvCrt zmqS@ko(^S``gSOr)W<`)d}90Sp-00bc!CH8$QSrOg*?9k(Sn8a!JIk69OYM$dIzIF z+g{bZU@Ne7cAi|QE|j_$@K}=4nKQ>^!we5k`V+Dd=MfD}f};$EC;>B5-sOR-8nGormWj@K+{?7}2m3#U;o(c*#s)JTWQjMfB3j zx@&}VjgYPpl3oP6HUSJ_BpM(=M*Zv@p+bUGNPsG!pYZYxNbM*%!)DLX{By z9hwR&L_`68S5^@d^~!qveSgMtGF^8n>o}ci*(`C|O?+xU@Cfkx@bcl+x%K`}PKq0q zb1C1KRqj>u@*AnkImj3xa|yZwFCiMtv8=OgeLCYjneykHRX;>Zx4)2azWD!cTjgFk zw|s8(M_E_LeM`o5Ds?FbQ~3c>`bcla)t7n&=B%_$Ki8dg^*rdwxCRT2pG`}jx{z^= zkme5es_ByoCRft{yQ5#9OS_*>mp_ktQMY>X6X(8M&7RfSHF2FwyN{*IkA3N^Rmp4m zb=~@G+I>7-etb(yx%T5^YWWfPK%*Yb;2^v^!s|2MhhR{)1cS=UG~{hTcq|Xa@-@a_ zkWEH{K^`4w#I>$?VmK_mi_xzjdY>X4lLNw|6Ou%zgA3@qd58L1V>DVGSLlt$A6Hn7 zj>nCR@$BOZbc?b2$t(0Z+IX3M@`~n=QJEiU>~SmC1+_kWmk_67;Nd~xM|}T0ib5V` zBaeOpxO<@l8xqK!QoQZxs6RF{-;NLd0*^1zJlax<)5w~ygtvK4(G9IwhYoq-MUt%o zCyLXJKVoe}_5TL**`*PAl7Xmj4H!p#&37LbRo&Y5KpY^3SQ7&nWvBR9A-T n`iydYMwNYQqUh5Z>eLrhUzY0oZzwf=tDDf$wT~%`N!R}e9L8k+ diff --git a/modules/__pycache__/setting_env.cpython-312.pyc b/modules/__pycache__/setting_env.cpython-312.pyc deleted file mode 100644 index cdae8a3c05d498c778cf7ef198561cf5170ddb77..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2928 zcmb_eOH3PA6rGRlv5iR}2?P(&SSeI75Ysk_s-#sLW1B$ZUu`EORLN*OGr*9s$DJ7< zq(~{RE6J8&>6y8G?9@6=%3?VfzdS3xr6Bq(Y2ZoE#-7;xZvp_Xv?*qb%eo zHA{wFFR+ppER#h^TiNaaVS$i{QBovEDUlqdafVTsNCS1xy25V66sF2jqC9;!?+=+H zrtrFcF%-)3nl6Qm2`QvYf+4GAJ;ayA(70rnM%j=`dMLz$H6+S{VFf$X&L!BDn)P^r z((v0Zz?Uu%RdUj)({x)zBt^>kzAC}h5eK5{-2ilbwgb>E(Jgu&Gi$gC7!YAgAn#v| zOZ1Aq*CL#M2}d+od3ICLCHlpN*CITQ|GiII<_gw=1H> zk$1Rwbj|Ml995;_!~MLay))8s0rI}q)_>DAj$D@Tr9{EgGh~^52!D1pEM@%ML7|x{gAY3gB4kr`nh%S$p z_@c^bJRER(m_*(&UQvp?a2vv6Q7)Ff`T}owQc08HnZ+@!=fu<_l44QYI50z%Ud5fh#0Z*(4N*e7Vd> zo--ZHNG5j$2f0(3cp`~>u^X{uf*Vf9kO%Za^g42<^C^=JCKK0?Czpt)Q?YOm(QrH? z-+(kOm*c!4Atq54EkT_!#$*L7qQG-PWeWKLT2xg<;>*aRDASTw1U{p|!H(QYMNtjp zt!U~vSL8M1HjjHm>xKqa-k9Ljih-CSuS=q=A&+kGMP5OSDi*O91wi27hNvjG3ZJIV z=$z?bj8cWeA-<9-N{X4rC(H_p#K~m=dBA4ycaXcRmrORMNYlupNn*Jqz-{HyC1nhG z1hqUSk0Yv5K{PIeDwT9M^xM3C{u$_>s0bBFEA#>7;yKPm{S z23;w@$Lsx2#*`xI{kmira(SGS%F~g`Eb?+tk8E(<-6s2jj5rp2gX@37-}?pej14XK zEcL96tlr*WFV1(}kL*}v%hI=LV{Sb0tq)ya?Ac&H1z?{81oID{VC#@={f+IdvAwIq zPwxEu-a6a6!REoy3-H4oGrJ;sc1O(aa-4nXCz@IxRKKb&Ufwv`HGlf|kD?oF?0fAg z{cUW8UWtC^`R2-FdNuZhd^EV$Q0qANRI9NuDB|p9TgP(aQsatTYkS+Q*48I$ZSR;x z?)7(smQzcq)kv-5!hF}S{`P0V&gIUf&Xvz=!E+E0arD_*5GUx{Y-(KyJPa(3Y&3QM zhr%M8>>JCGrO0Y~jXiJXwe_|d`>t7-t)Hy1cpm7Dh2N{!@y#u-w%#P#oV+a=UrCg8yP-F(?x1OZ+)#3b;&V~zxS{e`mQh5MDPQe#w}2$ShB zrA5G{od*)8wfBJ`Zx(h`)PSAmOW{3(MIBEEt6jT<5k8KK!xK!0)!MwXT5VGwAoNo} zV7ayg7j8bhIe!KVw;tY_4?k-TE?!t~?ppVEZ3Bczhdp@dm-&*!amdec*7g87z;T~V z@rvzXI8Iarj?)Uc590XWo-oxoyR?o?{UuMV}n6YK^W48-P}%!`Yk4;pcW(d&Ov03qA6N2 zLiGwyMX!RSmAxu>Do50#nqG|p^Hd|+QC+WYG^00zq}3ytqxxPwiEBo(Mzedf6_k=X zNYUDtDO&foQmCcZKyVp=%am~&2u=^UEE$(GrLbjxj!Ydk1+Qku8P2BUH7rNFCVBO! z%i#gE>RH+e4_)_!$92Fn*(cN?K!upNR{@bZ%0*#UdX=;Sx}toU>Q&P!fErp2P&=iu zX?g8Y#znjOeFyXw684WcSk}^N>2z@ujEA+1k!-ieMO(U_I`~Z6vzC4@W8o|f7AMzW z8Min*v}M?35xM&`39Z52tKfH!A&v1XXhku^{FEJ_5CEqXaZ=MMzX~YCHwV;eNMR`? z3?R2Oc9dTUDY-@(`Lq({6t$@C;C1`jx(>8I)yZdeKHJk}KibxP==T(L?N5LD(`Riq z9j|jbSQqVP_)NxohP975xItbsGU;NRymsUaG??ciK2uVEZc*1M>I z!EufB^BQDfnF6Sl$wOcSz+#ig%lxvdX=H+J8r;UX*fY&rn+C`F#$3#3Q{SXxu!$LU zJMA#AO#^OjaKhQt*4O7{=ypf9*E_;CjSUYpH8wVldg+M~7u!S{VRw2P$ENrUyWQ<^ zb9TEgM;N?D34!fk_W<}l6)V~pJP|3XnX#U0h?Q&(WkpIFAl?Xgc+|y;%HUDIW-{X= zW-28}Q^kz+)w+970M#eIWk~4YM@I7RVU*@5P9Ccvd7O|(2!3iPkypgh3LnI1&;DB>2^|Rw`#&!)3~N6-U-QrlowQi!;RY-xk~6PMzLC zQKz8&)7ojBUo(^=(Zrt|4k-zKEv>^gNZfP=rV{dXQV!zj%tTrRb#9lMf*SHkQAyx! zF{PlUkYfEdsH8fiCvq+7JHAvmc_+AT*b_2bFSuFG?j8P8gQzB6VdoWRKF2P9sm2+} zEA)$cUg>4|j8VrhPJNcaN_n++%;n+rZr1H#Iftju#j8i4C_dBWnHY63F!y7>M#oVWAYp$B;QQ2VefDehXX^v>+{o-mXjjB!Tk8H)-x|v;n9H8c z4)#QHtswvn;jTz-^HO0Xch^kSZKEkL5mH5s)|r}BLs78ml6A#UvuZ96?YwEW&E^O4 z0$XEw=DD)jvP=1)?}Ww|x-Oc-)eD>7&0qS?&rFf>maFPW`D35v?YX0(itE-41+Sex zd-_$o7OHr-mA)6q?wVxEyd&l11~+upH*70cu)_Ti|dlY!k2>PpdqSKw2fH z)sWr_X^oiHLfQstotVymba{N?(V38^e_Pxb6%;3Lk?EFHe44?Mleb7{pBUPyfZ}vF zSz_{18j=^1M1@=TP2} z*-Rb0f@5$u-S9xA7Gm^9*~>cr0TM1~z}r#aOm%dE8gmU>WnW ztlK%_vao|*hO?kdK$TKZ8bNcEv_?>_h<5oo%7o8xzu^^@FSV`B&G5%qA^oKag_gR* z>v47Sg{;?(40cq6S^FewN1cq%PDI;GAcPpVhvQZ36w9kYAaa6whT9&W)#-B3pg(tl zx<>>blM8634)ZkJ#282^n zi2SJV5!H{`O$gYDQCqek8$m;ax5H~ln|YO+et}og95xb~z-N;Fkf_$&>Hx{>DNowL#kerzZU0fHcr$>FcqTrM<+kk^Dfe`~;FxNO!wU$#5=zGN% z%A4*zvsE)xyD?7E^0bHGqGyi7<657tr)kg7H$qzg;fi6m!0pp7u_p`yJHnK zn7&o9JyNkfTCww1#hysTo@m7rffKQc>bL3_>cdl!irsni_W*b7v(5{8d%axb8rS|s@UpX9Y+8eEFi&)yORYfcZKT%zO^5d+CrR%2g*jnjE zD7~mSSRQN(oPp9wbYI{FD9cnFtehPPI%kJMW>7y;N)J~@EX_-%h^6I2)zv3Ifb#a= zG`8QVp-h%L4b%qXYbVd147T4eSYn2}*X(EQ%bRvCOy_4k4XZ;#yEkJ>XUpK#?QX|_Z&fcW6UmMhlLE3*Hfu#&wymRG zFl+Z&XyEpES~w=(I;GfW4s;)SwjK7k z6g*UWXy2jdyDfVxbgi?&VYAech}QP6H?Q9yZwvInZ`r&i# zz|Y`pid)vge8Z{eGvK((Ch#zlfhvpJUZM$HY6+D_O1FTS+ZZ#W?OPW!Z$jJmj+QD$ z>(>@Dm&6}igwz(vGA^n_%UDqIMWq(%_tt}8muy~%)}fY>>{Y+Ppi(As3}g~Q~YULhS(c`8_D7m})R6X&(W zkLF}?=MqGMs7V?2dq>A6I2X~ToIWF&6iK+h0~Yp00I;!S8)nK^L1deLa4NX<6+1W% zj$bSdkH5Kj)l~Y%{*dLySC}yXHZ2)bhCu|2fI~pWQy57v zi?d!BNkJt}8c1)Nf%N_mIM`7DNdtK@*mlEEcBwMdf3bck??%NoX($ihG}XdT<`)FD z!SVUbP<1$O!4@&sNAkDK=E2|;*>0Okf^Bc~g{sdF&p#72)g=yHQEAf9CHjZZ-XZI? z0A%GHI6Ot)kP>Wb^b`r6AtkLQDZwwKq_rd^c!iX7#c8_@bnD1aF?k>z31gPOLT?pI=(Ebrz&LbEjZs=i*5zic2 zSPXhW7}O$cIf5R23=us5cnws>x|ruMj<1e*pz+m_h?4H-mHh*}(&6EitQUSx<`nXt zM({iU7JDNaP|_pCCEf1Z&<^jeQF2JP{}lQD4Kzm3yA8&`{#SZ~_rv6T}iZ=G;PPB=e(f{s~gfXEQ& zkt0xAY1swaye-rdKC^6Xx$0Oc*&FBti%meEuMc#_N;Y1onXh@X91fMO^IL9}G(}39 zq9t3`%ULVgbPLSFl7_HzsUljk3u*7em`j56>-rEA?tar3GjF_*IiGpUTpKahhK(y` zaBkg?DZ7w8pM9w-Y`oY56l*|{HJ=sgUNPIia~@K^{z%vurvIrpW~m5mh1Tj|R7kWl zlo8w(+guUaK7S^}%=^N%k5VjdS4zjfjbd(=j<3aA9cGrw zm-dBU2v=NXuWf(-%r)kH-}Tza6P;JnwfSmwqd;*0jB@f%gUC4YbSjofmK=gj7WHY{mARA24>=*0UcKFC?gJ@g>_ zD!3J_I8%RQzr=;x-+5~Bsf#bc>BNB#^vi|&mJR#X#KVM?!mp}7(!Z}?$=#n&a2!`q zNjwo9woJx>TY?io9|9Ud2?DYYk)4a|VXq?2g#c{^!F@Y`F$BoZS^(g2ffkMna_R~44wI&`B9x!U`7_KO%9@Ki@H|_sKkQ6aV2ctA^Z$Ty1;d=Uw-WBH#SLmWP!?+k7r0VtkspjOkq1j70dVA^0??@w zaDwYB;T{{%X+&PxEJqbq#G&ho1iwbc$%Z=6Np;9Q0y+tYE(d{EO-uD=$vC(cDAtiJ zES{ei)r*vx{EM!CtW$2czshrw`= zJm$p6I7EiwQWFS&ZcKQv{x<-S(dM$C8>9-Hi8QR1lm+SYHgY^tvI*Q^c&)GoWRn=J zjYpHP#TGbY##5ELrIxE(ue64^OTL(lte255#*q)BT$9=BmJVKRyYh5ceOaH#o(5Sj zQ)aE1@cJX%dxV>ga0G;@SL+40s=O$n`?$z(;ob|7EgVp2fs-NYTMY{hk;2CCWTbHCioONK>BH|T zR{b^ZDx!@b1&ovVJ_r#a@&u1u9S9>KRt2I(xVxwWhrfc{T_Wy!6iwo-E5OHBieI)b z$>|S>8SsdLn|}&#Dv=l%OUxA1$4mkMdri|n;&pJM(;qfq*a7j_2KKN^K)`D`2Q%P; zt21yL6+|}5Mqx`!cKi$D;wuoxM?Aw3Qd(He>~Ua36C^Lt{mQ8z7vf&`$8t@(>ES{~u6?tWYGCCaVug*!AlGZ@oBwK|O=B6=^nO z3J$pv^58r(A*F$~$!bWvK3YvY*`J{j=YzM+$qES2qv%cv(vFF1B#Aan*)lDzB`vh= z{u)s8P6;yC*R28Xcivw^nwzrUxP}aPqxaW<`*}+E#x-Pe4rN(4)|ciF#sRPkd4XWW-Wcf zVKQxTPFc>l+yjFg7V5_kaba`X*aK&x{xleoWmfF2TDB0b}EOzy#t zf@&G>cwODLI`0A1g|4Fj{q+}G5?5Q7Qtl%KSb~&1V2Xb69l*mGfyGFU4jUCxQ*%Ih@C24 z)@0s+EM5igk5}VM!RRHh}HlEJSN6Tg5Lpq)u12T%B_gxR)pH$dV1mMCF^^2SL&{*f0^}*tn2;J+~YHq zi4=zy4g*E~mHI>qYa+!rEn^LCH-LBy8H4*{iBTdn0oyLhPmHd?%|`cGcd+dhFJ6#r zn;m##0`4XDEcBdzDPn4TR|7Xh-B;Yv=DiWq-WfWE$K@i;a|_R1kLcWB3a zZ@4N_*0cnmC2D#E+fW>A4{Z+@gt_Iy9m^Yb+}@DCT(Ey-Lp$7YT}O=5AddfEjKeBP z2q4(j7qRLT5t}SjlLao7?r2^RssRNvD2Y|5Grh1+qZpv7isBWoBnjdsz32w)dfZw` z90Hx}V~D?ohj8mSbghtp$Nj|IOZuV=4&d8u`NvdZZr6sAJ?Bp__9`$|okmK-pTaxNY z^Zupsr6=hR`G6)~2Go;rUp?f*nmA2TJ!$5HRKD~i^^i|j;FF!S7Zm6`+6Y%7Pyj&y z7l5E@Wb~_(6pehTM51UEI1VH$8bn%ybEl4Om98p-PGJ%5v*~47E<})StTpl5-Sl*f zv(ADaW=weSzO$f;;OD&pwS|1@OZ1ZXCmyL)F;2%M6d`LlqB4}GutY_HG?Q`eVV!hS zE$bYs6TDrvbft?LkCSLTW8+Qen5$<5m1T^Rl7fx`9y)?ZEQpww9n8<*<;NA0ZeJz? zUN~q+tHr0TYTVK9%d=SgkVem%l%dxH(YQdAm@T46HhlbyWh7>cs4SB2m>fvWhYkRW zxF*2QN32=q95m0@kg6rQJsz*14yI9(jDnJc=W5Ia1Vl*+VQc}xn+PrSp?7J=d#zVmuUUUt|BL$NqsOCp&w&E; z^+--GoP<&yz4B;MDONbEgHxAvRh%{WO5?=Y6T$KyJRi$91r@Wqg6$U$&L4!&UZxhN zA{(0{`P*i?*Uc&l=FPSM*P;1CNnFQP3-BVSb-n=<6F3$u*gDfKoy6d09Wis|rTt;; zQvPCg)Vecb-nqQ{aKwE0ll?14yOz73yVZR%(tR>|^i=f7>4^FC9ZKO;SihhY#nx48 zjm-9iN>RKkw)-$Xk0@4;JbNGW^O$8<+))yKm9p4+oA4Jb8lu)+UucxYEo;CJ&m)T4 zasE@0?o-jDr=v%n$1LDiE3tl!y4d=M463RYS|FB?sV;5-56L4(6kkB}hyt>oQ5;oB z4LKd@J{>*!eDsJNv%jkVQm3My(Do|=nmc+buaF#fnZrAls#bCzSp&Uof5;k!&yyOL z4UM-A#^r*2D~9%1fthG+9kz_r6R8q;ViS|%BTW7ZfeS(WFzPi-BKR-(v1sPLOx;!H z>&osnz-KFWEA+aOyCpTc-S>JG)zTL%NN&qzo`#3$VH|`1hu~nz!9M)M2KeUy#PHzt z@KH4fACSSvB@C#L!WUz_%1b*L)SdV&u=F_CgY7QQB!kXp;ZvFcjFE;VH Date: Sat, 21 Mar 2026 21:19:41 +0100 Subject: [PATCH 36/80] Clean module part --- scripts/obstool.py | 137 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100755 scripts/obstool.py diff --git a/scripts/obstool.py b/scripts/obstool.py new file mode 100755 index 0000000..a885b22 --- /dev/null +++ b/scripts/obstool.py @@ -0,0 +1,137 @@ +#!/usr/bin/python +import os, sys +import pandas as pd +from datetime import datetime +from pathlib import Path +import argparse + +# odb4py +from odb4py.utils import SqlParser , OdbObject + +# Obstool modules +from setting import Setting +from utils import OdbReader , Rows2Df +from conv_stats import DHLStat + + +# Insert parent directory to get "modules" directory +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + + + +# Simple methods to parse arguments from the command line +def Usage(): + help_= "Description :\n\ + Diagnostic tool 'obstool'.\n\ + Estimates the spatiale covariances , correlations and \n\ + standard deviations from the obs-guess and the obs-analysis\n\ + departures.\n\ + The output :\n\ + Estimated optimal thinning distance for a given observation\n\ + type variable.\n\ + \n\ + The implemented approaches are ones proposed by :\n\ + Hollingsworth/Lönnberg (1986) and Desroziers (2005)\n" + return help_ + +def ParseArgs(): + # The var list must be given as a liste sperated with ":" + var=lambda s:s.split(":") + parser = argparse.ArgumentParser( description=" ") + + # Args significations + arg1 ="Path to ODB directory. (default=.)" + arg2 ="ODB format/type. (default=CCMA)" + arg3 ="Begin date. Format YYYYMMDDHH. ( default=1970010100)" + arg4 ="End date. Format YYYYMMDDHH. ( default=1970010103)" + arg5 ="Observation category. (default=conv)" + arg6 ="List of observation types separated by ':' (default=airep_t:airep_v)" + arg7 ="Cycle increment in hours. (default= 3)" + arg8 ="Maximum distance for diagnostics in [Km] (default=100 Km)" + arg9 ="Bin distance for diagnostics in [Km] (default=10 Km)" + + # Required + parser.add_argument("-odb_path" ,required=True ,type=str ,default="." ,choices=None ,help=arg1) + parser.add_argument("-odb_type" ,required=True ,type=str ,default="CCMA" ,choices=["CCMA"],help=arg2) + parser.add_argument("-bdate" ,required=True ,type=str ,default="1970010100",choices=None ,help=arg3) + parser.add_argument("-edate" ,required=True ,type=str ,default="1970010103",choices=None ,help=arg4) + parser.add_argument("-obs_category",required=True ,type=str ,default="conv" ,choices=["conv","satem"], help=arg5) + parser.add_argument("-var_list" ,required=True ,type=var ,default="synop_t:airep_v",choices=None,help=arg6) + parser.add_argument("-cycle_inc" ,required=True ,type=int ,default=3 ,choices=None ,help=arg7) + parser.add_argument("-max_dist" ,required=False,type=float ,default=100.0 ,choices=None ,help=arg8) + parser.add_argument("-bin_dist" ,required=False,type=float ,default=10.0 ,choices=None ,help=arg9) + if len(sys.argv) == 1: + Usage() + parser.print_help() + sys.exit(1) + return parser.parse_args() + + +# Start +start_time = datetime.now() + +# Get args +args= ParseArgs () +# Odb +odbpath= args.odb_path +odbtype= args.odb_type + +# Period and cycle +bdate = args.bdate +edate = args.edate +cycle_inc=args.cycle_inc + +# Set variable list +var_list= [] +for v in args.var_list: var_list.append( v.lstrip () ) + +# Max and binning distances +max_dist= args.max_dist +bin_dist= args.bin_dist + +# Instantiate +st = Setting () +rd = Rows2Df () +rr = OdbReader(odbpath , odbtype ) + +# Set period +period=st.set_period( bdate, edate ) + +# Set and check the var list +st.set_obs_list( var_list ) + +# Collection of pre-selected frames with distances <= max_dist +frame_liste = rr.get_odb_rows (period ,var_list, max_dist ,bin_dist , odbpath , cycle_inc, pbar =True , verbosity =2) + +# Concat Df for the final stats +cdf = rd.DfPrep( frame_liste ) + +# Time duration of ODB extraction +ext_time = datetime.now() + +# Diff +extraction_time = ext_time - start_time +print(f"Extraction time : {extraction_time}") + +# Get the stats by var +for var in var_list: + dhl= DHLStat ( cdf[var] , max_dist = max_dist ,bin_dist = bin_dist ) + cov= dhl.getCov( var , inplace=False ) + sig= dhl.getSig( var , inplace=False ) + cor= dhl.getCor( var , inplace=False ) + diag=dhl.getStatFrame(var) + # Save stats in a csv file + diag.to_csv ( "py_"+var+".csv", index=False , header=True ) + + +# Finalize +end_time = datetime.now() + +# Time duration of stats computation +duration = end_time - ext_time +print(f"Statistics calculation duration : {duration}") + +# Total +tot_duration = end_time - start_time +print(f"Total duration runtime : {tot_duration}") +quit() From 88399b4983a08e82dfce2dbda910b27b0089db42 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:20:15 +0100 Subject: [PATCH 37/80] Add header python env --- scripts/obstool.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/obstool.py b/scripts/obstool.py index a885b22..b106222 100755 --- a/scripts/obstool.py +++ b/scripts/obstool.py @@ -1,4 +1,5 @@ -#!/usr/bin/python +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- import os, sys import pandas as pd from datetime import datetime From 512c6b89535785a9507d4bff5fa449390dd1bff0 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:21:03 +0100 Subject: [PATCH 38/80] Module to handle the obstool dataframes --- modules/handle_df.py | 115 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100755 modules/handle_df.py diff --git a/modules/handle_df.py b/modules/handle_df.py new file mode 100755 index 0000000..7a07a50 --- /dev/null +++ b/modules/handle_df.py @@ -0,0 +1,115 @@ +#-*- coding:utf-8 -*- +import gc +import pandas as pd +import numpy as np +from itertools import repeat + + +from io_base import DataIO + + + +class SplitDf: + """ + Class :Split and subset DF + """ + + def __init__ (self, df , var , cdtg , max_dist =None , bin_dist =None , time_int=None ): + self.max_dist = max_dist # Maxumum distance for binning in [Km] + self.bin_int = bin_dist # Binning interval in [Km] + self.time_int = time_int # Time between OmG/OmA pairs in [+/- min] + self.dist_df = df + self.var = var + self.cdtg = cdtg + return None + + def SubsetDf (self , bin_dist , max_dist ): + dbin = [0,1]+list(np.arange(bin_dist, max_dist + bin_dist , bin_dist )) + dlabel = [0 ]+list(np.arange(bin_dist, max_dist + bin_dist , bin_dist )) + ldist_bin= pd.cut( self.dist_df ["dist"], bins=dbin,labels=dlabel, right=True,include_lowest=True ) + + # DIVIDE BY DIST INTERVALS + df_dist = self.dist_df[self.dist_df ["dist"] <= max_dist ] + df_dist ["ldist"] = ldist_bin.astype("int32") + + # Prepare Desroziers /H.L statistics (departres cross product ) + stat_df = df_dist.assign( + AFGsqr = df_dist.OA1 * df_dist.FG2, + FGsqr = df_dist.FG1 * df_dist.FG2, + FGsqr1 = df_dist.FG1 * df_dist.FG1, + FGsqr2 = df_dist.FG2 * df_dist.FG2, + Asqr1 = df_dist.OA1 * df_dist.OA1, + A1F1 = df_dist.OA1 * df_dist.FG1 ) + + # Binning + spdf = ( stat_df.groupby("ldist", observed=True).agg( + Asum1 = ("OA1" , "sum"), + FGsum1 = ("FG1" , "sum"), + FGsum2 = ("FG2" , "sum"), + AFGsqr = ("AFGsqr", "sum"), + FGsqr = ("FGsqr" , "sum"), + num = ("dist" , "count"), + FGsqr1 = ("FGsqr1", "sum"), + FGsqr2 = ("FGsqr2", "sum"), + Asqr1 = ("Asqr1" , "sum") ).reset_index() ).round(4) + + # Add var and date + spdf["var" ]=[ self.var for v in range(len( spdf["num"] ) ) ] + spdf["date"]=[ pd.to_datetime( str(self.cdtg), format="%Y%m%d%H", errors="raise") for v in range(len( spdf["num"] ) ) ] + return spdf + + +class ConcatDf: + """ + Class :Concat DF + Returns a concatenated dataframes for one variable and the whole period + """ + + def __init__(self ): + return None + + def ConcatFromListe (self, sub_df ): + merged_dict={} + merged_df =pd.DataFrame () + for k , v in sub_df.items(): + if len(v) !=0 : + merged_df =pd.concat ( v ) + merged_dict[k]=merged_df.reset_index().drop(columns =["index"]) + return merged_dict + + +class GroupDf: + """ + Class : Group DF and produce the pre-stat for cov, cor and sigma computation + """ + def __init__(self ): + return None + + def GroupByBins(self, merged_df, max_dist, bin_int): + # Set dist and max dist + self.max_dist = max_dist + self.bin_int = bin_int + + # Binning at the middles of the intervals + d_bins = np.arange(0, max_dist + bin_int, bin_int) + d_label = np.arange(bin_int/2, max_dist, bin_int) + merged_df["DIST"] = pd.cut(merged_df["ldist"], bins=d_bins, labels=d_label, include_lowest=False ) + + # Group/sum by dist bins + g = merged_df.groupby("DIST", observed=True) + d1 = g["Asum1" ].sum() + d2 = g["FGsum1"].sum() + d3 = g["FGsum2"].sum() + d4 = g["AFGsqr"].sum() + d5 = g["FGsqr" ].sum() + d6 = g["FGsqr1"].sum() + d7 = g["FGsqr2"].sum() + d8 = g["Asqr1" ].sum() + dobs = g["num" ].sum() + + # Get var and period + var = merged_df["var" ].iloc[ 0] + dt1 = merged_df["date"].iloc[ 0] + dt2 = merged_df["date"].iloc[-1] + return d1, d2, d3, d4, d5, d6, d7, d8, dobs, d1.index, dt1, dt2, var + From 296554eec5434e1d1751d25d1a68e5959e8efb8d Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:21:34 +0100 Subject: [PATCH 39/80] Compute the Hollingwoorth-Lonneberh and Desroziers stats --- modules/conv_stats.py | 267 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100755 modules/conv_stats.py diff --git a/modules/conv_stats.py b/modules/conv_stats.py new file mode 100755 index 0000000..743f1b6 --- /dev/null +++ b/modules/conv_stats.py @@ -0,0 +1,267 @@ +#-*-coding:utf-8 -*- +import os +import pandas as pd +import numpy as np +from collections import defaultdict + +# obstool modules +from handle_df import GroupDf + + + +class DHLStat: + """ + Class : Compute the diffrent statistics + covariance, correlation and standard deviations + from FG1, FG2,OA1 and OA2 departures in observation space + + Following the methods : + Desroziers , Hollingsworth,Lonneberg ( DHL ) + + functions : + getCov + getSig + getCor + """ + def __init__(self, df, max_dist =100, bin_dist =10 , delta_t =60 ): + + # Stats + # Get the values in the middle of the bins (the maximum distance and bin interval could be reset !!) + # DEFAULT VALUES + self.dist_max=100 #Km + self.bin_int =10 #Km + + # Overwrite if different values + if max_dist != 100: + self.dist_max = max_dist + print("New value for maximum distance has been set. Maximum distance value = {} Km".format( self.dist_max ) ) + else: + print("Default value for maximum distance is used : {} Km".format( self.dist_max ) ) + + if bin_dist != 10 : + self.bin_int = bin_dist + print("New binning distance has been set. bin_interval= {} Km".format( self.bin_int) ) + else: + print("Default value for binning interval is used : {} Km".format( self.bin_int ) ) + + self.gp =GroupDf () + self.merged_df=df + return None + + + + def getCov(self , var , inplace=None ): + d1,d2, d3, d4, d5, d6, d7, d8 , dobs , _ , dt1, dt2 , var =self.gp.GroupByBins (self.merged_df, self.dist_max, self.bin_int ) + + # Bins distances + dist_list = d1.index.to_list() + + # Var list and period + lvar = [var]* len(dist_list) + lperiod = [ str(dt1) +"_"+str(dt2) for _ in range(len( dist_list )) ] + + # Nobs + nobs=list(dobs ) + + # HL (Holingsworth-LÖnnberg ) + t1 = np.divide (d5 ,dobs) + t2 = np.divide ( np.multiply(d2 ,d3 ) , np.power( dobs, 2 )) + cov_hl= np.subtract(t1, t2 ) + + # Desroziers B + tb1 = np.divide( d4, dobs ) + tb2 = np.divide( np.multiply(d1, d3 ) , np.power(dobs, 2 ) ) + drb_ =np.subtract( tb1, tb2 ) + cov_drB =np.subtract(cov_hl , drb_ ) + + # Desroziers R + tr1 = np.divide ( d4 , dobs ) + tr2 = np.divide (np.multiply( d1 , d3 ) , np.power(dobs , 2 ) ) + cov_drR = np.subtract(tr1 , tr2 ) + + # Covariances + df_cov = pd.DataFrame ({ + "dist" :dist_list, + "nobs" :nobs , + "var" :lvar , + "period" :lperiod , + "COV_HL" :cov_hl , + "COV_DR-B" :cov_drB , + "COV_DR-R" :cov_drR } ).astype( { + "nobs" :"int32" , + "var" :"category", + "period" :"category", + "COV_HL" :"float64" , + "COV_DR-B":"float64" , + "COV_DR-R":"float64" + }) + + df_cov = df_cov.reset_index(drop=True).dropna(inplace=False ).reset_index() + if inplace ==True : + return cov_hl , cov_drB, cov_drR # inside the class + else: + return df_cov + + + def getSig ( self , var , inplace =None ): + + d1,d2, d3, d4, d5, d6, d7, d8 , dobs , _ , dt1, dt2 , var =self.gp.GroupByBins (self.merged_df , self.dist_max ,self.bin_int ) + + # Bins distances + dist_list = d1.index.to_list() + + # Var list and period + lvar = [var]* len(dist_list) + lperiod = [ str(dt1) +"_"+str(dt2) for _ in range(len( dist_list )) ] + + # Nobs + nobs=list(dobs ) + + # SIGMA FG1 + st1=np.divide( d6 , dobs) + st2=np.divide( d2 , dobs)**2 + sigma_fg1=np.sqrt( np.subtract(st1, st2 )) + del st1 , st2 + + # SIGMA FG2 + st1=np.divide( d7 ,dobs ) + st2=np.divide( d3 ,dobs )**2 + sigma_fg2=np.sqrt( np.subtract(st1, st2 )) + del st1, st2 + + # SIGMA A1 + st1=np.divide( d8 ,dobs ) + st2=np.divide( d1 ,dobs )**2 + sigma_a1=np.sqrt( np.subtract(st1, st2 )) + df_sig =pd.DataFrame({ + "dist" :dist_list , + "nobs" :nobs , + "var" :lvar , + "period" :lperiod , + "sigma_fg1": sigma_fg1 , + "sigma_fg2": sigma_fg2 , + "sigma_a1" : sigma_a1 + }).astype( { "nobs" :"int32" , + "var" :"category", + "period" :"category", + "sigma_fg1":"float64" , + "sigma_fg2":"float64" , + "sigma_a1" :"float64" + }) + + df_sig = df_sig.reset_index(drop=True).dropna(inplace=False ).reset_index() + + + if inplace ==True: + return sigma_fg1 , sigma_fg2, sigma_a1 # used inside the class + else: + return df_sig + + + def getCor (self,var , inplace=None ): + d1 ,_, _ , _ , _ , _ , _ , _ , dobs , _ , dt1, dt2 , var =self.gp.GroupByBins (self.merged_df , self.dist_max ,self.bin_int ) + + # Bins distances + dist_list = d1.index.to_list() + + # Var list and period + lvar = [var]* len(dist_list) + lperiod = [ str(dt1) +"_"+str(dt2) for _ in range(len( dist_list )) ] + + # Nobs + nobs=list(dobs ) + + # COV & Sigma + cov_hl, cov_drB, cov_drR =self.getCov ( var , inplace =True ) + sfg1 , sfg2 , sa1 =self.getSig ( var , inplace =True ) + + + # H.L and DRZ CORRELATIONS + cor_hl =np.divide( cov_hl , np.multiply( sfg1 , sfg2 ) ) + cor_drB =np.divide( cov_drB, np.multiply( sfg1 , sfg2 ) ) + cor_drR =np.divide( cov_drR, np.multiply( sa1 , sfg2 ) ) + + + df_cor=pd.DataFrame({ + "dist" : dist_list , + "nobs" : nobs , + "var" : lvar , + "period" : lperiod , + "COR_HL" : cor_hl , + "COR_DR-R" : cor_drR , + "COR_DR-B" : cor_drB + } ).astype( { "nobs":"int32" , + "var" :"category", + "period":"category", + "COR_HL" :"float64" , + "COR_DR-R" :"float64" , + "COR_DR-B" :"float64" + }) + + + df_cor = df_cor.reset_index(drop=True).dropna(inplace=False ).reset_index() + + if inplace ==True: + return cor_hl , cor_drB , cor_drR + else: + return df_cor + + + def getStatFrame (self , var ): + d1 ,_, _ , _ , _ , _ , _ , _ , dobs , _ , dt1, dt2 , var =self.gp.GroupByBins (self.merged_df, self.dist_max ,self.bin_int ) + + # Bins distances + dist_list = d1.index.to_list() + + # Var list and period + lvar = [var]* len(dist_list) + lperiod = [ str(dt1) +"_"+str(dt2) for _ in range(len( dist_list )) ] + + # Nobs + nobs=list(dobs ) + + # Gather all Statistics + cov_hl, cov_drB, cov_drR =self.getCov ( var , inplace =True ) + cor_hl, cor_drB, cor_drR =self.getCor ( var , inplace =True ) + sigma_fg1,sigma_fg2,sigma_a1 =self.getSig ( var , inplace =True ) + + # DataFrame: Contains the Desroziers , Hollingsworth/Lonnberg Cov, Sigma and Corr + drhl_frame={ + "dist" :dist_list , + "nobs" :nobs , + "var" :lvar , + "period" :lperiod , + "COV_HL" :cov_hl , + "COV_DR-B" :cov_drB , + "COV_DR-R" :cov_drR , + "sigma_FG1":sigma_fg1 , + "sigma_FG2":sigma_fg2 , + "sigma_a1" :sigma_a1 , + "COR_HL" :cor_hl , + "COR_DR-R" :cor_drR , + "COR_DR-B" :cor_drB + } + + # Return statistics DF + # Remark that the dataframe is rounded to 4 decimal digits + # It's done to allow a consistent comparison with the values given by R code + stat_frame =pd.DataFrame ( drhl_frame ).astype({ + "nobs":"int32" , + "var" :"category", + "period":"category", + "COV_HL" :"float64" , + "COV_DR-R" :"float64" , + "COV_DR-B" :"float64" , + "sigma_FG1":"float64" , + "sigma_FG2":"float64" , + "sigma_a1" :"float64" , + "COR_HL" :"float64" , + "COR_DR-R" :"float64" , + "COR_DR-B" :"float64" + }).round(4) + + + + stat_frame = stat_frame.dropna(inplace=False ).reset_index(drop=True) + return stat_frame + # To be plotted !!! From 499cdc191fd80258ec17c289ce756c819769e783 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:22:08 +0100 Subject: [PATCH 40/80] Build the needed sql query for obstool --- modules/build_sql.py | 148 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100755 modules/build_sql.py diff --git a/modules/build_sql.py b/modules/build_sql.py new file mode 100755 index 0000000..ecb4c2d --- /dev/null +++ b/modules/build_sql.py @@ -0,0 +1,148 @@ +# -*- coding:utf-8 -*- +import os ,sys +from odb4py.utils import SqlParser + + + +class SqlHandler: + """ + Class : Parse the different observation dictionaries and build the + SQL query according to obstype, codetype, varno , sensor and level range + Checks the SQL statement before sending. + Returns :The sql statement according to variable and varno + + Methods : BuildQuery + CheckQuery + + + """ + def __init__(self ): + + # For the moment ! + self.obstool_select=None + + #self.derozier_select=None + #self.jarvinen_select=None + return None + + + def BuildQuery(self , **kwarg ): + self.cols =",".join(kwarg["columns" ])[1:] + self.tabs =",".join(kwarg["tables" ]) + self.obs_ =kwarg["obs"] + self.varno =kwarg["obsvano"] + self.ctype =kwarg["codetype"] + self.obst =kwarg["obstype"] + self.levels =kwarg["lrange"] + self.vtype =kwarg["vertco_type"] + self.vert_coord=kwarg["vertco"] + self.sensor =kwarg["sensor" ] + self.other =kwarg["remaining_sql"] + + # Obstool columns and tables + self.obstool_select="SELECT "+self.cols+" FROM " +self.tabs + + # Check obstype + type_list=[] + if self.obst is not None: + if isinstance (self.obst, list ): + for tp in self.obst: + if tp is not None: + type_list.append (" obstype =="+str( tp ) ) + elif isinstance (self.obst, int ): + type_list.append (" obstype =="+str( self.obst ) ) + + else: + print(self.obs_, " : obstype must be integer or a list" ) + sys.exit() + + + + # Check varno + varno_list=[] + if self.varno is not None: + if isinstance (self.varno , list ) : + for vr in self.varno: + if vr is not None: + varno_list.append (" varno =="+str( vr ) ) + elif isinstance( self.varno , int ): + varno_list.insert(0, " varno =="+ str(self.varno) ) + + else: + print(self.obs_ , ": varno must be integer ot list" ) + sys.exit() + + # Check codetype + """ + Not needed for the moment ! + ctype_list=[""] + if self.ctype is not None: + if isinstance( self.ctype, list ): + for ct in self.ctype: + if ct is not None: + ctype_list.append (" codetype =="+str( ct ) ) + elif isinstance( self.ctype , int ): + ctype_list.append( " codetype =="+str(self.ctype ) ) + else: + print(self.obs_, ": codetype must be integer or list " ) + sys.exit()""" + + + # Check sensor + sensor_list =[] + if self.sensor != None: + if isinstance (self.sensor , list ): + for s in self.sensor: + if s is not None: + sensor_list.append( " sensor=="+str(s) ) + elif isinstance (self.sensor , int ): + sensor_list.append( " sensor =="+str( self.sensor ) ) + else: + print(self.obs_, ": sensor must be integer or list" ) + sys.exit() + + + # Check level range + if self.levels != None : + if len (self.levels) <2 or len( self.levels ) > 2: + print ( "Level range must have two limites [l1 , l2], but got argument:", self.levels ) + sys.exit(0) + elif not isinstance (self.levels[0], int ) or not isinstance ( self.levels [1], int): + print( "One or the both level limites in the list is/are not intger(s)" ) + sys.exit(0) + elif self.levels[1] < self.levels[0]: + print( "Bottom level limit can't be greater than top level limit" ) + sys.exit(0) + else: + l1 =str(self.levels[0] ) + l2 =str(self.levels[1] ) + level_cond="vertco_reference_1 >="+l1+" AND vertco_reference_1 <= "+l2 + + + where_cond_list=[] + for tp in type_list: + for vr in varno_list: + where_cond_list.append( " AND ".join( (tp, vr )) ) + + + query=" " + if self.other !=None and len(where_cond_list) == 0: + query= self.obstool_select +" WHERE "+self.other + + elif self.other !=None and len(where_cond_list) > 0: + for condition in where_cond_list: + query=self.obstool_select +" WHERE " + condition + " AND "+self.other + + elif self.other ==None and len(where_cond) > 0: + for condition in where_cond_list: + query= self.obstool_select +" WHERE " + condition + + return query + + + + def CheckQuery(self , query ): + p =SqlParser() + nfunc =p.get_nfunc ( query ) # N Columns=N pure columns - N functions in the query + sql =p.clean_string ( query ) # Remove any unprintable character ( \r\t ....etc) + return nfunc , sql From 5ff7c4f373c314df21bd29b94f4b9750a13f960e Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:22:55 +0100 Subject: [PATCH 41/80] All needed info about the obstypes are defined here --- modules/obstype_info.py | 294 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100755 modules/obstype_info.py diff --git a/modules/obstype_info.py b/modules/obstype_info.py new file mode 100755 index 0000000..8d900de --- /dev/null +++ b/modules/obstype_info.py @@ -0,0 +1,294 @@ +# -*-coding:utf-8 -*- +import sys , os + + +class ObsType: + """ + Class: Contains the variable names and their attributes + NOTE : ONLY THE CONV DATA ARE PROCESSED IN THIS VERSION ! + + Methods: + ConvDict + RenameVarno + SelectConv + """ + + def _init__(self): + # ObsType lists + # Init the observation lists : Conventional and Satem + self.conv_obs=[ + 'gpssol' , + 'synop' , + 'dribu' , + 'airep' , + 'airepl' , + 'radar' , + 'temp' , + 'templ' ] + + self.sat_obs = [ + 'amsua' , + 'amsub' , + 'atms' , + 'iasi' , + 'mwhs' , + 'msh' , + 'seviri' ] + return None + + def ConvDict(self) : + # CONVENTIONAL + self.obs_conv=[ + + + # MAYBE WILL BE CHANGED TO A LIST AND LOOP ! + # SURFACE + {"obs_name" : "gpssol", + "obstype" : 1 , + "codetype" : 110 , + "varno" : 128 , + "vertco_reference_1": None, + "sensor" : None, + "level_range" : None } , + { "obs_name" : "synop_z", + "obstype" : 1 , + "codetype" : [11, 14, 170, 182] , + "varno" : 1, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : None }, + { "obs_name" : "synop_v", + "obstype" : 1 , + "codetype" : [11, 14, 170, 182] , + "varno" : 42, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : None }, + { "obs_name" : "synop_u", + "obstype" : 1 , + "codetype" : [11, 14, 170, 182] , + "varno" : 41, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : None }, + { "obs_name" : "synop_h", + "obstype" : 1 , + "codetype" : [11, 14, 170, 182] , + "varno" : 58, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : None }, + { "obs_name" : "synop_t", + "obstype" : 1 , + "codetype" : [11, 14, 170, 182] , + "varno" : 39, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : None }, + { "obs_name" : "dribu_z", + "obstype" : 4 , + "codetype" : None, + "varno" : [1, 39, 41, 42], + "vertco_reference_1": None, + "sensor" : None, + "level_range" : None }, + { "obs_name" : "dribu_t", + "obstype" : 4 , + "codetype" : None, + "varno" : 39, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : None }, + { "obs_name" : "dribu_u", + "obstype" : 4 , + "codetype" : None, + "varno" : 41, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : None }, + { "obs_name" : "dribu_v", + "obstype" : 4 , + "codetype" : None, + "varno" : 42, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : None }, + + { "obs_name" : "ascat", + "obstype" : 9 , + "codetype" : None, + "varno" : None, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : None } , + # RADAR + { "obs_name" : "radar_rh", + "obstype" : 13, + "codetype" : None, + "varno" : 29, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : None }, + { "obs_name" : "radar_dow", + "obstype" : 13, + "codetype" : None, + "varno" : 195 , + "vertco_reference_1": None, + "sensor" : None, + "level_range" : None }, + + + # UPPER AIR + { "obs_name" : "airep_t", + "obstype" : [2] , + "codetype" : None, + "varno" : [2] , + "vertco_reference_1": None, + "sensor" : None, + "level_range" : None }, + { "obs_name" : "airep_u", + "obstype" : 2 , + "codetype" : None, + "varno" : 3, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : None }, + { "obs_name" : "airep_v", + "obstype" : 2 , + "codetype" : None, + "varno" : 4 , + "vertco_reference_1": None, + "sensor" : None, + "level_range" : None }, + { "obs_name" : "airepl_t" , + "obstype" : 2 , + "codetype" : None, + "varno" : 2, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : [25000, 35000] }, + { "obs_name" : "airepl_u" , + "obstype" : 2 , + "codetype" : None, + "varno" : 3, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : [25000, 35000] }, + { "obs_name" : "airepl_v" , + "obstype" : 2 , + "codetype" : None, + "varno" : 4, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : [25000, 35000] }, + { "obs_name" : "temp_t", + "obstype" : 5, + "codetype" : None, + "varno" : 2, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : None }, + { "obs_name" : "temp_u", + "obstype" : 5, + "codetype" : None, + "varno" : 3, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : None }, + { "obs_name" : "temp_v", + "obstype" : 5, + "codetype" : None, + "varno" : 4, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : None }, + { "obs_name" : "temp_q", + "obstype" : 5, + "codetype" : None, + "varno" : 7, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : None }, + { "obs_name" : "templ_t" , + "obstype" : 5 , + "codetype" : None, + "varno" : 2, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : [40000, 60000]} , + { "obs_name" : "templ_u" , + "obstype" : 5 , + "codetype" : None, + "varno" : 3, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : [40000, 60000]} , + { "obs_name" : "templ_v" , + "obstype" : 5 , + "codetype" : None, + "varno" : 4, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : [40000, 60000]} , + { "obs_name" : "templ_q" , + "obstype" : 5 , + "codetype" : None, + "varno" : 7, + "vertco_reference_1": None, + "sensor" : None, + "level_range" : [40000, 60000] } ] + + + # Varno dict + self.varno_dict ={ 128 : "ztd" , # gpssol Zenith total delay + 1 : "z" , # geopotential + 42 : "u" , # 10m wind speed u + 41 : "v" , # 10m wind speed v + 58 : "h" , # 2 relative humidity + 39 : "t" , # 2 meter temperature + 2 : "t" , # T temperature (upper air) + 3 : "u" , # U component of wind (upper air) + 4 : "v" , # V component of wind (upper air) + 7 : "q" , # specific humidity + 29 : "rh" , # upper rh (radar) + 195 : "dw" # Dopp radial wind + } + + # Add Units + self.unit= { + 128 : "m" , # gpssol Zenith total delay + 1 : "mgp" , # geopotential + 42 : "m/s" , # 10m wind speed u + 41 : "m/s" , # 10m wind speed v + 58 : "%" , # 2 relative humidity + 39 : "K" , # 2 meter temperature + 2 : "K" , # T temperature (upper air) + 3 : "m/s" , # U component of wind (upper air) + 4 : "m/s" , # V component of wind (upper air) + 7 : "g/kg" , # specific humidity + 29 : "%" , # upper rh (radar) + 195 : "m/s" # Dopp radial wind + } + return self.obs_conv , self.varno_dict , self.unit + + + + def RenameVarno (self, string_var ): + dict_={} + obslist , dict_ = self.ConvDict() + obsname=[ dobs["obs_name"] for dobs in obslist ] + name = string_var.split("_")[0] + vr =int(string_var.split("_")[1] ) + vname =name + "_"+ dict_[vr] + return vname + + + def SelectConv(self, list_ ): + varobs=[] + if not isinstance ( list_ ): + print("Parameter list must be a python list") + sys.exit () + self.list = list_ + obs_list, var_dict , var_unit =self.ConvDict() + + From f0048341f88239f071a738cc23df924eb4f5d5b7 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:23:19 +0100 Subject: [PATCH 42/80] Basic module for performing I/O --- modules/io_base.py | 90 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100755 modules/io_base.py diff --git a/modules/io_base.py b/modules/io_base.py new file mode 100755 index 0000000..712c790 --- /dev/null +++ b/modules/io_base.py @@ -0,0 +1,90 @@ +# -*-coding :utf-8 -*- +import os , sys +import pandas as pd +import numpy as np +from pathlib import Path + + + +class DataIO: + """ + Class: DataIO : Contains methods to write and read the statstics dataframes + for each cycle (in .csv files) + It uses the format 'feather'. Fast for I/O Compatible with pandas + Methods : + FlushFrame + ReadFrame + + """ + def __init__(self, compression="lz4"): + # If binary format is used , feather is the best for compression + # Write frames into feather format file with either "lz4" or "zstd" compression + # We use ".csv" for the moment + self.compression = compression + if self.compression not in ["ls4", "zstd"]: + self.compression = "ls4" # Fallback to default + + def FlushFrame(self, df, subdir, cdtg, var, fid=None, verbose =None, filetype=None ): + """ + Flush a DataFrame to Feather format with lz4 compression algo'. + Naming based on var, cdtg + """ + vrb = verbose + if df is None or df.empty: + print("Warning : Empty DataFrame for var={} datatime={}, nothing written".format( var , cdtg ) ) + return None + + # Final directory: dbpath / subdir / cdtg + outdir = Path(subdir) / cdtg + outdir.mkdir(parents=True, exist_ok=True) + if fid is not None: + filename = "_".join( (var,cdtg,fid ) ) +".csv" + else: + filename = "_".join( (var,cdtg)) +".csv" + + # Dir+Filename + filepath = "/".join( (str(outdir) , str(filetype) +"_"+str(filename) ) ) + try: + df.to_csv(filepath, index = False , sep=",") + if vrb in [3]: + print( "The dataframe has been written into {} for var: {} , datetime: {}".format(os.path.basename(filepath) , var , cdtg )) + except Exception: + print( "ERROR writing dataframe : var: {} and datetime: {}".format( var , cdtg )) + return None + return str(filepath) + + + + + + def ReadFrame(self, filepaths , verbose =None ): + """ + Read files. If var is provided, filters only matching files. + Returns a single concatenated DataFrame. + """ + vrb = verbose + frames = [] + if isinstance( filepaths, str ): + df = pd.read_csv( filepaths , header='infer', engine='python', sep=",") + return df + + elif isinstance (filepaths, list ): + for fp in filepaths: + if not os.path.isfile (fp): + print("File not found : " , fp ) + frames.append( pd.DataFrame() ) + else: + df = pd.read_csv( fp , header='infer', engine='python', sep=",") + if vrb in [3]: + print( "Dataframe loaded with : {} rows)".format( df.shape[0] )) + frames.append(df) + + if not frames: + return None + + # Concatenate all chunks + all_df = pd.concat(frames, axis=0, ignore_index=True) + print( "Total: {} rows merged".format( all_df.shape[0] )) + return all_df + + From e7394f571fb107012672f9a1ea8656dcdcc923aa Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:24:50 +0100 Subject: [PATCH 43/80] python modules __ini__ file --- modules/__init__.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/modules/__init__.py b/modules/__init__.py index f327247..e69de29 100755 --- a/modules/__init__.py +++ b/modules/__init__.py @@ -1,14 +0,0 @@ -# -*- coding :utf-8 -*- - - -from modules.sigma_bo import Predef, Diag , Ratios -from modules.config_env import TuneEnv -from modules.read_odb import Odb -from modules.gsacov import GSA - -__all__ = [ "Predef" , - "Diag" , - "Ratios" , - "TuneEnv", - "Odb" , - "GSA" ] From fbf3f264496b249d3b7635236f92a4d23520bf70 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:25:47 +0100 Subject: [PATCH 44/80] Set the obslist , the period and other vars according to the obstool.py arguments --- modules/setting.py | 149 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100755 modules/setting.py diff --git a/modules/setting.py b/modules/setting.py new file mode 100755 index 0000000..234dd1a --- /dev/null +++ b/modules/setting.py @@ -0,0 +1,149 @@ +# -*- coding:utf-8 -*- +import os , sys +from datetime import datetime ,timedelta + +# Obstool modules +from obstype_info import ObsType + + +class Setting: + """ + class : contains methods to set datetime period and varlist for obstool + Methods : (Better to use snake_case names for the methods !) + set_period + set_obs_list + """ + def __init__ (self ): + self.obt = ObsType () + + def set_period(self, bdate , edate , cycle_inc=3 ): + # Create datetime liste + if not isinstance (bdate, str) or not isinstance ( edate , str): + btype, etype = type(bdate ) , type( edate) + print("Start and end dates period argument must be strings. Got {} {} ".format( btype, etype ) ) + sys.exit(0) + + if len( bdate) != 10: print("Malformatted start date") ; sys.exit(1) + if len( edate) != 10: print("Malformatted end date") ; sys.exit(2) + period=[] + bdate =datetime.strptime( bdate , "%Y%m%d%H") + edate =datetime.strptime( edate , "%Y%m%d%H") + delta =timedelta(hours=int(cycle_inc)) + while bdate <= edate: + strdate=bdate.strftime("%Y%m%d%H") + period.append( strdate ) + bdate += delta + return period + + + def set_obs_list(self, obs_list ): + obsdict , obsvarno , units = self.obt.ConvDict() + if len(obs_list ) == 0 : + print("Can not process empty observation liste. At least one obstype is required" ) + sys.exit(1) + + if not isinstance ( obs_list , list): + print("Observation types variable expects liste." ) + sys.exit(2) + + + obs_name =[ item["obs_name"] for item in obsdict ] + for obs in obs_list: + if obs.strip() not in obs_name: + print("Can t process the given obstype {}. Not predefined in the list of conventional obstypes".format( obs )) + sys.exit(3) + + # Obs attributes + obs_names=[] + codetype=[] + obstype =[] + varno =[] + lrange =[] + vertco =[] + sensor =[] + for item in obsdict: + obsname = item["obs_name"] + if obsname in obs_list: + obs_names.append(item["obs_name"]) + varno.append (item["varno" ]) + codetype.append (item["codetype"]) + obstype.append (item["obstype" ]) + lrange.append (item["level_range"]) + vertco.append (item["vertco_reference_1"]) + sensor.append (item["sensor" ] ) + return obs_names , codetype , obstype , varno , lrange , vertco ,sensor + + + + +# TODO +# class Satem: +# def __init__(): +# pass +# DATA CETEGORIES SATEM +# What we need ? +# SATEM +# NAME : sensor +# AMSUA : 3 +# AMSUB : 4 +# MHS :15 +# IASI :16 +# ATMS :19 +# MWHS :73 +# SEVIRI :29 +# return None + + + +# CONV OBSTYPE LIST +# obs varno +# gpssol : 128 +# synop : 1,29,58,41,42,7,39 +# dribu : +# airep : 2,3,4 +# airepl : 2,3,4 +# radar : 29,195 +# temp : 2,3,4,39,58,41,42 +# templ : 2,3,4,39,58,41,42 + +class Conv: + """ + class : instantiate Conv obs category and corresponding sql env + """ + def __init__(self ): + # ---> CAUTION !: NEVER CHANGE THE ORDER OF COLUMNS + # TO ADD NEW ONEs , ONE CAN DO ONLY AN APPEND ! + self.cols =[ "", + "obstype" , # 1 + "codetype" , # 2 + "statid" , # 3 + "varno" , # 4 + "degrees(lat)" , # 5 + "degrees(lon)" , # 6 + "vertco_reference_1" , # 7 + "date" , # 8 + "time@hdr" , # 9 + "an_depar" , # 10 + "fg_depar" , # 11 + "obsvalue" , # 12 + ] + + # Considered tables & additional sql statement + ODB_CONSIDER_TABLES="/hdr/desc/body/radar/radar_body" + self.tables = ["hdr","desc","body",] #"errstat", "radar_body"] + self.tbl_env = "/".join( self.tables ) + self.other_sql = " (an_depar is not NULL) AND (fg_depar is not NULL) AND (datum_event1.fg2big@body = 0)" + + # ObsType list + self.conv_obs=[ 'gpssol' , + 'synop' , + 'dribu' , + 'airep' , + 'airepl' , + 'radar' , + 'temp' , + 'templ' ] + obstype = ObsType() + self.obs, self.varno , self.unit = obstype.ConvDict() + return None + From a81b147cbda6962d4e3b726700581d7f5bbd5aaa Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:26:37 +0100 Subject: [PATCH 45/80] Contains the methods called by obstool.py --- modules/utils.py | 482 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 482 insertions(+) create mode 100755 modules/utils.py diff --git a/modules/utils.py b/modules/utils.py new file mode 100755 index 0000000..a17c180 --- /dev/null +++ b/modules/utils.py @@ -0,0 +1,482 @@ +# -*-coding :utf-8 -*- +import os , sys , gc +import pandas as pd +import numpy as np +from pathlib import Path +from collections import defaultdict +import multiprocessing as mp +from multiprocessing import Pool , cpu_count, shared_memory +from itertools import chain + + +# odb4py +from odb4py.utils import SqlParser , OdbObject +from odb4py.core import odb_dca, odb_open , odb_close ,odb_dict, odb_gcdist + +# Obstool, Desroziers & Jarvinen , Tools & modules +from build_sql import SqlHandler +from obstype_info import ObsType +from setting import Setting , Conv +from handle_df import * +from io_base import DataIO + + + + +class DCAFiles: + """ + Class : Checks and creates the DCA (Direct Access Column )files + if they are not already in the ODB + If the DCA files are already there , this class is not called + Methods : + CheckDca + + """ + def __init__(self): + return None + + def CheckDca( self, dbpath , sub_base=None , verbose = False ): + # Prepare DCA files if not in ODB + db = OdbObject ( dbpath ) + dbname = db.get_attrib()["name"] + + if not os.path.isdir ("/".join( [dbpath , "dca"] ) ): + if verbose in [2,3 ]: + print( "No DCA files in {} 'directory'".format(dbpath ) ) + #env.OdbVars["CCMA.IOASSIGN"]="/".join( (dbname, "CCMA.IOASSIGN" ) ) + #env.OdbVars["ECMA.IOASSIGN"]="/".join( (dbname, "ECMA.IOASSIGN" ) ) + status = odb_dca ( dbpath=dbpath , db=dbname , ncpu=8 ) + if status < 0 : + print("Failed to create DCA files ... \n Another attempt will be done with odb_dict !" ) + else : + if verbose ==True : + print("DCA files already in database: '{}'".format( dbname ) ) + else: + pass + + + + + +class OdbReader: + """ + Class : Prepare the query according to the user setting (Obs list , period etc) + The SQL query is sent directly to the ECMA or CCMA ODB . + + Returns data are as python dictionary + Methods : get_odb_rows + + """ + def __init__(self , dbpath ,type_ ): + # Path to the directory containing the ODB(s) + self.odb_path = dbpath + self.odb_type = type_ + self.vrb = 1 + if not os.path.isdir (self.odb_path): + print("ODB(s) directory '{}' not found.".format( self.odb_path )) + sys.exit(0) + + # Setting + self.st = Setting () + + # DCA Files + self.dca_f = DCAFiles() + + # CONV + self.conv = Conv() + self.conv_obs = self.conv.conv_obs + self.cols = self.conv.cols + self.tables = self.conv.tables + self.other_sql = self.conv.other_sql + + # IO + self.io = DataIO () + # SQL + self.sql =SqlHandler() + + # DF + self.rd =Rows2Df () + self.cnt =ConcatDf() + + # List of odb rows + self.list_rows =[] + + # return a list of dicts + self.dlist = defaultdict(list) + + def get_odb_rows (self, period , + obs_list , + max_dist , + bin_dist , + #file_io , later ! + file_path , + cycle_inc =3, + pbar =False , + verbosity =0, + chunk_size = None + ): + + vrb=verbosity + if vrb not in [0,1,2,3]: + print("Min and max verbosity levels: 0 -> 3. Got : ", vrb ) + print("Fallback to default value: verbosity=", self.vrb ) + + + # Default dict (Collect dataframes with variable as a key ) + df_vars = defaultdict(list) + + # Diags period + period_interval= [ min( period ) , max(period) ] + + # ODB Paths + paths = [ "/".join( (self.odb_path , prd , self.odb_type )) for prd in period ] + + # Obs attributes + obs_names , codetype , obstype , varno , lrange , vertco ,sensor =self.st.set_obs_list(obs_list ) + + list_dict=[] + for i, cma_path in enumerate(paths): + + # Open ODB + conn= odb_open ( cma_path ) + + # UPDATE IOASSIGN , ODB_SRCPATH & ODB_DATAPATH + os.environ["IOASSIGN"]=cma_path+"/IOASSIGN" + os.environ["ODB_SRCPATH_CCMA"] =cma_path + os.environ["ODB_DATAPATH_CCMA"]=cma_path + os.environ["ODB_IDXPATH_CCMA" ]=cma_path + + # Check DCA directory (if not there they will be created ) + dca_f=DCAFiles() + dca_f.CheckDca ( cma_path ) + + if vrb == [ 2 , 3]: + print("ODB PATHS set to :") + print("IOASSIGN :",cma_path+"IOASSIGN" ) + print("ODB_SRCPATH_CCMA :",cma_path ) + print("ODB_DATAPATH_CCMA :",cma_path ) + print("ODB_IDXPATH_CCMA :",cma_path ) + + for jo, obs in enumerate(obs_names) : + query=self.sql.BuildQuery( columns =self.cols , + tables =self.tables , + obs =obs , + obstype =obstype [jo] , + obsvano =varno [jo] , + codetype =codetype [jo] , + lrange =lrange [jo] , + vertco_type ="height" , + vertco =vertco [jo] , + sensor =sensor [jo] , + remaining_sql =self.other_sql ) + + nfunc , sql_query = self.sql.CheckQuery( query) + cdtg = period [i] + if vrb in [0, 1, 2, 3]: + print( "Process observation type: {} ODB date : {} ".format( obs , cdtg )) + query_file=None ; + poolmask = None ; + pool =None ; + float_fmt= 15 ; # 10 digits float values + verbose = False ; + pbar = False + + # Progress bar & Verbosity inside pyodb + # Progress bar is useful for huge ODBs + if vrb in [3 ]: + verbose= True + if vrb in [2,3]: + pbar = True + + + # Write odb rows once , if there is a rerun of the same period + # the odb extraction is skipped + filename = "_".join( ( "df_rows" , obs ,cdtg)) +".csv" + fpath = "/".join( (self.odb_path,cdtg , filename) ) + if os.path.isfile( fpath ): + if vrb in [1,2,3]: + print("ODB rows already in file obstype: {} , date: {}".format( obs, cdtg ) ) + rows = self.io.ReadFrame( fpath ) + # Process rows from file + df_dist = self.rd.DfDist (rows, obs, cdtg, max_dist ) + + # Subset df + spl = SplitDf (df_dist , obs , cdtg ) + ndist = df_dist.dropna(subset=["dist"]).copy() + df_stat = spl.SubsetDf( bin_dist , max_dist) + self.dlist[obs].append( df_stat ) + else: + # Get and process the rows from ODB + if vrb in [ 2,3]: + print( "ODB rows not available from file :", filename ) + print( "Proceed to data extraction ...") + try: + rows= conn.odb_dict (cma_path , + sql_query , + nfunc , + float_fmt , + query_file, + poolmask , + pbar , + verbose ) + conn.odb_close() + if not rows: + print( "WARNING : Data not available for the variable {} and varno={}".format( obs , varno[jo]) ) + empty_df= pd.DataFrame([]) + self.dlist[obs].append( empty_df ) + + except: + RuntimeError + print( "WARNING : Data not available for the variable {} and varno={}".format( obs , varno[jo]) ) + empty_df= pd.DataFrame([]) + self.dlist[obs].append( empty_df ) + pass + else: + # Process rows directly if the file not there + df_dist = self.rd.DfDist (rows, obs, cdtg, max_dist ) + + # Subset df + spl = SplitDf (df_dist , obs , cdtg ) + ndist = df_dist.dropna(subset=["dist"]).copy() + df_stat= spl.SubsetDf( bin_dist , max_dist) + self.dlist[obs].append( df_stat ) + + # Save data. + io_fpath = "/".join( ( self.odb_path, cdtg )) + rows_file = "_".join( ( "df_rows" , obs ,cdtg)) +".csv" + stat_file = "_".join( ( "df_stat" , obs ,cdtg)) +".csv" + + rows_fpath = "/".join( ( self.odb_path,cdtg , rows_file) ) + stat_fpath = "/".join( ( self.odb_path,cdtg , stat_file) ) + + # Write if not done ! + if not os.path.isfile(rows_fpath ): + df_rows = pd.DataFrame( rows ) + self.io.FlushFrame( df_rows , self.odb_path, cdtg , obs, fid=None ,verbose =vrb ,filetype="df_rows") + + if not os.path.isfile (stat_file): + self.io.FlushFrame( df_stat , self.odb_path, cdtg , obs, fid=None ,verbose =vrb ,filetype="df_stat") + return self.dlist + + + + + + + +# The methods below are ouside classes +# Since they may be used by multiple processes +def CreateSharedMfile (name, size): + # Delete and recreate the shared memory file if exists + # under /dev/shm + try: + # If exists close and delete + old = shared_memory.SharedMemory(name=name) # unlink and close + old.close() + old.unlink() + except FileNotFoundError: + pass + return shared_memory.SharedMemory(name=name, create=True, size=size) + + + + +def GcdistChunkShared (i0, i1, N): + """ + Method : Worker shared memory . + Only receives i0, i1 and global N. + Accesses shared memory numpy arrays directly. + returns: The matrix chunked blocks and begin/end index + """ + + # + CreateSharedMfile + shm_lon = shared_memory.SharedMemory(name="shm_lons") + shm_lat = shared_memory.SharedMemory(name="shm_lats") + + # Recreate numpy arrays as buffers + lon = np.ndarray((N,), dtype=np.float64, buffer=shm_lon.buf) + lat = np.ndarray((N,), dtype=np.float64, buffer=shm_lat.buf) + + # Get latlon by chunk indices + sub_lon = lon[i0:i1] + sub_lat = lat[i0:i1] + + # The method for computing the great circle distances is written in C + # The C code is more or less the same as the one used in R "sp" package + # --In order to get the same values as in the original obstool , the method + # in sp package was adapted to odb4py python package (The method is called odb_gcdist) + block = odb_gcdist (sub_lon, sub_lat, lon, lat) + + # Close memory buffers handle + shm_lon.close() + shm_lat.close() + return (i0, i1, block) + + + + +class DistMatrix: + """@Class : DistMatrix: computes the interdistances between all + the latlon pairs ( for N coordinates > 3000 , the process starts to slow down) + + Approach to speed up : Divide the coordinates arrays into chuncks and + compute each block in a sparated process + + @Returns: A reconstructed matrix of all distances between latlon pairs. + + Methods : + GcdistParallel + + """ + + def __init__(self, lons, lats): + self.lons = np.asarray(lons, dtype=np.float64) # Init with 64 bits + self.lats = np.asarray(lats, dtype=np.float64) + self.N = len(self.lons) + + def GcdistParallel(self, chunk_size=200, workers=None): + N = self.N + lon = self.lons + lat = self.lats + + # If the number of CPUs is set otherwise get from the machine + if workers is None: + workers = mp.cpu_count() + + # Create shared memory file + # More Safe + shm_lons =CreateSharedMfile ( size=lon.nbytes , name="shm_lons" ) + shm_lats =CreateSharedMfile ( size=lon.nbytes , name="shm_lats" ) + + # Copy data once + np.ndarray(lon.shape, dtype=np.float64, buffer=shm_lons.buf)[:] = lon + np.ndarray(lat.shape, dtype=np.float64, buffer=shm_lats.buf)[:] = lat + + # Chunk ranges + chunk_ranges = [(i0, min(i0 + chunk_size, N)) for i0 in range(0, N, chunk_size)] + + # Init the final matrix + distmat = np.zeros((N, N), dtype=np.float64) + try: + with mp.Pool(processes=workers) as pool: + # Use starmap method and give chunk ranges as args + results = pool.starmap( GcdistChunkShared , [(i0, i1, N) for (i0, i1) in chunk_ranges]) + + # Fill the matrix by block + for (i0, i1, block) in results: + distmat[i0:i1] = block + finally: + # Cleanup and unlink memory + shm_lons.close() + shm_lons.unlink() + shm_lats.close() + shm_lats.unlink() + + return distmat + + + +class Rows2Df : + + """ + Class: Build the dataframes with O-G and O-A departures. + To speed up the computation of the distances between the + latlon pairs , the matrix is computed by chunks + + Returns a DF of O-G , O-A , indices and distances + Methods : + DfDist + DfPrep + """ + + def __init__(self): + # I/O + self.io = DataIO () + return None + + + + def DfDist (self , rows , + obs , + cdtg , + max_dist , + verbosity =0 , + write_file=None, + file_path =None, + filetype =None + ): + #pd.set_option('display.max_rows', 20 ) + vrb=verbosity + if vrb not in [0,1,2,3]: + print("WARNING : Min and max verbosity levels: 0 -> 3. Got : ", vrb ) + print("Fallback to default value: verbosity=", vrb ) + + if rows is None: + print("Rows from ODB not available for var : {}".format(var ) ) + sys.exit() + else: + df_rows = pd.DataFrame(rows ) + lats = np.array(rows["degrees(lat)" ]) + lons = np.array(rows["degrees(lon)" ]) + an_d = np.array(rows["an_depar@body"]) + fg_d = np.array(rows["fg_depar@body"]) + + # ARGS order : lons1, lats1, lon2, lat2 : Compute distances between latlon pairs + d = DistMatrix (lons , lats ) + + # The distances ar rounded to 0 decimals + # Else , there will be a number of observations sets in each bins. + # It leads to different statistics between R and Python whil perfomrming the cut , subset etc . + dist = d.GcdistParallel().round(0) #astype(int) + + N = len(lats) + dist_1d = dist.ravel() + + # Indices as in as.table(matdist) : row/col in R version + d1, d2 = np.indices((N, N)) + d1 = d1.ravel().astype(np.int32, copy=False) + d2 = d2.ravel().astype(np.int32, copy=False) + + # Invert d1, d2 to match the ones in R + ndist = pd.DataFrame({ + "n1" : d2, + "n2" : d1, + "dist": dist_1d }) + + # Subset using the max distance + df_dist = ndist[ndist["dist"] <= max_dist].copy() + + # Add OA/FG departures to corresponding indices + df_dist["OA1"] = an_d[df_dist["n1"].values] + df_dist["OA2"] = an_d[df_dist["n2"].values] + df_dist["FG1"] = fg_d[df_dist["n1"].values] + df_dist["FG2"] = fg_d[df_dist["n2"].values] + + # Set datatypes + df_dist = df_dist.astype({ + "n1": "int32", + "n2": "int32", + "dist": "int32", + "OA1": "float64", + "OA2": "float64", + "FG1": "float64", + "FG2": "float64" + }) + # For safety + df_dist = df_dist[df_dist["dist"] <= max_dist].copy() + + # Round to 4 decimal precision + return df_dist.round(4) + + + + def DfPrep (self , + frame_liste, + dir_=None , + period=None , + var_list=None ): + # Concat df by vars for the whole period + cnt= ConcatDf () + cdf= cnt.ConcatFromListe ( frame_liste) + return cdf From 02975abbb255c4bfb8e9fc4962601cb802c53bd4 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:29:30 +0100 Subject: [PATCH 46/80] Add installation of obstool modules --- modules/CMakeLists.txt | 19 ++++++++++++++++--- modules/readgsa.so | Bin 17152 -> 0 bytes 2 files changed, 16 insertions(+), 3 deletions(-) delete mode 100755 modules/readgsa.so diff --git a/modules/CMakeLists.txt b/modules/CMakeLists.txt index 8f2d623..7d45697 100644 --- a/modules/CMakeLists.txt +++ b/modules/CMakeLists.txt @@ -7,8 +7,21 @@ function (install_modules ) endforeach () endfunction() -# Python modules For tuneBR -install_modules (gsacov.py read_odb.py config_env.py sigma_bo.py __init__.py) -install_modules (readgsa.so) +# Py modules For tuneBR +install_modules (__init__.py + gsacov.py + read_odb.py + config_env.py + sigma_bo.py ) + +# Py modules for obstool +install_modules (build_sql.py + conv_stats.py + handle_df.py + io_base.py + setting.py + utils.py + obstype_info.py) + diff --git a/modules/readgsa.so b/modules/readgsa.so deleted file mode 100755 index 2892ae58a069ae26a4b7398b8696ab8e55b65a4d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17152 zcmeG^3vg7|b$2Be#K){~{E@+pUof%3ft3Ku7TBt_NGrXiD+voJp>dpLwY&N>(k|?- z_;?Z{S8T@X?Is!~)Qp?N%+%9R;u$k(a0xCQS$r1iHdfoj#hoS*IS7Y|DH>sUyb2lwqun z)v$T+dna4QWt~OVwA{p}1&JVI3MXJ0bK&p&8jb=~c!uDK5Y2GF7*g9 zl9#Ydm|=4O;@^2$RWn>6R5q)Y8F{_tbso-k-;=J)1jlyu$N~?6>c)S&2m;AIqZnKkZrhDL~7R zb~F58I{DUtuhx5idbR99|Hl`1ZoBi%Z?(QFIMaV`1w*8#=c0W@PmuY5jr=6=t?Zn& zp`*VF{F&2gASh-c;7vB=uC}579q3s36SJW=0)09BnbTz;SlKyXL;sQu{Q~$|&TeB* zS1BM-OfPf#t!&UlKMEo@vxV%t3xw~|r2Kp-e^7`s76^3r$D)BmG8j(= z0xaNbZw`bb@kn=XA{mLdH`n*YqLKDsXJ16{tT}HyZ5geImkwow4{@u+`la ziznkjv>wS0o+?^#tR=5R^4dgtP)-aDl}m!Eh+H%VajTBNDZ;+8a$q zx~*-ZAuAQl5gij;Kyl%Lh4Zo)orYE~(CgFD!BVI%mWZ&v-cAaFI}vkNBcUI24{%l} z6i6Vp7O|ZJ!Ek>t*@J{CB!q-uAQBEHgV1)JiG*l67KlW{tjX`&R3BL9UI%8I8e7}i z+dM6S8f@wkm=Q|&KZlkpf!U%+L%R6)BJV?tDWEs7v=^QfrR>K*!-NuX$w$2B;{)^1 z`hQA3&F?fXGuhw9PD_!SYMMuZEl+sEaot^>6tJ2Z+&&e7co$8G8bvn8X0g{=jqsx$S`W791A*YC+ zZ92NXUfXr_n{@I$I{IQAeTR;Y&lgI2baWRH0lx40^e5`j)iO2XeCl3>si~1<$<&xS zbf)Yymzt_R0OYC9ehYqGchx}-@gC$A#-|{C_8G)6h}<=x}m;$qf|{#sITT9Q61;hf4z~s1q^tlfwHObF4tWRMSYC-VC`{Wvw_cbs6(~O zF{`HEPA&q|*Y8$X{tj>}ABLZE&MYt;k3-p7`KVvm4a^SEQJ=c1K%~|5sr>JtD(U0n z2r?-S(xX2@$Dw8lJkZp1Y7&^a6uU7swF~IE)CAyzJof?Q=5kq1RMWZqPe4q~925q# z+mppIRNbS!1#q=D1uP|0J|qJ#sp)fU{nFXehQ3H z!FXCQhNXsRGLV4A8-X&JtHhB@oq=3ESK~x%oWDnhGAWE{hNVVa``-{|Zx?2ZRa&jt z8ss^&B|8EZfVN&knB}HVR&X=bZf+)*-;S&dZ%C5Bajm9#4=K?p;&6i@bCXoOmn(Ku zfHIG!Ffa8M==4MYt1rQfg}z+-jphrY|2a;|TWxLzKC$tdN4*$?`}GceZ~qEV|w z18rGOY>S0s71o{c*hf4dxkXgPY0a1Hvbp=5PqEO>?~UA#W_3<~S?3HqG%p!L@0QH0Nru zc!#6TWw1mvGzToo=sBuY-SixNUX-(1*WGjvIFI12wC3I|xO1$zPQhi?++TgbV^e3% zy&$*`TXVn9xmxZ0i0XU79b!ga1XH{r9Qarg%ZfoQ9H!nmQZ!9HVLvyM%XbK$Y$Em1 z`@B9jk$O{bZ6fuG;Mzp$JA!KyseOWL6R9BQYLOa6G?O|EVo=eDf#97M$fU6IbAek$LC>SWCm`iKddTV}j9b;I@;a;B$;$U8TrYZC+0UN`n~>7UPY_V{v7T(isdZ;Ay;fj}jf| z?~KHW5sf94U6FWFf%g+aA{0sJnPGUD67TI)I`=BOgS#S1l$dBT9O+W}KtSn>M7xtc zde&N?hI{)X(L`^|ptCLtdS*r_20>2%y$jx}keS`_ShQOS^+ZBnQFa3#8X40UJZ(H8 z8``?+G!dJxqa%E}?ZFF9pn%WPL z&Ws&Tv@yq?a>ueo^U8*yC#?Y9o$xnNn3@^~$QGA3F0R<@TDZGxkUg^Wq5IaaSdL_r zX@I{V*yZn}Jd4Y}UQ&P4+>&9i#MnyUVR>%=pTFZG{+4f+eq&D7`4@AC=9PYP36}-2 z*)$u0*$B)=U^W7?5txm@Yy@T_FdKo{2+T%cHUhH|`2QUNc!h?^{M{TL+F-(O@-PuT zWWuv1{I5(ne)ES3-WBt-1@_*U@CYAM18&qI;lVa0`pyrJqcKTwwp5^uzW0-A8vnx^ zQ!x~okmbSYGEd4J0r71Drj>9ghDlbNy)EVOgBMJAq=^a7CU}zHt4TFhBjxFQVxAPc zUBU`UAo>njFxBITl%sF-s603`<^ESmc{p<9IF$>h&m5mC`S3cPq+?*A3gzMD4{ z_;p#)zm)KABz#%Ia}vHG;bjT0OE^~s<~9lMk?;Wt8zpS2uivPwgfG@tDK+kDca^fP zYTeqZ>b3VPE88MrMGYneea*U>Rp=s`$5$`dk()d#9P5K?wYv&*XTefN`qd?&l??S^ zzwDG&GE@eACv7FON$L~)LuJUHvjYlKh$*5l&>2n#42P1tRqj6=VmoXZDM*4h4^Nx|efbEbmGSU}neZxp!q|H}G z`b}(`MI1}nV(tEiQQpP2%X=Y4dbu|L8R<*3dCy3{nbCL6M*64Vo{OR<@q8(qs?zt* zdS)qucXD!`GtzIHc0U05rHp1fy)@D*;GV&D0}-doIqcmj0}&8>9+bd60~!~^$8U&{ zJ}AdI(LW>U^qe9(e!Ap<`wfHAe>pv~Moym|w|;3ydjLp7d%67d_#XngmH&42dV$+< zv8xvCYh~vLy!`3?1mEYCK$K{1;bQRf8vHpd+Ut^}QyfVC*PK4RpM1jU)B8y&RIG$e zA20KPj$Z%}-52W&_65SpSUeF34(wr}SpSZ`2)tu+ zKe)bXy|obD-{}nmKQNGc(X>kqf=L>yY^zf+CyCgTG3P!8VIM%>-e0e9y>Z(n##Zi_jB+Lc>3kmmk zxJQC%iYHoM5?mqa1mrf3Q09gleAgKxeOiYStdWwmKQb@#kdJ#l(x-hP!AW?IV8U`S zOZv25s{jeHoo|r5 z+5cyxKD8h19|+=w1%0CU6YT(GP=xgD?w6Cix%|T>ecF!@9418x&E+2jI0D=Q4GIWA=S6@EutHvuu35o#PFX Date: Sat, 21 Mar 2026 21:30:21 +0100 Subject: [PATCH 47/80] Add the obstool.py script --- scripts/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 0f2df6e..91a656a 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -15,4 +15,5 @@ install_scripts (rundiacov.sh diag_diacov.sh plotdiacov.py) install_scripts (clddet_diagnosis.sh clddet_plotter.py) install_scripts (datool_dfs.py) install_scripts (tune_br.py) +install_scripts (obstool.py) #install_scripts (diacov.sh festat.sh) From 5b2fc91d8ae8a11e1af1286d5e28c7d67f40098a Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:31:58 +0100 Subject: [PATCH 48/80] obstool bash test script --- tests/tools/test_obstool_help.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100755 tests/tools/test_obstool_help.sh diff --git a/tests/tools/test_obstool_help.sh b/tests/tools/test_obstool_help.sh new file mode 100755 index 0000000..69fa20a --- /dev/null +++ b/tests/tools/test_obstool_help.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -uex + +# A unique working directory +wd=$(pwd) +test_wd=$(pwd)/test_obstool_help + +mkdir -p ${test_wd} +cd ${test_wd} + +obstool.py -h > /dev/null && echo "obstool help/usage works" + + +# Clean up +cd ${wd} +rm -rf ${test_wd} From f66a060b9890dc595769113ae514f0898a1e28d3 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:32:15 +0100 Subject: [PATCH 49/80] Include obstool.py testing --- tests/tools/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tools/CMakeLists.txt b/tests/tools/CMakeLists.txt index 89da080..400ddf0 100644 --- a/tests/tools/CMakeLists.txt +++ b/tests/tools/CMakeLists.txt @@ -9,6 +9,7 @@ list( APPEND tools_tests_scripts test_varbcdiagnose_help.sh test_plotvarbcpred_help.sh test_tune_br_help.sh + test_obstool_help.sh ) From 198d4503b76ad2d685731dc1ba3acf56e0396d7a Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:33:33 +0100 Subject: [PATCH 50/80] Compile readgsa.F90 --- CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5879692..86c95c7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,12 +20,12 @@ set(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} ${bounds}") set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${dialect}") -# Compile readgsa.F90 routine but as python module -add_subdirectory(src) - # -# Compile. +# .Compile # +# compilation of readgsa.F90 routine as a python extention +add_subdirectory(src) + add_executable(jbdiagnose.x src/jbdiagnose.F90) add_executable(cv_header_list.x src/cv_header_list.F90) From 40169178cb90724b6d7c6bd6072705910e18bd25 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:37:34 +0100 Subject: [PATCH 51/80] Fix the import directory --- scripts/obstool.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/obstool.py b/scripts/obstool.py index b106222..ece21dc 100755 --- a/scripts/obstool.py +++ b/scripts/obstool.py @@ -9,18 +9,18 @@ # odb4py from odb4py.utils import SqlParser , OdbObject + +# Insert parent directory to get "modules" directory +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + # Obstool modules from setting import Setting from utils import OdbReader , Rows2Df from conv_stats import DHLStat -# Insert parent directory to get "modules" directory -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - - -# Simple methods to parse arguments from the command line +# Parse arguments from the command line def Usage(): help_= "Description :\n\ Diagnostic tool 'obstool'.\n\ From 029a48fd3a199c8790e159e2b74eb18e86781022 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:44:41 +0100 Subject: [PATCH 52/80] Expose the python modules to the main callers --- modules/__init__.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/modules/__init__.py b/modules/__init__.py index e69de29..7a60669 100755 --- a/modules/__init__.py +++ b/modules/__init__.py @@ -0,0 +1,18 @@ +#-*- cvoding: utf-8 -*- + +# TuneBR modules +import handle_df +import gsacov +import config_env +import sigma_bo +import read_odb + + +# Obstool modules +import conv_stats +import build_sql +import obstype_info +import io_base +import setting +import utils + From 6534da4f3106884d812c05d9ef4ecaa4ff37a892 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:57:56 +0100 Subject: [PATCH 53/80] Add readgsa.so modules --- modules/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/CMakeLists.txt b/modules/CMakeLists.txt index 7d45697..b8da4aa 100644 --- a/modules/CMakeLists.txt +++ b/modules/CMakeLists.txt @@ -7,8 +7,11 @@ function (install_modules ) endforeach () endfunction() + +# __init__ file +install_modules (__init__.py) # Py modules For tuneBR -install_modules (__init__.py +install_modules (readgsa.so gsacov.py read_odb.py config_env.py From 48b7cabc9c094df507b72e2182477efffc2a276a Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:58:24 +0100 Subject: [PATCH 54/80] use the directory modules as a package path --- modules/__init__.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/modules/__init__.py b/modules/__init__.py index 7a60669..ed693ac 100755 --- a/modules/__init__.py +++ b/modules/__init__.py @@ -1,18 +1,17 @@ #-*- cvoding: utf-8 -*- # TuneBR modules -import handle_df -import gsacov -import config_env -import sigma_bo -import read_odb +from . import gsacov +from . import config_env +from . import sigma_bo +from . import read_odb # Obstool modules -import conv_stats -import build_sql -import obstype_info -import io_base -import setting -import utils +from . import conv_stats +from . import build_sql +from . import obstype_info +from . import io_base +from . import setting +from . import utils From 1ff262df13cd25fd2078678e32337bc0e647db11 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:58:48 +0100 Subject: [PATCH 55/80] Fix import handle_df to .handle_df --- modules/conv_stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/conv_stats.py b/modules/conv_stats.py index 743f1b6..56a4d20 100755 --- a/modules/conv_stats.py +++ b/modules/conv_stats.py @@ -5,7 +5,7 @@ from collections import defaultdict # obstool modules -from handle_df import GroupDf +from .handle_df import GroupDf From a6cc98f965a36ee4e7ba08d7797d262b906c1013 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:59:06 +0100 Subject: [PATCH 56/80] Fix import readgsa to .readcov --- modules/gsacov.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/gsacov.py b/modules/gsacov.py index cdb988d..288152a 100755 --- a/modules/gsacov.py +++ b/modules/gsacov.py @@ -5,7 +5,7 @@ import numpy as np # Py/Fortran module (Reading B matrix ) -from .readgsa import readcov +from .readgsa import readcov From f520dc03bb66167d7273f1d2338fc0bee4e17202 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:59:28 +0100 Subject: [PATCH 57/80] Import .io_base --- modules/handle_df.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/handle_df.py b/modules/handle_df.py index 7a07a50..19077a4 100755 --- a/modules/handle_df.py +++ b/modules/handle_df.py @@ -5,7 +5,7 @@ from itertools import repeat -from io_base import DataIO +from .io_base import DataIO From 7657365d0fbb74992358ced451a1a8f2d2479597 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 21:59:51 +0100 Subject: [PATCH 58/80] import obstype_info --- modules/setting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/setting.py b/modules/setting.py index 234dd1a..9cca91f 100755 --- a/modules/setting.py +++ b/modules/setting.py @@ -3,7 +3,7 @@ from datetime import datetime ,timedelta # Obstool modules -from obstype_info import ObsType +from .obstype_info import ObsType class Setting: From 61e88db73b15874a053aab75a5201fd1340b82b7 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 22:00:16 +0100 Subject: [PATCH 59/80] import sql_build --- modules/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/utils.py b/modules/utils.py index a17c180..a473603 100755 --- a/modules/utils.py +++ b/modules/utils.py @@ -14,11 +14,11 @@ from odb4py.core import odb_dca, odb_open , odb_close ,odb_dict, odb_gcdist # Obstool, Desroziers & Jarvinen , Tools & modules -from build_sql import SqlHandler -from obstype_info import ObsType -from setting import Setting , Conv -from handle_df import * -from io_base import DataIO +from .build_sql import SqlHandler +from .obstype_info import ObsType +from .setting import Setting , Conv +from .handle_df import * +from .io_base import DataIO From 2abe5caf67aa1349c015e5f2e98f5414ce2c443c Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 22:00:45 +0100 Subject: [PATCH 60/80] Set modules as a path package and use .import --- scripts/obstool.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/obstool.py b/scripts/obstool.py index ece21dc..e58d45f 100755 --- a/scripts/obstool.py +++ b/scripts/obstool.py @@ -14,9 +14,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # Obstool modules -from setting import Setting -from utils import OdbReader , Rows2Df -from conv_stats import DHLStat +from modules.setting import Setting +from modules.utils import OdbReader , Rows2Df +from modules.conv_stats import DHLStat From 1153b5eb2f9514aa7022c3dd83e2a6c3f1eefc22 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 22:00:53 +0100 Subject: [PATCH 61/80] Set modules as a path package and use .import --- scripts/tune_br.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/tune_br.py b/scripts/tune_br.py index fe0d6c7..fbce5f4 100755 --- a/scripts/tune_br.py +++ b/scripts/tune_br.py @@ -12,9 +12,9 @@ # Insert parent directory to get "modules" directory sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from modules import Predef, Diag , Ratios -from modules import Odb ,TuneEnv -from modules import GSA +from modules.sigma_bo import Predef, Diag , Ratios +from modules.read_odb import Odb ,TuneEnv +from modules.gsacov import GSA @@ -24,7 +24,7 @@ parser.add_argument( "config_file", type=Path, - help="Path to tuneBR config file (.ini file format )" + help="Path to tuneBR config file ( .ini format )" ) args = parser.parse_args() From 7b6ffe6c0fc83fc291140c6f47efc489e6bbf2f6 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 22:07:53 +0100 Subject: [PATCH 62/80] Fix the short documentation --- modules/io_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/io_base.py b/modules/io_base.py index 712c790..02525a3 100755 --- a/modules/io_base.py +++ b/modules/io_base.py @@ -10,7 +10,6 @@ class DataIO: """ Class: DataIO : Contains methods to write and read the statstics dataframes for each cycle (in .csv files) - It uses the format 'feather'. Fast for I/O Compatible with pandas Methods : FlushFrame ReadFrame From 3432847f089b926080bb1ebd25794f1441973863 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 22:13:17 +0100 Subject: [PATCH 63/80] Rewrite the short class decription --- modules/read_odb.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/read_odb.py b/modules/read_odb.py index 033298e..100069b 100755 --- a/modules/read_odb.py +++ b/modules/read_odb.py @@ -17,17 +17,17 @@ class Odb: """ - Class : Odb , GET THE PATHS FROM TuneEnv OBJECT - PRFORMS A PARALLEL ODB EXRACTION. - CONTAINS : ReadOdbRows Method TO READ - OBS-ERROR , obs-guess DEPARTURES AND obs-analysis - DEPARTURES - Return :Obs_err , obs-fg , obs-an departures + Class : read_odb. Get the paths from TuneEnv object and extrcat odb rows + + Method: ReadOdbRows. read obs_error , obs-guess, obs-analysis departures """ + def __init__(self, Paths ): self.paths =Paths self.dbtype ="CCMA" # Can be hard coded since it's NOT possible to use ECMA in TuneBR + # Static query can be used here ! self.sql_query="SELECT obstype,varno,an_depar,fg_depar,final_obs_error FROM hdr, body, errstat" + self.ext_list=[ item [0] for item in shutil.get_archive_formats() ] self.rows_path =Path (Paths["ROWS_PATH"] ) From 7733093a2815d9deb2c07895e86a28836b5a79e6 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 22:14:12 +0100 Subject: [PATCH 64/80] Rewrite the short class decription --- modules/setting.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/setting.py b/modules/setting.py index 9cca91f..35bc754 100755 --- a/modules/setting.py +++ b/modules/setting.py @@ -9,8 +9,7 @@ class Setting: """ class : contains methods to set datetime period and varlist for obstool - Methods : (Better to use snake_case names for the methods !) - set_period + Methods : set_period set_obs_list """ def __init__ (self ): From 9405e9dc9e85ff6f82bafcfe3461d7abd8664774 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 22:14:59 +0100 Subject: [PATCH 65/80] Rewrite the short class decription --- modules/setting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/setting.py b/modules/setting.py index 35bc754..3c6c755 100755 --- a/modules/setting.py +++ b/modules/setting.py @@ -107,7 +107,7 @@ def set_obs_list(self, obs_list ): class Conv: """ - class : instantiate Conv obs category and corresponding sql env + class : instantiate Conv obs category and corresponding SQL query """ def __init__(self ): # ---> CAUTION !: NEVER CHANGE THE ORDER OF COLUMNS From 6a654c4f519e15319227881b4ad01cde59ab1443 Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sat, 21 Mar 2026 22:17:10 +0100 Subject: [PATCH 66/80] Correct a typo --- scripts/obstool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/obstool.py b/scripts/obstool.py index e58d45f..ce775c9 100755 --- a/scripts/obstool.py +++ b/scripts/obstool.py @@ -125,7 +125,7 @@ def ParseArgs(): diag.to_csv ( "py_"+var+".csv", index=False , header=True ) -# Finalize +# Finish end_time = datetime.now() # Time duration of stats computation From 989985ea4d7df8b50459523ff7ebaea5e658fc4e Mon Sep 17 00:00:00 2001 From: idir Dehmous Date: Sun, 22 Mar 2026 00:31:32 +0100 Subject: [PATCH 67/80] Fix build_sql when a level condition exists --- modules/build_sql.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/modules/build_sql.py b/modules/build_sql.py index ecb4c2d..06f5d5b 100755 --- a/modules/build_sql.py +++ b/modules/build_sql.py @@ -6,10 +6,10 @@ class SqlHandler: """ - Class : Parse the different observation dictionaries and build the - SQL query according to obstype, codetype, varno , sensor and level range - Checks the SQL statement before sending. - Returns :The sql statement according to variable and varno + Class : @ Parse the different observation dictionaries and build the + SQL query according to obstype, codetype, varno , sensor and level range + @ Checks the SQL statement before sending. + Returns : The sql statement according to variable and varno Methods : BuildQuery CheckQuery @@ -103,6 +103,7 @@ def BuildQuery(self , **kwarg ): # Check level range + level_list=[] if self.levels != None : if len (self.levels) <2 or len( self.levels ) > 2: print ( "Level range must have two limites [l1 , l2], but got argument:", self.levels ) @@ -117,13 +118,16 @@ def BuildQuery(self , **kwarg ): l1 =str(self.levels[0] ) l2 =str(self.levels[1] ) level_cond="vertco_reference_1 >="+l1+" AND vertco_reference_1 <= "+l2 + level_list.append(level_cond) where_cond_list=[] for tp in type_list: for vr in varno_list: - where_cond_list.append( " AND ".join( (tp, vr )) ) - + where_cond_list.append( " AND ".join( (tp, vr )) ) + if len(level_list) !=0: + for lv in level_list: + where_cond_list.append( " AND ".join( (tp, vr , lv )) ) query=" " if self.other !=None and len(where_cond_list) == 0: From df8c1413e8c3d302743a3d532955953ae10a9209 Mon Sep 17 00:00:00 2001 From: idirdehmous Date: Thu, 26 Mar 2026 14:39:49 +0100 Subject: [PATCH 68/80] Add try , except when creating DCA --- modules/utils.py | 64 +++++++++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/modules/utils.py b/modules/utils.py index a473603..fd510ad 100755 --- a/modules/utils.py +++ b/modules/utils.py @@ -9,16 +9,32 @@ from itertools import chain +"""sys.path.insert(0,"/home/idehmous/Desktop/rmib_dev/github/pyodb_1.1.0/build/lib.linux-x86_64-cpython-39") +from pyodb_extra import OdbEnv +env= OdbEnv ("/home/idehmous/Desktop/rmib_dev/github/pkg", "libodb.so") +env.InitEnv () +# pyodb modules +from pyodb_extra.odb_ob import OdbObject +env= OdbEnv ("/home/idehmous/Desktop/rmib_dev/github/pkg", "libodb.so") +env.InitEnv () +# --> NOW pyodb could be imported ! +import pyodb +from pyodb import odbDict , odbGcdistance +from pyodb import odbConnect , odbClose +from pyodb import odbDca """ + # odb4py from odb4py.utils import SqlParser , OdbObject from odb4py.core import odb_dca, odb_open , odb_close ,odb_dict, odb_gcdist + + # Obstool, Desroziers & Jarvinen , Tools & modules -from .build_sql import SqlHandler -from .obstype_info import ObsType -from .setting import Setting , Conv -from .handle_df import * -from .io_base import DataIO +from build_sql import SqlHandler +from obstype_info import ObsType +from setting import Setting , Conv +from handle_df import * +from io_base import DataIO @@ -37,22 +53,27 @@ def __init__(self): def CheckDca( self, dbpath , sub_base=None , verbose = False ): # Prepare DCA files if not in ODB - db = OdbObject ( dbpath ) - dbname = db.get_attrib()["name"] + try: # os.path.isdir( dbpath ): + db = OdbObject ( dbpath ) + dbname = db.get_attrib()["name"] - if not os.path.isdir ("/".join( [dbpath , "dca"] ) ): - if verbose in [2,3 ]: - print( "No DCA files in {} 'directory'".format(dbpath ) ) - #env.OdbVars["CCMA.IOASSIGN"]="/".join( (dbname, "CCMA.IOASSIGN" ) ) - #env.OdbVars["ECMA.IOASSIGN"]="/".join( (dbname, "ECMA.IOASSIGN" ) ) - status = odb_dca ( dbpath=dbpath , db=dbname , ncpu=8 ) - if status < 0 : - print("Failed to create DCA files ... \n Another attempt will be done with odb_dict !" ) - else : - if verbose ==True : - print("DCA files already in database: '{}'".format( dbname ) ) - else: - pass + if not os.path.isdir ("/".join( [dbpath , "dca"] ) ): + if verbose in [2,3 ]: + print( "No DCA files in {} 'directory'".format(dbpath ) ) + #env.OdbVars["CCMA.IOASSIGN"]="/".join( (dbname, "CCMA.IOASSIGN" ) ) + #env.OdbVars["ECMA.IOASSIGN"]="/".join( (dbname, "ECMA.IOASSIGN" ) ) + status = odb_dca ( database=dbpath , dbtype=dbname , ncpu=8 ) + if status < 0 : + print("Failed to create DCA files ... \n Another attempt will be done with odb_dict !" ) + else : + if verbose ==True : + print("DCA files already in database: '{}'".format( dbname ) ) + else: + pass + except: + FileNotFoundError + print("WARNING : ODB path {} not found".format(dbpath)) + pass @@ -64,7 +85,7 @@ class OdbReader: The SQL query is sent directly to the ECMA or CCMA ODB . Returns data are as python dictionary - Methods : get_odb_rows + Methods : get_odb_rows """ def __init__(self , dbpath ,type_ ): @@ -169,7 +190,6 @@ def get_odb_rows (self, period , vertco =vertco [jo] , sensor =sensor [jo] , remaining_sql =self.other_sql ) - nfunc , sql_query = self.sql.CheckQuery( query) cdtg = period [i] if vrb in [0, 1, 2, 3]: From 50ab091be8abc7844151d7aa8b6555bd53a92978 Mon Sep 17 00:00:00 2001 From: idirdehmous Date: Thu, 26 Mar 2026 16:33:50 +0100 Subject: [PATCH 69/80] Add --force_extract option : extract ODB rows even if they are archived --- modules/utils.py | 52 ++++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/modules/utils.py b/modules/utils.py index fd510ad..c449542 100755 --- a/modules/utils.py +++ b/modules/utils.py @@ -30,11 +30,11 @@ # Obstool, Desroziers & Jarvinen , Tools & modules -from build_sql import SqlHandler -from obstype_info import ObsType -from setting import Setting , Conv -from handle_df import * -from io_base import DataIO +from .build_sql import SqlHandler +from .obstype_info import ObsType +from .setting import Setting , Conv +from .handle_df import * +from .io_base import DataIO @@ -129,8 +129,9 @@ def get_odb_rows (self, period , obs_list , max_dist , bin_dist , - #file_io , later ! + #file_io , later ! file_path , + fextract , cycle_inc =3, pbar =False , verbosity =0, @@ -193,7 +194,7 @@ def get_odb_rows (self, period , nfunc , sql_query = self.sql.CheckQuery( query) cdtg = period [i] if vrb in [0, 1, 2, 3]: - print( "Process observation type: {} ODB date : {} ".format( obs , cdtg )) + print( "Process observation type: {} ODB date : {} ".format( obs , cdtg )) query_file=None ; poolmask = None ; pool =None ; @@ -208,29 +209,36 @@ def get_odb_rows (self, period , if vrb in [2,3]: pbar = True - + # Write odb rows once , if there is a rerun of the same period # the odb extraction is skipped filename = "_".join( ( "df_rows" , obs ,cdtg)) +".csv" fpath = "/".join( (self.odb_path,cdtg , filename) ) - if os.path.isfile( fpath ): - if vrb in [1,2,3]: - print("ODB rows already in file obstype: {} , date: {}".format( obs, cdtg ) ) - rows = self.io.ReadFrame( fpath ) - # Process rows from file - df_dist = self.rd.DfDist (rows, obs, cdtg, max_dist ) + + if fextract is True : + os.remove ( fpath ) + elif os.path.isfile(fpath) and fextract is False and vrb in [1,2,3]: + print("ODB rows already in file for : obstype: {} , date: {}".format( obs, cdtg ) ) + + + if os.path.isfile(fpath): + rows = self.io.ReadFrame( fpath ) + # Process rows from file + df_dist = self.rd.DfDist (rows, obs, cdtg, max_dist ) - # Subset df - spl = SplitDf (df_dist , obs , cdtg ) - ndist = df_dist.dropna(subset=["dist"]).copy() - df_stat = spl.SubsetDf( bin_dist , max_dist) - self.dlist[obs].append( df_stat ) + # Subset df + spl = SplitDf (df_dist , obs , cdtg ) + ndist = df_dist.dropna(subset=["dist"]).copy() + df_stat = spl.SubsetDf( bin_dist , max_dist) + self.dlist[obs].append( df_stat ) + + else: - # Get and process the rows from ODB + # Get (or Get again if fextracted =True) and process the rows from ODB if vrb in [ 2,3]: - print( "ODB rows not available from file :", filename ) + print( "ODB rows NOT available from file :", filename ) print( "Proceed to data extraction ...") - try: + try: rows= conn.odb_dict (cma_path , sql_query , nfunc , From 91945976a1f406f0cd368aaf395a1b6bbe94e19a Mon Sep 17 00:00:00 2001 From: idirdehmous Date: Thu, 26 Mar 2026 16:34:24 +0100 Subject: [PATCH 70/80] Get the --force_extract option --- scripts/obstool.py | 53 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/scripts/obstool.py b/scripts/obstool.py index ce775c9..fdb4c21 100755 --- a/scripts/obstool.py +++ b/scripts/obstool.py @@ -38,7 +38,17 @@ def Usage(): def ParseArgs(): # The var list must be given as a liste sperated with ":" var=lambda s:s.split(":") + ft =lambda x: x.lower() == "true" parser = argparse.ArgumentParser( description=" ") + + obslist=["synop_t" ,"synop_z" ,"synop_u" ,"synop_v","synop_h", + "airep_t" ,"airep_u" ,"airep_v" , + "airepl_t","airepl_u","airepl_v", + "temp_t" ,"temp_u" ,"temp_v" ,"temp_q" , + "templ_u" ,"templ_u" ,"templ_v" ,"templ_q", + "dribu_t" ,"dribu_u" ,"dribu_v" ,"dribu_z", + "radar_rh","radar_dow" ] + # Args significations arg1 ="Path to ODB directory. (default=.)" @@ -46,21 +56,29 @@ def ParseArgs(): arg3 ="Begin date. Format YYYYMMDDHH. ( default=1970010100)" arg4 ="End date. Format YYYYMMDDHH. ( default=1970010103)" arg5 ="Observation category. (default=conv)" - arg6 ="List of observation types separated by ':' (default=airep_t:airep_v)" + arg6 ="List of observation types separated by ':' ( e.g: airep_t:synop_u)" arg7 ="Cycle increment in hours. (default= 3)" arg8 ="Maximum distance for diagnostics in [Km] (default=100 Km)" - arg9 ="Bin distance for diagnostics in [Km] (default=10 Km)" + arg9 ="Bin distance for diagnostics in [Km] (default=10 Km)" + arg10="--force_extract ( without value ) :" \ + "The ODB rows are archived when obstool is running. " \ + "If the same period has been chosen the ODB rows " \ + "won't be extracted except if --force_extract is used in command line" # Required - parser.add_argument("-odb_path" ,required=True ,type=str ,default="." ,choices=None ,help=arg1) - parser.add_argument("-odb_type" ,required=True ,type=str ,default="CCMA" ,choices=["CCMA"],help=arg2) - parser.add_argument("-bdate" ,required=True ,type=str ,default="1970010100",choices=None ,help=arg3) - parser.add_argument("-edate" ,required=True ,type=str ,default="1970010103",choices=None ,help=arg4) - parser.add_argument("-obs_category",required=True ,type=str ,default="conv" ,choices=["conv","satem"], help=arg5) - parser.add_argument("-var_list" ,required=True ,type=var ,default="synop_t:airep_v",choices=None,help=arg6) - parser.add_argument("-cycle_inc" ,required=True ,type=int ,default=3 ,choices=None ,help=arg7) - parser.add_argument("-max_dist" ,required=False,type=float ,default=100.0 ,choices=None ,help=arg8) - parser.add_argument("-bin_dist" ,required=False,type=float ,default=10.0 ,choices=None ,help=arg9) + parser.add_argument("--odb_path" ,required=True ,type=str ,default="." ,help=arg1) + parser.add_argument("--odb_type" ,required=True ,type=str ,default="CCMA" ,choices=["CCMA"], help=arg2 ) + parser.add_argument("--bdate" ,required=True ,type=str ,default="1970010100",help=arg3) + parser.add_argument("--edate" ,required=True ,type=str ,default="1970010103",help=arg4) + parser.add_argument("--obs_category" ,required=True ,type=str ,default="conv" ,choices=["conv","satem"], help=arg5) + parser.add_argument("--var_list" ,required=True ,type=var ,default="synop_t" ,choices=None ,help=arg6) + parser.add_argument("--cycle_inc" ,required=True ,type=int ,default=3 ,choices=None ,help=arg7) + + # Optional + parser.add_argument("--max_dist" ,required=False ,type=float ,default=100.0 ,choices=None ,help=arg8) + parser.add_argument("--bin_dist" ,required=False ,type=float ,default=10.0 ,choices=None ,help=arg9) + parser.add_argument("--force_extract",action="store_true", help=arg10 ) + # if len(sys.argv) == 1: Usage() parser.print_help() @@ -86,6 +104,9 @@ def ParseArgs(): var_list= [] for v in args.var_list: var_list.append( v.lstrip () ) +# Extract ODB rows again +fextract=args.force_extract + # Max and binning distances max_dist= args.max_dist bin_dist= args.bin_dist @@ -102,7 +123,15 @@ def ParseArgs(): st.set_obs_list( var_list ) # Collection of pre-selected frames with distances <= max_dist -frame_liste = rr.get_odb_rows (period ,var_list, max_dist ,bin_dist , odbpath , cycle_inc, pbar =True , verbosity =2) +frame_liste = rr.get_odb_rows (period , + var_list , + max_dist , + bin_dist , + odbpath , + fextract , + cycle_inc , + pbar =True , + verbosity =2 ) # Concat Df for the final stats cdf = rd.DfPrep( frame_liste ) From e07cd708252e1e2aa1c4077fb1bbf07ed1f9a203 Mon Sep 17 00:00:00 2001 From: idirdehmous Date: Sun, 29 Mar 2026 13:25:00 +0200 Subject: [PATCH 71/80] Fix the build query when obstype ==9 ascat --- modules/build_sql.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/modules/build_sql.py b/modules/build_sql.py index 06f5d5b..126925b 100755 --- a/modules/build_sql.py +++ b/modules/build_sql.py @@ -39,6 +39,7 @@ def BuildQuery(self , **kwarg ): self.sensor =kwarg["sensor" ] self.other =kwarg["remaining_sql"] + # Obstool columns and tables self.obstool_select="SELECT "+self.cols+" FROM " +self.tabs @@ -50,14 +51,11 @@ def BuildQuery(self , **kwarg ): if tp is not None: type_list.append (" obstype =="+str( tp ) ) elif isinstance (self.obst, int ): - type_list.append (" obstype =="+str( self.obst ) ) - + type_list.append (" obstype =="+str( self.obst ) ) else: print(self.obs_, " : obstype must be integer or a list" ) sys.exit() - - # Check varno varno_list=[] if self.varno is not None: @@ -120,11 +118,14 @@ def BuildQuery(self , **kwarg ): level_cond="vertco_reference_1 >="+l1+" AND vertco_reference_1 <= "+l2 level_list.append(level_cond) - + where_cond_list=[] - for tp in type_list: + for tp in type_list: + if len( type_list) !=0 : + where_cond_list.append( "".join( ( tp ) ) ) for vr in varno_list: - where_cond_list.append( " AND ".join( (tp, vr )) ) + if len( varno_list) !=0: + where_cond_list.append( " AND ".join( (tp, vr )) ) if len(level_list) !=0: for lv in level_list: where_cond_list.append( " AND ".join( (tp, vr , lv )) ) From 2359bbc8e881e306fdcd7820412881cd38b4bcd3 Mon Sep 17 00:00:00 2001 From: idirdehmous Date: Sun, 29 Mar 2026 13:25:21 +0200 Subject: [PATCH 72/80] Fix verbosity level --- modules/read_odb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/read_odb.py b/modules/read_odb.py index 100069b..753eb10 100755 --- a/modules/read_odb.py +++ b/modules/read_odb.py @@ -42,7 +42,7 @@ def CheckTarball (self, tarpath ): return False except: FileNotFoundError - print("WARNING : tar file {} not found. ".format(tarpath)) + print("**WARNING : tar file {} not found. ".format(tarpath)) pass @@ -208,7 +208,7 @@ def OdbExtract (self, dates, rows_path, progress=False , verbose =False ): print(f"Rows are already extracted for the ODB {dbpath} \n" ) continue else: - print(f"WARNING : CCMA directory not found for the ODB {dbpath} \n") + print(f"**WARNING : CCMA directory not found for the ODB {dbpath} \n") def ReadOdbRows ( basedir, rows_path, cdtg, target ): From a7231ebfb8932d86dd3621f2a9fabf715ccee20b Mon Sep 17 00:00:00 2001 From: idirdehmous Date: Sun, 29 Mar 2026 13:25:40 +0200 Subject: [PATCH 73/80] Fix the progress bar and verbosity --- modules/utils.py | 40 +++++++++++++--------------------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/modules/utils.py b/modules/utils.py index c449542..439a36e 100755 --- a/modules/utils.py +++ b/modules/utils.py @@ -9,21 +9,7 @@ from itertools import chain -"""sys.path.insert(0,"/home/idehmous/Desktop/rmib_dev/github/pyodb_1.1.0/build/lib.linux-x86_64-cpython-39") -from pyodb_extra import OdbEnv -env= OdbEnv ("/home/idehmous/Desktop/rmib_dev/github/pkg", "libodb.so") -env.InitEnv () -# pyodb modules -from pyodb_extra.odb_ob import OdbObject -env= OdbEnv ("/home/idehmous/Desktop/rmib_dev/github/pkg", "libodb.so") -env.InitEnv () -# --> NOW pyodb could be imported ! -import pyodb -from pyodb import odbDict , odbGcdistance -from pyodb import odbConnect , odbClose -from pyodb import odbDca """ - -# odb4py +# odb4py mdules from odb4py.utils import SqlParser , OdbObject from odb4py.core import odb_dca, odb_open , odb_close ,odb_dict, odb_gcdist @@ -72,7 +58,7 @@ def CheckDca( self, dbpath , sub_base=None , verbose = False ): pass except: FileNotFoundError - print("WARNING : ODB path {} not found".format(dbpath)) + print("**WARNING : ODB path {} not found".format(dbpath)) pass @@ -193,7 +179,7 @@ def get_odb_rows (self, period , remaining_sql =self.other_sql ) nfunc , sql_query = self.sql.CheckQuery( query) cdtg = period [i] - if vrb in [0, 1, 2, 3]: + if vrb in [0, 1, 2, 3,4]: print( "Process observation type: {} ODB date : {} ".format( obs , cdtg )) query_file=None ; poolmask = None ; @@ -204,9 +190,9 @@ def get_odb_rows (self, period , # Progress bar & Verbosity inside pyodb # Progress bar is useful for huge ODBs - if vrb in [3 ]: + if vrb in [3,4]: verbose= True - if vrb in [2,3]: + if vrb in [2,3,4]: pbar = True @@ -215,12 +201,10 @@ def get_odb_rows (self, period , filename = "_".join( ( "df_rows" , obs ,cdtg)) +".csv" fpath = "/".join( (self.odb_path,cdtg , filename) ) - if fextract is True : + if fextract is True and os.path.isfile(fpath): os.remove ( fpath ) elif os.path.isfile(fpath) and fextract is False and vrb in [1,2,3]: print("ODB rows already in file for : obstype: {} , date: {}".format( obs, cdtg ) ) - - if os.path.isfile(fpath): rows = self.io.ReadFrame( fpath ) # Process rows from file @@ -235,7 +219,7 @@ def get_odb_rows (self, period , else: # Get (or Get again if fextracted =True) and process the rows from ODB - if vrb in [ 2,3]: + if vrb in [ 1,2,3,4]: print( "ODB rows NOT available from file :", filename ) print( "Proceed to data extraction ...") try: @@ -249,13 +233,15 @@ def get_odb_rows (self, period , verbose ) conn.odb_close() if not rows: - print( "WARNING : Data not available for the variable {} and varno={}".format( obs , varno[jo]) ) + if vrb in [0,1,2,3,4]: + print( "**WARNING : Data not available for obstype={} and varno={}".format( obs , varno[jo]) ) empty_df= pd.DataFrame([]) self.dlist[obs].append( empty_df ) except: RuntimeError - print( "WARNING : Data not available for the variable {} and varno={}".format( obs , varno[jo]) ) + if vrb in [0,1,2,3,4]: + print( "**WARNING : Data not available for obstype={} and varno={}".format( obs , varno[jo] )) empty_df= pd.DataFrame([]) self.dlist[obs].append( empty_df ) pass @@ -436,8 +422,8 @@ def DfDist (self , rows , ): #pd.set_option('display.max_rows', 20 ) vrb=verbosity - if vrb not in [0,1,2,3]: - print("WARNING : Min and max verbosity levels: 0 -> 3. Got : ", vrb ) + if vrb not in [0,1,2,3,4]: + print("**WARNING : Min and max verbosity levels: 0 -> 4 Got : ", vrb ) print("Fallback to default value: verbosity=", vrb ) if rows is None: From b7fc98962059bb5bcac3a5e40de06c452c57ebd9 Mon Sep 17 00:00:00 2001 From: idirdehmous Date: Sun, 29 Mar 2026 13:31:54 +0200 Subject: [PATCH 74/80] Add message for virtual en creation --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8aaf107..1fe4a69 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -15,7 +15,7 @@ message(STATUS "System python: ${Python_EXECUTABLE}") # Create venv inside build set(VENV_DIR ${CMAKE_BINARY_DIR}/venv) if(NOT EXISTS ${VENV_DIR}/bin/python) - message(STATUS "Creating virtualenv") + message(STATUS "Creating virtualenv ...") execute_process( COMMAND ${Python_EXECUTABLE} -m venv ${VENV_DIR} RESULT_VARIABLE venv_status From bdcbea7a401c13ad8af66406dc7bee07eda02cc6 Mon Sep 17 00:00:00 2001 From: idirdehmous Date: Tue, 31 Mar 2026 17:32:06 +0200 Subject: [PATCH 75/80] Fix the issue : obstype in select statement --- modules/build_sql.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/build_sql.py b/modules/build_sql.py index 126925b..e0870c5 100755 --- a/modules/build_sql.py +++ b/modules/build_sql.py @@ -142,6 +142,8 @@ def BuildQuery(self , **kwarg ): for condition in where_cond_list: query= self.obstool_select +" WHERE " + condition + print(query ) + #quit() return query From 43ddedf1862cf83a4e587c1a181311fd2dfa7471 Mon Sep 17 00:00:00 2001 From: idirdehmous Date: Tue, 31 Mar 2026 17:32:23 +0200 Subject: [PATCH 76/80] Ensure rounding to 10-6 --- modules/conv_stats.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/conv_stats.py b/modules/conv_stats.py index 56a4d20..179d142 100755 --- a/modules/conv_stats.py +++ b/modules/conv_stats.py @@ -243,7 +243,7 @@ def getStatFrame (self , var ): } # Return statistics DF - # Remark that the dataframe is rounded to 4 decimal digits + # Remark that the dataframe is rounded to 6 decimal digits # It's done to allow a consistent comparison with the values given by R code stat_frame =pd.DataFrame ( drhl_frame ).astype({ "nobs":"int32" , @@ -258,7 +258,7 @@ def getStatFrame (self , var ): "COR_HL" :"float64" , "COR_DR-R" :"float64" , "COR_DR-B" :"float64" - }).round(4) + }).round(6) From 38519cac6f50ccbe60719959f7560524a896cd04 Mon Sep 17 00:00:00 2001 From: idirdehmous Date: Tue, 31 Mar 2026 17:32:29 +0200 Subject: [PATCH 77/80] Ensure rounding to 10-6 --- modules/handle_df.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/handle_df.py b/modules/handle_df.py index 19077a4..2e2bb97 100755 --- a/modules/handle_df.py +++ b/modules/handle_df.py @@ -51,7 +51,7 @@ def SubsetDf (self , bin_dist , max_dist ): num = ("dist" , "count"), FGsqr1 = ("FGsqr1", "sum"), FGsqr2 = ("FGsqr2", "sum"), - Asqr1 = ("Asqr1" , "sum") ).reset_index() ).round(4) + Asqr1 = ("Asqr1" , "sum") ).reset_index() ).round(6) # Add var and date spdf["var" ]=[ self.var for v in range(len( spdf["num"] ) ) ] From c8ae204bb06294977cf92dd96088f69bb0d89e41 Mon Sep 17 00:00:00 2001 From: idirdehmous Date: Tue, 31 Mar 2026 17:32:45 +0200 Subject: [PATCH 78/80] Ensure rounding to 10-6 and distance to integer --- modules/utils.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/modules/utils.py b/modules/utils.py index 439a36e..84128e9 100755 --- a/modules/utils.py +++ b/modules/utils.py @@ -203,8 +203,9 @@ def get_odb_rows (self, period , if fextract is True and os.path.isfile(fpath): os.remove ( fpath ) - elif os.path.isfile(fpath) and fextract is False and vrb in [1,2,3]: + elif os.path.isfile(fpath) and fextract is False and vrb in [1,2,3,4]: print("ODB rows already in file for : obstype: {} , date: {}".format( obs, cdtg ) ) + if os.path.isfile(fpath): rows = self.io.ReadFrame( fpath ) # Process rows from file @@ -219,9 +220,6 @@ def get_odb_rows (self, period , else: # Get (or Get again if fextracted =True) and process the rows from ODB - if vrb in [ 1,2,3,4]: - print( "ODB rows NOT available from file :", filename ) - print( "Proceed to data extraction ...") try: rows= conn.odb_dict (cma_path , sql_query , @@ -442,7 +440,7 @@ def DfDist (self , rows , # The distances ar rounded to 0 decimals # Else , there will be a number of observations sets in each bins. # It leads to different statistics between R and Python whil perfomrming the cut , subset etc . - dist = d.GcdistParallel().round(0) #astype(int) + dist = d.GcdistParallel().round(0) N = len(lats) dist_1d = dist.ravel() @@ -480,8 +478,8 @@ def DfDist (self , rows , # For safety df_dist = df_dist[df_dist["dist"] <= max_dist].copy() - # Round to 4 decimal precision - return df_dist.round(4) + # Round to 6 decimal precision + return df_dist.round(6) From af9fbacfc9c7e85bc8145ee84aa07c1b7e3773af Mon Sep 17 00:00:00 2001 From: idirdehmous Date: Tue, 31 Mar 2026 17:33:32 +0200 Subject: [PATCH 79/80] Add verbosity option --- scripts/obstool.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/obstool.py b/scripts/obstool.py index fdb4c21..43d4774 100755 --- a/scripts/obstool.py +++ b/scripts/obstool.py @@ -60,7 +60,8 @@ def ParseArgs(): arg7 ="Cycle increment in hours. (default= 3)" arg8 ="Maximum distance for diagnostics in [Km] (default=100 Km)" arg9 ="Bin distance for diagnostics in [Km] (default=10 Km)" - arg10="--force_extract ( without value ) :" \ + arg10="Verbosity level. 0 to 4" + arg11="--force_extract ( without value ) :" \ "The ODB rows are archived when obstool is running. " \ "If the same period has been chosen the ODB rows " \ "won't be extracted except if --force_extract is used in command line" @@ -77,7 +78,8 @@ def ParseArgs(): # Optional parser.add_argument("--max_dist" ,required=False ,type=float ,default=100.0 ,choices=None ,help=arg8) parser.add_argument("--bin_dist" ,required=False ,type=float ,default=10.0 ,choices=None ,help=arg9) - parser.add_argument("--force_extract",action="store_true", help=arg10 ) + parser.add_argument("--verbose" ,required=False ,type=int ,default=1 ,choices=None ,help=arg10) + parser.add_argument("--force_extract",action="store_true", help=arg11 ) # if len(sys.argv) == 1: Usage() @@ -91,6 +93,7 @@ def ParseArgs(): # Get args args= ParseArgs () + # Odb odbpath= args.odb_path odbtype= args.odb_type @@ -107,10 +110,16 @@ def ParseArgs(): # Extract ODB rows again fextract=args.force_extract +# verbose +verb=args.verbose + # Max and binning distances max_dist= args.max_dist bin_dist= args.bin_dist + + + # Instantiate st = Setting () rd = Rows2Df () @@ -130,8 +139,8 @@ def ParseArgs(): odbpath , fextract , cycle_inc , - pbar =True , - verbosity =2 ) + pbar =True , + verbosity = verb ) # Concat Df for the final stats cdf = rd.DfPrep( frame_liste ) From 7e240d7743727736caabd812cfabef9c780ea33b Mon Sep 17 00:00:00 2001 From: idirdehmous Date: Tue, 31 Mar 2026 17:37:46 +0200 Subject: [PATCH 80/80] Remove intermediate print query --- modules/build_sql.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/build_sql.py b/modules/build_sql.py index e0870c5..126925b 100755 --- a/modules/build_sql.py +++ b/modules/build_sql.py @@ -142,8 +142,6 @@ def BuildQuery(self , **kwarg ): for condition in where_cond_list: query= self.obstool_select +" WHERE " + condition - print(query ) - #quit() return query