Commit CRTMv3 configuration (backward compatible with 2.4)

CMakeLists.txt

@@ -1,60 +1,117 @@
 cmake_minimum_required(VERSION 3.10.2)
 project(skbuild_pycrtm)
 enable_language(Fortran)
+
 list( APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake )
 find_package(F2PY REQUIRED)
 find_package(PythonLibs REQUIRED)
 find_package(Python3 REQUIRED COMPONENTS NumPy)
 find_package( OpenMP COMPONENTS Fortran )
-find_package( NetCDF REQUIRED COMPONENTS Fortran )
+find_package( NetCDF REQUIRED COMPONENTS Fortran)
 include_directories(${PYTHON_INCLUDE_DIRS})
 include_directories(${_Python3_NumPy_INCLUDE_DIR})
-include_directories(${NetCDF_INCLUDE_DIRS})
+include_directories(NetCDF_F90_INCLUDE_DIRS)
 get_filename_component(CDF_LIB_PATH ${NetCDF_Fortran_LIBRARY} DIRECTORY)
-set(crtm_INCLUDE $ENV{CRTM_INSTALL}/module/crtm/${CMAKE_Fortran_COMPILER_ID}/${CMAKE_Fortran_COMPILER_VERSION}/)
-set(crtm_LIB $ENV{CRTM_INSTALL}/lib/)
+find_package(Python REQUIRED COMPONENTS Interpreter Development.Module NumPy)
+
+if (EXISTS $ENV{CRTM_INSTALL}/module/crtm/${CMAKE_Fortran_COMPILER_ID}/${CMAKE_Fortran_COMPILER_VERSION}/)
+    set(crtm_INCLUDE $ENV{CRTM_INSTALL}/module/crtm/${CMAKE_Fortran_COMPILER_ID}/${CMAKE_Fortran_COMPILER_VERSION}/)
+else()
+    set(crtm_INCLUDE $ENV{CRTM_INSTALL}/include/)
+endif()
+
+if (EXISTS $ENV{CRTM_INSTALL}/lib64/)
+    set(crtm_LIB $ENV{CRTM_INSTALL}/lib64/)
+elseif(EXISTS $ENV{CRTM_INSTALL}/lib/)
+    set(crtm_LIB $ENV{CRTM_INSTALL}/lib/)
+else()
+    set(crtm_LIB $ENV{CRTM_INSTALL})
+endif()   
 
 if ("${CMAKE_Fortran_COMPILER_ID}" MATCHES "Intel")
     set(ompFlag "-liomp5")
+    set(switch_fun "intelem")
+    set(preproc "-fpp")
 else()
     set(ompFlag "-lgomp")
+    set(switch_fun "gfortran")
+    set(preproc "-cpp")
 endif()
-
+set(f2py_module_name "pycrtm_")
 set(fortran_src_file "${CMAKE_CURRENT_SOURCE_DIR}/pycrtm.f90")
-set(f2py_module_name "pycrtm")
 set(generated_module_file ${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_name}${PYTHON_EXTENSION_MODULE_SUFFIX})
-
 #_dumb is used to avoid circular dependency which breaks Ninja.
 add_custom_target(${f2py_module_name}_dumb ALL
   DEPENDS ${generated_module_file}
   )
-
-add_custom_command(
-  OUTPUT ${generated_module_file}
-  COMMAND ${F2PY_EXECUTABLE}
-    -m ${f2py_module_name}
-    -h ${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_name}.pyf ${fortran_src_file}
-    --overwrite-signature
-  COMMAND ${F2PY_EXECUTABLE}
-    -m ${f2py_module_name}
-    -c
-    ${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_name}.pyf 
-    ${fortran_src_file}
-    -I${crtm_INCLUDE}
-    -I${NetCDF_INCLUDE_DIRS}
-    -L${CDF_LIB_PATH}
-    -L${NetCDF_INCLUDE_DIRS}
-    -L${crtm_LIB}
-    -lcrtm
-    -lnetcdff
-    ${ompFlag}
-    only: 
-    wrap_forward 
-    wrap_k_matrix
-  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-  )
+if (EXISTS $ENV{CRTM_INSTALL}/module/crtm/${CMAKE_Fortran_COMPILER_ID}/${CMAKE_Fortran_COMPILER_VERSION}/crtm_active_sensor.mod)
+  set(preproc "${preproc} -DPYCRTM_ACTIVE")
+  add_custom_command(
+    OUTPUT ${generated_module_file}
+    COMMAND python -m numpy.f2py
+      -m ${f2py_module_name}
+      -h ${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_name}.pyf ${fortran_src_file}
+    COMMAND python -m numpy.f2py
+      --fcompiler=${switch_fun}
+      --f90flags=${preproc} 
+      -c
+      ${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_name}.pyf 
+      ${fortran_src_file}
+      -I${crtm_INCLUDE}
+      -I${NETCDF_FORTRAN_INCLUDE_DIRS}
+      -L${CDF_LIB_PATH}
+      -L${crtm_LIB}
+      -lcrtm
+      -lnetcdff
+      ${ompFlag}
+# should work according to f2py documentation but doesn't. Have to set it through --f90flags as above.
+#      -DPYCRTM_ACTIVE
+      only: 
+      wrap_forward 
+      wrap_k_matrix
+      wrap_forward_active
+      wrap_k_matrix_active
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    )
+else()
+  add_custom_command(
+    OUTPUT ${generated_module_file}
+    COMMAND python -m numpy.f2py
+      -m ${f2py_module_name}
+      -h ${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_name}.pyf ${fortran_src_file}
+    COMMAND python -m numpy.f2py
+      --fcompiler=${switch_fun}
+      -c
+      ${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_name}.pyf 
+      ${fortran_src_file}
+      --f90flags=${preproc} 
+      -I${crtm_INCLUDE}
+      -I${NETCDF_FORTRAN_INCLUDE_DIRS}
+      -L${CDF_LIB_PATH}
+      -L${crtm_LIB}
+      -lcrtm
+      -lnetcdff
+      ${ompFlag}
+      only: 
+      wrap_forward 
+      wrap_k_matrix
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    )
+endif()
 #Shouldn't need this, but scikit-build won't grab the shared object otherwise.
 list(GET _Python3_INTERPRETER_PROPERTIES 6 version)
 set(produced_so ${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_name}.${version}.so)
 #Move to pycrtm directory
-install(FILES ${produced_so} DESTINATION pycrtm)
+if (EXISTS ${crtm_LIB}/libcrtm.so)
+    set(libcrtm_so ${crtm_LIB}/libcrtm.so)
+endif()
+if (EXISTS ${crtm_LIB}/libcrtm.dylib)
+    set(libcrtm_so ${crtm_LIB}/libcrtm.dylib)
+endif()
+
+
+#Move to pycrtm directory
+install(FILES ${produced_so} DESTINATION pycrtm_)
+install(FILES ${libcrtm_so} DESTINATION pycrtm_)
+#Move configuration into install 
+install(FILES pycrtm_setup.txt DESTINATION pycrtm_)

README.md

@@ -2,7 +2,7 @@
 
 ## Bryan M. Karpowicz, Ph.D. - USRA/GESTAR/NASA 610.1 Global Modeling and Assimilation Office, with Contributions from Patrick Stegmann, Dr.-Ing. - JCSDA
 
-This is a basic python interface to CRTM v2.4.0. 
+This is a basic python interface to CRTM v2.4.x or CRTMv3. 
 
 The user interface is designed to be very similar to the python RTTOV interface. So, the user sets profiles, passes them to an object for a desired sensor, runs either the forward model/K-matrix, and pulls the brightness temperature/Jacobian/transmission/emissivity out of the object.  
 
@@ -14,57 +14,46 @@ This `README` has 4 parts:
 3. Importing -- how to use this library in a project.
 4. Using the interface -- HOWTO/run through on how to use this interface
 
-- Bryan Karpowicz -- March 20, 2021
+- Bryan Karpowicz -- May 7, 2025
 ---------------------------------------------------------------------------------------- 
 
 ## 1. Installation:
-- For the novice that doesn't care about where or how this installs, look at the crtm-bundle and run kickstart_pyCRTM.sh Otherwise...
-- Dependencies CRTM, h5py, numpy and scikit-build (install those first, if you don't have them). Note crtm must be built with the static option (`ecbuild --static`) 
-- Configuration
-First modify `setup.cfg` to point to the crtm install location (path underneath should contain `lib/libcrtm.a`). 
+For a quicker install experience users may choose to install using the `make_it_so.sh` which will install conda along with the required packages in a miniconda environment `pycrtm`. There are four options `apple_silicon` for Macs with an M2/M3/M4/M? processor, `apple_intel` to install miniconda for Macs with an intel processor, `linux` for all other linux systems, and `skip` which will skip installing miniconda and use an existing conda install a `pycrtm` miniconda environment. Users are cautioned to look over the script to make sure it will not overwrite existing installs, or fill up your home directory, if space is limited. For example if your system does not have a python install and you a starting from scratch on an M4 Mac simply type:
 ```
-[Setup]
-# Specify the location of the crtm install (ecbuild install ONLY)
-crtm_install = /discover/nobackup/bkarpowi/github/JCSDA_crtm/crtm-bundle/crtm/build
-# Download Coefficients
-# Controls whether coefficients are downloaded
-download = True
-#This will move the coefficients with the package install.
-coef_with_install = True
-[Coefficients]
-# Use to specify alternative coefficient file location where Little Endian Coefficient files are stored.
-# If user desires coefficients to be stored with the installed package, leave this alone.
-# If user selects coef_with_install = False, this must be specified.
-# set argument below (path) to the full path of the coefficients.
-path = /discover/nobackup/projects/gmao/obsdev/bkarpowi/tstCoef/
+./make_it_so.sh apple_silicon 
 ```
-In the example above the coefficients will be included with the pycrtm install. To change this, set `coef_with_install` and set `path` to the location where you would like crtm coefficients stored. If you already have a directory with coefficients, you can set `download` and `coef_with_install` to False, and set `path` to that location. The pycrtm configuration will then point to the location in `path`.  
-
-- Installation 
-There are two recommended ways to install. The first, if the user has full write access to their python distribution, it may be installed globally using:
+The script will install miniconda3, CRTMv3, pyCRTM, and run the `test_atms.py` script to verify pyCRTM is working. If you already have miniconda on your machine you can simply run `skip` which will just install CRTMv3, pyCRTM and run the `test_atms.py` script to verify pyCRTM has been installed and is functioning properly. Once installed a user may use the new `pycrtm` conda environment by typing:
 ```
-python3 setup.py install 
+conda activate pycrtm
 ```
-This will take some time as it will download coefficients, move them around, compile the pycrtm module, and link against th crtm library.
 
-The second, if the user doesn't have full write access to their python distribution is to first build a wheel, and install using pip:
-```
-python3 setup.py bdist_wheel
-```
-This will take some time as it will download coefficients, move them around, compile the pycrtm module, and link against the crtm library. Once the wheel has been built, it may be installed locally using pip:
+If that doesn't suit your taste, read on for a more step-by-step approach. 
+
+- Dependencies CRTM, h5py, numpy and scikit-build (install those first, if you don't have them). 
+- Configuration
+First modify `setup.cfg` to point to the crtm install location (path underneath should contain one of the following: `lib/libcrtm.a`,`lib64/libcrtm.a`, `lib/libcrtm.so`, or `lib64/libcrtm.so`). 
 ```
-pip install dist/pyCRTM_JCSDA*.whl --target /discover/nobackup/projects/gmao/obsdev/bkarpowi/pythonModules/
+# Specify the location of the crtm install (ecbuild install ONLY)
+crtm_install = /discover/nobackup/projects/gmao/obsdev/bkarpowi/pycrtm_builds/CRTMv3Cmake_gnu/build/
+link_from_source_to_path_used = True
+[Coefficients]
+# source specify coefficient directory will grab little endian binary coefficients and netcdf and link them to path_used
+source_path = /discover/nobackup/projects/gmao/obsdev/bkarpowi/pycrtm_builds/CRTMv3Cmake_gnu/build/test_data/crtm
+# path used by pycrtm to read coefficients 
+path_used =  /discover/nobackup/projects/gmao/obsdev/bkarpowi/pycrtm_builds/pycrtmV3cmake/coefficients
 ```
-paired with appending `/discover/nobackup/projects/gmao/obsdev/bkarpowi/pythonModules/` to the `PYTHONPATH` environment variable in your .bashrc or .cshrc.
-
-For Bash this is:
+Next, pycrtm must have a location where all desired coefficients are expanded in a flat directory. In the configuration above, the installer will create `path_used` and populate it with symbolic links to all available coefficients in `source_path.` If `link_from_source_to_path_used` is set to `False`, `source_path` will be ignored and it is assumed the user has placed coefficients in `path_used` and pyCRTM will search for coefficients in this directory. 
+- Installation 
+If the user has full write access to their python distribution, it may be installed globally using:
 ```
-export PYTHONPATH="${PYTHONPATH}:/discover/nobackup/projects/gmao/obsdev/bkarpowi/pythonModules/"
+pip install .
 ```
-For Tcsh/csh:
+Otherwise, the standard --user option is also available which will install under $HOME/.local/
 ```
-setenv PYTHONPATH ${PYTHONPATH}:/discover/nobackup/projects/gmao/obsdev/bkarpowi/pythonModules
+pip install . --user
 ```
+Optionally, you may supply "-v" for a more verbose output while it is installing. Either way, this will take some time as it will symlink coefficients downloaded when you install CRTM, compile the pycrtm module, and link against th crtm library.
+
 
 Compiler options are handled autmoatically through cmake. On HPC systems this means loading the right set of modules. For example, if you would like pycrtm compiled with intel, you would load the same intel modules you used to build crtm. 
 
@@ -91,6 +80,17 @@ The following scripts will run CRTM without aerosols or clouds:
 For those Jupyter notebook fans, there is even Jupyter notebook example simulating ATMS:
 * `$PWD/testCases/test_atms.ipynb`
 
+Additonal More Advanced Examples:
+* `$PWD/testCases/test_atms_jacobian.py` Provides cloud jacobians (provide --plot command line argument to generate plot)
+* `$PWD/testCases/test_atms_subset_cloudnames.py` Provides example of using a channel subset, along with Cloud type names.
+* `$PWD/testCases/test_atms_subset.py` Provides exmaple using a channel subset.
+* `$PWD/testCases/test_cris_jacobian.py` Provides cloud/aerosol jacobians (provide --plot command line argument to generate plot)
+* `$PWD/testCases/test_cris_subset.py` Provides exmaple using a channel subset.
+
+Active Sensor Examples (Available with CRTMv3.1.x)
+* `$PWD/testCases/test_cloudsat.py` tests forward model of active sensor (provide --plot for plot of reflectivity/attenuated reflectivity)
+* `$PWD/testCases/test_cloudsat_jacobian.py` tests cloud jacobians (provide --plot for cloud jacobian plot, --attenuated for attenuated reflectivity, otherwise jacobians of reflectivity are plotted.
+
 ## 3. Importing 
 
 ```Python

crtm_io.py

@@ -1,4 +1,92 @@
-import os, struct, configparser 
+import os, sys, glob, struct, configparser,netCDF4
+import numpy as np 
+def findLib(thisDir):
+    """
+    Find crtm library and check for shared library *.so. If so is found, return
+    library path, otherwise return nothing to use static library.
+    """
+    cfg = configparser.ConfigParser()
+    if ( os.path.exists( os.path.join(thisDir,'pycrtm_','pycrtm_setup.txt') ) ):
+        pycrtm_setup = os.path.join(thisDir,'pycrtm_','pycrtm_setup.txt')
+    else:
+        print("Error. File not present: {}".format(os.path.join(thisDir,'pycrtm_','pycrtm_setup.txt')))
+        sys.exit()
+    cfg.read( pycrtm_setup )
+    setupdir = cfg['Setup']['crtm_install']
+    if( os.path.exists( os.path.join( setupdir, 'lib') ) ):
+        libdir = os.path.join( setupdir, 'lib')
+    elif( os.path.exists( os.path.join( setupdir, 'lib64') ) ):
+        libdir = os.path.join( setupdir, 'lib64')
+    else:
+        libdir = os.path.join(setupdir,'')
+    so = glob.glob(os.path.join(libdir,'*.so'))
+    if(len(so)>0):
+        return libdir
+    else:
+        return ""
+def findLibDyld(thisDir):
+    """
+    Find crtm library and check for shared library *.so. If so is found, return
+    library path, otherwise return nothing to use static library.
+    """
+    cfg = configparser.ConfigParser()
+    if ( os.path.exists( os.path.join(thisDir,'pycrtm_','pycrtm_setup.txt') ) ):
+        pycrtm_setup_dir = os.path.join(thisDir,'pycrtm_','pycrtm_setup.txt')
+    else:
+        print("Error. File not present: {}".format(os.path.join(thisDir,'pycrtm_','pycrtm_setup.txt')))
+        sys.exit()
+    cfg.read( os.path.join(thisDir,pycrtm_setup_dir) )
+    setupdir = cfg['Setup']['crtm_install']
+    if( os.path.exists( os.path.join( setupdir, 'lib') ) ):
+        libdir = os.path.join( setupdir, 'lib')
+    elif( os.path.exists( os.path.join( setupdir, 'lib64') ) ):
+        libdir = os.path.join( setupdir, 'lib64')
+    else:
+        libdir = os.path.join(setupdir,'') 
+    so = glob.glob(os.path.join(libdir,'libcrtm.*'))
+    if(len(so)>0):
+        return libdir
+    else:
+        return ""
+
+
+
+def setLD_LIBRARY_PATH(libdir):
+    """
+    For a given path add or create LD_LIBRARY_PATH and do os.execv thing to do it in the current
+    python environment.
+    """    
+    #Set the LD_LIBRARY_PATH to make it possible to used shared object. 
+    old_ld = os.environ.get("LD_LIBRARY_PATH")
+    if old_ld:
+        if(libdir not in os.environ["LD_LIBRARY_PATH"]):
+            os.environ["LD_LIBRARY_PATH"] = old_ld + ":" + libdir
+            os.execv(sys.argv[0], sys.argv)
+    elif(len(libdir)>0):
+        os.environ["LD_LIBRARY_PATH"] = libdir
+        os.execv(sys.argv[0], sys.argv)
+
+def setDYLD_LIBRARY_PATH(libdir):
+    """
+    Warn user if dyld, that you need to set environment variable.
+    """   
+    #Set the LD_LIBRARY_PATH to make it possible to used shared object. 
+    so = glob.glob(os.path.join(libdir,'libcrtm.*')) 
+    old_ld = os.environ.get("DYLD_LIBRARY_PATH")
+    if(len(so)>0):
+        if old_ld:
+            if(libdir not in os.environ["DYLD_LIBRARY_PATH"]):
+                #os.environ["DYLD_LIBRARY_PATH"] = old_ld + ":" + libdir
+                print('set DYLD_LIBRARY_PATH!')
+                print('export DYLD_LIBRARY_PATH={}'.format(libdir))
+                os.execv(sys.argv[0], sys.argv)
+        elif(len(libdir)>0):
+            #os.environ["DYLD_LIBRARY_PATH"] = libdir
+             print('set DYLD_LIBRARY_PATH!')
+             print('export DYLD_LIBRARY_PATH={}'.format(libdir))
+             sys.exit()
+             #os.execv(sys.argv[0], sys.argv)
+
 
 def crtmLevelsToLayers( pLevels ):
     num = pLevels[1::] - pLevels[0:pLevels.shape[0]-1]
@@ -168,12 +256,14 @@ def readSpcCoeff(fname):
     n_Channels = o['n_Channels']
     f.read(8)
     # sensor information.
-    o['sensor_string'], o['sensor_type'], o['wmo_satellite_id'], o['wmo_sensor_id'] = struct.unpack('20s3i',f.read(struct.calcsize('20s3i')))
+    o['sensor_string'], o['Sensor_Type'], o['wmo_satellite_id'], o['wmo_sensor_id'] = struct.unpack('20s3i',f.read(struct.calcsize('20s3i')))
     f.read(8)
     #information we probably care about.
     fmt = '{:d}i'.format(n_Channels)
     o['Sensor_Channel'] = struct.unpack(fmt,f.read(struct.calcsize(fmt)))
     o['Polarization'] = struct.unpack(fmt,f.read(struct.calcsize(fmt)))
+    if(o['version']>2):
+        o['PolAngle'] = struct.unpack('{:d}d'.format(n_Channels),f.read(struct.calcsize('{:d}d'.format(n_Channels))))
     o['Channel_Flag'] = struct.unpack(fmt,f.read(struct.calcsize(fmt)))
 
     fmt = '{:d}d'.format(n_Channels)
@@ -197,6 +287,18 @@ def readSpcCoeff(fname):
     f.close()
 
     return spcCoeff
+def readSpcCoeffNc(fname):
+    """
+    Read Spectral Coefficient information from netcdf file.
+    """
+    o = {}
+    ds = netCDF4.Dataset(fname,'r')
+    for v in ds.variables:
+        o[v] = np.asarray(ds.variables[v])
+
+    for v in list(ds.ncattrs()):
+        o[v.lower()] = np.asarray(ds.getncattr(v))
+    return o
 
 if __name__ == "__main__":
     pathInfo = configparser.ConfigParser()