Add ISCE3 processor support for data loading (prep_isce3.py)

.github/workflows/build-n-publish-to-pypi.yml

@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python 3.10
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: "3.10"
 

.pre-commit-config.yaml

@@ -27,7 +27,7 @@ repos:
         exclude: tests/data/
 
   - repo: https://github.com/PyCQA/isort
-    rev: "9.0.0b1"
+    rev: "9.0.0b5"
     hooks:
       - id: isort
         name: sort imports

pyproject.toml

@@ -67,6 +67,7 @@ Issues = "https://github.com/insarlab/MintPy/issues"
 "prep_gmtsar.py" = "mintpy.cli.prep_gmtsar:main"
 "prep_hyp3.py" = "mintpy.cli.prep_hyp3:main"
 "prep_isce.py" = "mintpy.cli.prep_isce:main"
+"prep_isce3.py" = "mintpy.cli.prep_isce3:main"
 "prep_nisar.py" = "mintpy.cli.prep_nisar:main"
 "prep_roipac.py" = "mintpy.cli.prep_roipac:main"
 "prep_snap.py" = "mintpy.cli.prep_snap:main"

src/mintpy/cli/prep_isce3.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+############################################################
+# Program is part of MintPy                                #
+# Copyright (c) 2013, Zhang Yunjun, Heresh Fattahi         #
+# Author: Zhang Yunjun, 2024                               #
+############################################################
+
+
+import glob
+import os
+import sys
+
+from mintpy.utils.arg_utils import create_argument_parser
+from mintpy.utils.isce3_utils import GEOMETRY_FILENAMES
+
+#########################################################################
+# Default geometry files to extract from static_layers HDF5
+# (single source of truth defined in mintpy.utils.isce3_utils)
+
+EXAMPLE = """example:
+  ## Dolphin/ISCE-3 topsStack (auto‑generate metadata)
+  prep_isce3.py -f "../../dolphin/unwrapped/*.unw.tif" -b ../baselines -g ../merged/geom/
+
+  ## with existing metadata file
+  prep_isce3.py -f "../../dolphin/unwrapped/*.unw.tif" -m ../reference/IW1.xml -g ../merged/geom/
+
+  ## force overwrite existing .rsc files
+  prep_isce3.py -f "../../dolphin/unwrapped/*.unw.tif" -g ../merged/geom/ --force
+"""
+
+def create_parser(subparsers=None):
+    """Command line parser."""
+    synopsis = 'Prepare ISCE-3 / Dolphin metadata files.'
+    epilog = EXAMPLE
+    name = __name__.split('.')[-1]
+    parser = create_argument_parser(
+        name, synopsis=synopsis, description=synopsis, epilog=epilog, subparsers=subparsers)
+
+    # observations
+    parser.add_argument('-f', dest='obs_files', type=str, nargs='+',
+                        default=['../../dolphin/unwrapped/*.unw.tif'],
+                        help='Wildcard path pattern(s) for observation files.\n'
+                             'E.g.: unwrapped phase:   ../../dolphin/unwrapped/*.unw.tif\n'
+                             '      coherence:        ../../dolphin/interferograms/*.int.cor.tif\n'
+                             '      wrapped phase:    ../../dolphin/interferograms/*.int.tif\n'
+                             '      connected comp:   ../../dolphin/unwrapped/*.unw.conncomp.tif')
+
+    # metadata
+    parser.add_argument('-m', '--meta-file', dest='meta_file', type=str, default=None,
+                        help='Metadata file to extract common metadata for the stack.\n'
+                             'E.g.: reference burst XML (e.g., reference/IW1.xml). '
+                             'If not provided, one will be generated from static_layers.h5.')
+
+    # geometry and baseline
+    parser.add_argument('-b', '--baseline-dir', dest='baseline_dir', type=str, default=None,
+                        help='Directory with baseline files (e.g., ../baselines). '
+                             'If omitted, baseline info will not be added to metadata.')
+    parser.add_argument('-g', '--geometry-dir', dest='geom_dir', type=str, required=True,
+                        help='Directory containing burst subdirectories with static_layers*.h5 files.\n'
+                             'E.g.: ../merged/geom/')
+    parser.add_argument('--geom-files', dest='geom_files', type=str, nargs='*',
+                        default=GEOMETRY_FILENAMES,
+                        help='List of geometry file basenames to extract/merge. Default: %(default)s.')
+
+    # processing flag
+    parser.add_argument('--force', dest='update_mode', action='store_false',
+                        help='Force to overwrite all .rsc metadata files (disable update mode).')
+
+    parser.add_argument('--out-dir', dest='out_dir', type=str, default=None,
+                        help='Output directory for merged geometry files. '
+                             'If not provided, defaults to (geometry_dir)/../merged_geom')
+
+    return parser
+
+
+def cmd_line_parse(iargs=None):
+    parser = create_parser()
+    inps = parser.parse_args(args=iargs)
+
+    if inps.meta_file and '*' in inps.meta_file:
+        fnames = glob.glob(inps.meta_file)
+        if fnames:
+            inps.meta_file = fnames[0]
+        else:
+            raise FileNotFoundError(inps.meta_file)
+
+    # Expand glob patterns in geometry directory (e.g. "../../t124*/20210104/")
+    inps.geom_dirs = [inps.geom_dir]
+    if inps.geom_dir and ('*' in inps.geom_dir or '?' in inps.geom_dir):
+        matches = sorted(glob.glob(inps.geom_dir))
+        if matches:
+            inps.geom_dir = matches[0]
+            inps.geom_dirs = matches
+
+    # Set default output directory if not provided
+    if inps.out_dir is None:
+        inps.out_dir = os.path.join(os.path.dirname(inps.geom_dir), 'merged_geom')
+
+    inps.processor = 'tops'
+    return inps
+
+
+#########################################################################
+def main(iargs=None):
+    # parse
+    inps = cmd_line_parse(iargs)
+
+    # import core function
+    from mintpy.prep_isce3 import prep_isce3
+
+    # run
+    prep_isce3(inps)
+
+
+#########################################################################
+if __name__ == '__main__':
+    main(sys.argv[1:])

src/mintpy/defaults/smallbaselineApp.cfg

@@ -26,14 +26,14 @@ mintpy.compute.config    = auto #[none / slurm / pbs / lsf ], auto for none (sam
 ## no   - save   0% disk usage, fast [default]
 ## lzf  - save ~57% disk usage, relative slow
 ## gzip - save ~62% disk usage, very slow [not recommend]
-mintpy.load.processor       = auto  #[isce, aria, hyp3, gmtsar, snap, gamma, roipac, nisar], auto for isce
+mintpy.load.processor       = auto  #[isce, isce3, aria, hyp3, gmtsar, snap, gamma, roipac, nisar], auto for isce
 mintpy.load.autoPath        = auto  #[yes / no], auto for no, use pre-defined auto path
 mintpy.load.updateMode      = auto  #[yes / no], auto for yes, skip re-loading if HDF5 files are complete
 mintpy.load.compression     = auto  #[gzip / lzf / none / default], auto for default (none/lzf for stack/geometry).
 mintpy.load.frequency       = auto  #[auto / A / B], auto for A, NISAR only
 ##---------for ISCE only:
-mintpy.load.metaFile        = auto  #[path of common metadata file for the stack], i.e.: ./reference/IW1.xml, ./referenceShelve/data.dat
-mintpy.load.baselineDir     = auto  #[path of the baseline dir], i.e.: ./baselines
+mintpy.load.metaFile        = auto  #[path of common metadata file for the stack], i.e.: ./reference/IW1.xml, ./referenceShelve/data.dat. For isce3: reference burst XML; auto generates one from static_layers.h5
+mintpy.load.baselineDir     = auto  #[path of the baseline dir], i.e.: ./baselines. For isce3/Dolphin: the burst dir, e.g. ../../baselines/t124*/
 ##---------interferogram stack:
 mintpy.load.unwFile         = auto  #[path pattern of unwrapped interferogram files]
 mintpy.load.corFile         = auto  #[path pattern of spatial coherence       files]
@@ -51,6 +51,7 @@ mintpy.load.azOffStdFile    = auto  #[path pattern of azimuth offset variance fi
 mintpy.load.rgOffStdFile    = auto  #[path pattern of range   offset variance file], optional but recommended
 mintpy.load.offSnrFile      = auto  #[path pattern of offset signal-to-noise ratio file], optional
 ##---------geometry:
+mintpy.load.geomSrcDir      = auto  #[path of static-layer h5 files for geometry info], optional, default to the directory of DEM file. For isce3: burst dir(s) with static_layers*.h5, e.g. ../../t124*/20210104/
 mintpy.load.demFile         = auto  #[path of DEM file]
 mintpy.load.lookupYFile     = auto  #[path of latitude /row   /y coordinate file], not required for geocoded data
 mintpy.load.lookupXFile     = auto  #[path of longitude/column/x coordinate file], not required for geocoded data

src/mintpy/ifgram_inversion.py

@@ -521,7 +521,7 @@ def calc_weight_sqrt(stack_obj, box, weight_func='var', dropIfgram=True, chunk_s
         L = float(stack_obj.metadata['NCORRLOOKS'])
     else:
         # use the typical ratio of resolution vs pixel size of Sentinel-1 IW mode
-        L = int(stack_obj.metadata['ALOOKS']) * int(stack_obj.metadata['RLOOKS'])
+        L = int(float(stack_obj.metadata['ALOOKS'])) * int(float(stack_obj.metadata['RLOOKS']))
         L /= 1.94
     # make sure L >= 1
     L = max(np.rint(L).astype(int), 1)

src/mintpy/load_data.py

@@ -24,7 +24,7 @@
 from mintpy.utils import ptime, readfile, utils as ut
 
 #################################################################
-PROCESSOR_LIST = ['isce', 'aria', 'hyp3', 'gmtsar', 'snap', 'gamma', 'roipac', 'cosicorr', 'nisar']
+PROCESSOR_LIST = ['isce', 'aria', 'hyp3', 'gmtsar', 'snap', 'gamma', 'roipac', 'cosicorr', 'nisar', 'isce3']
 
 # primary observation dataset names
 OBS_DSET_NAMES = ['unwrapPhase', 'rangeOffset', 'azimuthOffset']
@@ -449,7 +449,7 @@ def read_inps_dict2geometry_dict_object(iDict, dset_name2template_key):
         # for processors with lookup table in geo-coordinates, remove latitude/longitude
         dset_name2template_key.pop('latitude')
         dset_name2template_key.pop('longitude')
-    elif iDict['processor'] in ['aria', 'gmtsar', 'hyp3', 'snap', 'cosicorr']:
+    elif iDict['processor'] in ['aria', 'gmtsar', 'hyp3', 'snap', 'cosicorr', 'isce3']:
         # for processors with geocoded products support only, do nothing for now.
         # check again when adding products support in radar-coordiantes
         pass
@@ -676,6 +676,49 @@ def prepare_metadata(iDict):
         except:
             warnings.warn('prep_nisar.py failed. Assuming its result exists and continue...')
 
+    elif processor == 'isce3':
+
+        meta_files = sorted(glob.glob(iDict['mintpy.load.metaFile'])) if iDict.get('mintpy.load.metaFile') else []
+        meta_file = meta_files[0] if meta_files else 'auto'
+
+        baseline_dir = iDict.get('mintpy.load.baselineDir', None)
+
+        # Geometry source directory (contains burst subdirs with HDF5)
+        geom_src_dir = iDict.get('mintpy.load.geomSrcDir', None)
+        if geom_src_dir is None or geom_src_dir.lower() == 'auto':
+            dem_path = iDict.get('mintpy.load.demFile', '')
+            if dem_path and dem_path.lower() != 'auto':
+                geom_src_dir = os.path.dirname(dem_path)
+            else:
+                geom_src_dir = os.path.abspath('.')
+        # NOTE: keep glob patterns (e.g. "../CSLC/t124*/20240611/") as-is,
+        # prep_isce3.py expands them to support multiple burst directories.
+        geom_src_dir = os.path.abspath(geom_src_dir)
+
+        # Output directory for merged geometry (same as demFile's directory)
+        dem_file = iDict.get('mintpy.load.demFile', '')
+        if dem_file and dem_file.lower() != 'auto':
+            out_dir = os.path.dirname(os.path.abspath(dem_file))
+        else:
+            first_match = (sorted(glob.glob(geom_src_dir)) or [geom_src_dir])[0]
+            out_dir = os.path.join(os.path.dirname(first_match), 'merged_geom')
+
+        obs_keys = ['mintpy.load.unwFile', 'mintpy.load.corFile', 'mintpy.load.connCompFile']
+        obs_paths = [iDict[key] for key in obs_keys if iDict.get(key, 'auto').lower() != 'auto']
+        obs_paths = [x for x in obs_paths if glob.glob(x)]
+
+        iargs = ['-m', meta_file, '-g', geom_src_dir, '--out-dir', out_dir]
+        if baseline_dir:
+            iargs += ['-b', baseline_dir]
+        if obs_paths:
+            iargs += ['-f'] + obs_paths
+        if not iDict.get('updateMode', True):
+            iargs.append('--force')
+
+        ut.print_command_line('prep_isce3.py', iargs)
+        prep_module = importlib.import_module('mintpy.cli.prep_isce3')
+        prep_module.main(iargs)
+
     elif processor == 'isce':
         from mintpy.utils import isce_utils, s1_utils
 

src/mintpy/objects/sensor.py

@@ -476,6 +476,7 @@ def get_unavco_mission_name(meta_dict):
     'IW2' : {'range_resolution' : 3.1, 'azimuth_resolution': 22.7},
     'IW3' : {'range_resolution' : 3.5, 'azimuth_resolution': 22.6},
     'noise_equivalent_sigma_zero': -22,       # dB
+    'incidence_angle'            : [20, 47],  # degrees for Strip Map mode; 31-46 for IW mode
 }
 
 
@@ -651,6 +652,7 @@ def get_unavco_mission_name(meta_dict):
     'chirp_bandwidth'            : 84e6,      # Hz, 84/42/28
     'range_resolution'           : 3,         # m
     'noise_equivalent_sigma_zero': -20,       # dB, -20/-24/-28
+    'incidence_angle'            : [30, 56],  # degrees for Strip Map mode
 }
 
 # SAOCOM-1A/B stripmap
@@ -695,6 +697,7 @@ def get_unavco_mission_name(meta_dict):
     'range_pixel_size'           : 1.67,      # m
     'range_resolution'           : 2.50,      # m
     'noise_equivalent_sigma_zero': -28,       # dB
+    'incidence_angle'            : [20,46],   # degree for STRIP1/2 InSAR
 }
 
 # UAVSAR-L
@@ -744,6 +747,7 @@ def get_unavco_mission_name(meta_dict):
         '80MHz'                  : 1.87,      # m
     },
     'noise_equivalent_sigma_zero': -25,       # dB
+    'incidence_angle'            : [34, 48],  # degrees
 }
 
 

src/mintpy/objects/stack.py

@@ -12,6 +12,7 @@
 import itertools
 import os
 import time
+import warnings
 
 import h5py
 import numpy as np
@@ -993,6 +994,10 @@ def nonzero_mask(self, datasetName=None, print_msg=True, dropIfgram=True):
             for i in range(num2read):
                 prog_bar.update(i+1, suffix=f'{i+1}/{num2read}')
                 data = dset[idx2read[i], :, :]
+                if np.all(data == 0.) or np.all(np.isnan(data)):
+                    if print_msg:
+                        print(f'WARNING: ifgram {idx2read[i]} has all-zero/all-NaN {datasetName}, skipping')
+                    continue
                 mask[data == 0.] = 0
                 mask[np.isnan(data)] = 0
             prog_bar.close()
@@ -1048,6 +1053,11 @@ def temporal_average(self, datasetName='coherence', dropIfgram=True, max_memory=
 
                 # referencing / normalizing for phase
                 if 'unwrapPhase' in datasetName:
+                    # keep MintPy's convention that a phase value of 0.0 indicates
+                    # no-data: convert them to NaN so they are excluded from the
+                    # temporal average (np.nanmean below), consistent with the
+                    # data != 0. check in read_stack_obs().
+                    data[data == 0.] = np.nan
                     # spatial referencing
                     if ref_val is not None:
                         data -= np.tile(ref_val.reshape(-1, 1, 1), (1, data.shape[1], data.shape[2]))
@@ -1056,7 +1066,11 @@ def temporal_average(self, datasetName='coherence', dropIfgram=True, max_memory=
                         data[j,:,:] *= (phase2range / tbase[j])
 
                 # use nanmean to better handle NaN values
-                dmean[r0:r1, :] = np.nanmean(data, axis=0)
+                # suppress the Mean of empty slice RuntimeWarning for pixels
+                # with no valid observation (all 0.0 / NaN)
+                with warnings.catch_warnings():
+                    warnings.simplefilter('ignore', RuntimeWarning)
+                    dmean[r0:r1, :] = np.nanmean(data, axis=0)
             prog_bar.close()
         return dmean