Skip to content

Commit

Permalink
Merge branch 'master' of https://github.com/geodesymiami/rsmas_insar
Browse files Browse the repository at this point in the history
  • Loading branch information
disilvestro committed Sep 4, 2024
2 parents 5cb2566 + 20c8153 commit f2bf795
Show file tree
Hide file tree
Showing 13 changed files with 439 additions and 127 deletions.
61 changes: 35 additions & 26 deletions minsar/create_insarmaps_jobfile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import sys
import glob
import argparse
import re
import h5py
from pathlib import Path
from minsar.objects import message_rsmas
from minsar.objects.auto_defaults import PathFind
Expand All @@ -17,6 +17,9 @@
sys.path.insert(0, os.getenv('SSARAHOME'))
import password_config as password

REMOTEHOST_INSARMAPS = os.getenv('REMOTEHOST_INSARMAPS')
REMOTEHOST_INSARMAPS2 = os.getenv('REMOTEHOST_INSARMAPS2')

pathObj = PathFind()
############################################################
EXAMPLE = """example:
Expand All @@ -30,7 +33,7 @@ def create_parser():
epilog = EXAMPLE
parser = argparse.ArgumentParser(description=synopsis, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('data_dir', nargs=1, help='Directory with hdf5eos file.\n')
parser.add_argument('--dataset', dest='dataset', choices=['PS', 'DS', 'PSDS', 'geo', 'all'], default='PS', help='Plot data as image or scatter (default: %(default)s).')
parser.add_argument('--dataset', dest='dataset', choices=['PS', 'DS', 'PSDS', 'geo', 'filt*DS','all'], default='geo', help='Plot data as image or scatter (default: %(default)s).')
parser.add_argument("--queue", dest="queue", metavar="QUEUE", default=os.getenv('QUEUENAME'), help="Name of queue to submit job to")
parser.add_argument('--walltime', dest='wall_time', metavar="WALLTIME (HH:MM)", default='1:00', help='job walltime (default=1:00)')

Expand All @@ -48,21 +51,20 @@ def main(iargs=None):
inps.num_data = 1
job_obj = JOB_SUBMIT(inps)

files = glob.glob(inps.work_dir + '/' + inps.data_dir[0] + '/*.he5')
all_files = glob.glob(inps.work_dir + '/' + inps.data_dir[0] + '/*.he5')

file_DS = None
file_PS = None
file_geo = [file for file in all_files if 'DS' not in file and 'PS' not in file]
file_PS = [file for file in all_files if 'PS' in file]
file_DS = [file for file in all_files if 'DS' in file and 'filt' not in file]
file_filtDS = [file for file in all_files if 'DS' in file and 'filt' in file]

for file in files:
if re.search(r'filt.*DS', file):
file_DS = file
elif 'PS' in file:
file_PS = file
else:
file_geo = file


job_name = f"insarmaps"
job_file_name = job_name

with h5py.File(file_geo[0], 'r') as f:
ref_lat = float(f.attrs['REF_LAT'])
ref_lon = float(f.attrs['REF_LON'])

files = []
suffixes = []
Expand All @@ -75,6 +77,9 @@ def main(iargs=None):
if inps.dataset == "DS":
files.append(file_DS)
suffixes.append("_DS")
if inps.dataset == "filt*DS":
files.append(file_filtDS)
suffixes.append("_filtDS")
if inps.dataset == "PSDS" or inps.dataset == "DSPS":
files.append(file_PS)
files.append(file_DS)
Expand All @@ -90,29 +95,33 @@ def main(iargs=None):

command = []
for file, suffix in zip(files, suffixes):
command.append( f'rm -r {inps.data_dir[0]}/JSON{suffix}\n' )
command.append( f'hdfeos5_2json_mbtiles.py {file} {inps.work_dir}/{inps.data_dir[0]}/JSON{suffix} --num-workers 8\n' )
command.append( f'rm -rf {inps.data_dir[0]}/JSON{suffix}' )
command.append( f'hdfeos5_2json_mbtiles.py {file[0]} {inps.work_dir}/{inps.data_dir[0]}/JSON{suffix} --num-workers 8' )

command.append('wait\n\n')
command.append('wait\n')
for file, suffix in zip(files, suffixes):
path_obj = Path(file)
path_obj = Path(file[0])
mbtiles_file = f"{path_obj.parent}/JSON{suffix}/{path_obj.name}"
mbtiles_file = mbtiles_file.replace('he5','mbtiles')
command.append( f'json_mbtiles2insarmaps.py --num-workers 8 -u {password.insaruser} -p {password.insarpass} --host insarmaps.miami.edu -P rsmastest -U rsmas\@gmail.com --json_folder {inps.work_dir}/{inps.data_dir[0]}/JSON{suffix} --mbtiles_file {mbtiles_file} &\n' )
command.append( f'json_mbtiles2insarmaps.py --num-workers 8 -u {password.docker_insaruser} -p {password.docker_insarpass} --host 149.165.174.11 -P insarmaps -U insarmaps@insarmaps.com --json_folder {inps.work_dir}/{inps.data_dir[0]}/JSON{suffix} --mbtiles_file {mbtiles_file} &\n' )
command.append( f'json_mbtiles2insarmaps.py --num-workers 8 -u {password.insaruser} -p {password.insarpass} --host {REMOTEHOST_INSARMAPS} -P {password.databasepass} -U {password.databaseuser} --json_folder {inps.work_dir}/{inps.data_dir[0]}/JSON{suffix} --mbtiles_file {mbtiles_file} &' )
command.append( f'json_mbtiles2insarmaps.py --num-workers 8 -u {password.docker_insaruser} -p {password.docker_insarpass} --host {REMOTEHOST_INSARMAPS2} -P {password.docker_databasepass} -U {password.docker_databaseuser} --json_folder {inps.work_dir}/{inps.data_dir[0]}/JSON{suffix} --mbtiles_file {mbtiles_file} &' )

command.append('wait\n\n')
str = [f'cat >> insarmaps.log<<EOF\n']
command.append('wait\n')
str = [f'cat >> insarmaps.log<<EOF']
for file in files:
base_name = os.path.basename(file)
base_name = os.path.basename(file[0])
name_without_extension = os.path.splitext(base_name)[0]
str.append(f"https://insarmaps.miami.edu/start/25.78/-80.3/11.0?flyToDatasetCenter=true&startDataset={name_without_extension}\n")
str.append(f"149.165.174.11/start/25.78/-80.3/11.0?flyToDatasetCenter=true&startDataset={name_without_extension}\n")
# str.append(f"https://insarmaps.miami.edu/start/25.78/-80.3/11.0?flyToDatasetCenter=true&startDataset={name_without_extension}")
# str.append(f"149.165.174.11/start/25.78/-80.3/11.0?flyToDatasetCenter=true&startDataset={name_without_extension}")

str.append(f"https://{REMOTEHOST_INSARMAPS}/start/{ref_lat:.1f}/{ref_lon:.1f}/11.0?flyToDatasetCenter=true&startDataset={name_without_extension}")
str.append(f"{REMOTEHOST_INSARMAPS2}/start/{ref_lat:.1f}/{ref_lon:.1f}/11.0?flyToDatasetCenter=true&startDataset={name_without_extension}")

str.append( 'EOF' )
command.append( "".join(str) )
command.append( '\n'.join(str) )
final_command =[ '\n'.join(command) ]

job_obj.submit_script(job_name, job_file_name, command, writeOnly='True')
job_obj.submit_script(job_name, job_file_name, final_command, writeOnly='True')
print('jobfile created: ',job_file_name + '.job')

return None
Expand Down
10 changes: 5 additions & 5 deletions minsar/create_runfiles.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,13 +95,13 @@ def get_bbox_from_template(inps, delta_lat, delta_lon):
"""generates boundingBox string from miaplpy.subset.lalo, mintpy.subset.lalo or intersectsWith POLYGON string"""

if 'miaplpy.subset.lalo' in inps.template.keys():
print("Creating topsStack.boundingBox using miaplpy.subset.lalo")
print("QQ0 Creating topsStack.boundingBox using miaplpy.subset.lalo: ",inps.template['miaplpy.subset.lalo'] )
boundingBox_string = convert_subset_lalo_to_boundingBox_string(inps.template['miaplpy.subset.lalo'], delta_lat, delta_lon)
elif 'mintpy.subset.lalo' in inps.template.keys():
print("Creating topsStack.boundingBox using mintpy.subset.lalo")
print("QQ0 Creating topsStack.boundingBox using mintpy.subset.lalo: ", inps.template['mintpy.subset.lalo'])
boundingBox_string = convert_subset_lalo_to_boundingBox_string(inps.template['mintpy.subset.lalo'], delta_lat, delta_lon)
else:
print("Creating topsStack.boundingBox using ssaraopt.intersectsWith")
print("QQ0 Creating topsStack.boundingBox using ssaraopt.intersectsWith: ", inps.template['ssaraopt.intersectsWith'])
#boundingBox_string = convert_intersectsWith_string_to_boundingBox_string(inps.template['ssaraopt.intersectsWith'], delta_lat=0.0, delta_lon=0.0)
boundingBox_string = convert_intersectsWith_string_to_boundingBox_string(inps.template['ssaraopt.intersectsWith'], delta_lat, delta_lon)

Expand All @@ -114,8 +114,8 @@ def main(iargs=None):

if 'topsStack.boundingBox' in inps.template:
if inps.template['topsStack.boundingBox'] == 'None':
inps.template['topsStack.boundingBox'] = get_bbox_from_template(inps, delta_lat=0.15, delta_lon=3)
print('QQ0',inps.template['topsStack.boundingBox'])
inps.template['topsStack.boundingBox'] = get_bbox_from_template(inps, delta_lat=0.10, delta_lon=3)
print('QQ0 New topsStack.boundingBox using delta_lat=0.10: ',inps.template['topsStack.boundingBox'])

if not iargs is None:
input_arguments = iargs
Expand Down
1 change: 1 addition & 0 deletions minsar/create_save_hdf5_jobfile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ def main(iargs=None):
command.append( 'source ' + os.path.dirname(os.path.abspath(__file__)) + '/utils/minsar_functions.bash' )
command.append( f'h5file=`ls *_??????_??????_???????_???????*_{prefix}PS.he5` ; add_ref_lalo_to_file $h5file' )
command.append( f'h5file=`ls *_??????_??????_???????_???????*_{prefix}DS.he5` ; add_ref_lalo_to_file $h5file' )
command.append( f'h5file=`ls *_??????_??????_???????_???????*_filt{prefix}DS.he5` ; add_ref_lalo_to_file $h5file' )

command.append( f'view.py --dpi 150 --noverbose --nodisplay --update --memory 4.0 temporalCoherence_lowpass_gaussian.h5 -c gray -v 0 1 &')
command.append( f'view.py --dpi 150 --noverbose --nodisplay --update --memory 4.0 maskTempCoh_lowpass_gaussian.h5 -c gray -v 0 1 &')
Expand Down
94 changes: 66 additions & 28 deletions minsar/generate_download_command.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@

import os
import sys
import time
import datetime
import re
import argparse
from minsar.objects.dataset_template import Template
from minsar.objects import message_rsmas
Expand Down Expand Up @@ -55,7 +54,7 @@ def create_parser():

###############################################
def generate_intersects_string(dataset_template, delta_lat=0.0):
"""generates intersectsWith polygon string from miaplpy.subset.lalo or *Stack.boundingBox"""
"""generates intersectsWith polygon string from miaplpy.subset.lalo, mintpy.subset.lalo or *Stack.boundingBox"""

if not 'acquisition_mode' in dataset_template.get_options():
print('WARNING: "acquisition_mode" is not given --> default: tops (available options: tops, stripmap)')
Expand All @@ -65,28 +64,28 @@ def generate_intersects_string(dataset_template, delta_lat=0.0):


if 'miaplpy.subset.lalo' in dataset_template.get_options():
print("Creating intersectsWith string using miaplpy.subset.lalo")
intersects_string = convert_subset_lalo_to_intersects_string(dataset_template.get_options()['miaplpy.subset.lalo'])
print("QQ0 Creating intersectsWith string using miaplpy.subset.lalo: ", dataset_template.get_options()['miaplpy.subset.lalo'])
intersects_string = convert_subset_lalo_to_intersects_string(dataset_template.get_options()['miaplpy.subset.lalo'], delta_lat)
elif 'mintpy.subset.lalo' in dataset_template.get_options():
print("Creating intersectsWith string using mintpy.subset.lalo")
intersects_string = convert_subset_lalo_to_intersects_string(dataset_template.get_options()['mintpy.subset.lalo'])
print("QQ0 Creating intersectsWith string using mintpy.subset.lalo: dataset_template.get_options()['mintpy.subset.lalo']")
intersects_string = convert_subset_lalo_to_intersects_string(dataset_template.get_options()['mintpy.subset.lalo'], delta_lat)
else:
print("Creating intersectsWith string using *Stack.boundingBox")
print("QQ0 Creating intersectsWith string using *Stack.boundingBox: ", dataset_template.get_options()[prefix + 'Stack.boundingBox'])
intersects_string = convert_bounding_box_to_intersects_string(dataset_template.get_options()[prefix + 'Stack.boundingBox'], delta_lat)

return intersects_string

###############################################
def convert_subset_lalo_to_intersects_string(subset_lalo):
def convert_subset_lalo_to_intersects_string(subset_lalo, delta_lat=0):
""" Converts a subset.lalo string in S:N,E:W format (e.g., "2.7:2.8,125.3:125.4") to an intersectsWith polygon string."""

lat_string = subset_lalo.split(',')[0]
lon_string = subset_lalo.split(',')[1]

min_lat = float(lat_string.split(':')[0])
max_lat = float(lat_string.split(':')[1])
min_lon = float(lon_string.split(':')[0])
max_lon = float(lon_string.split(':')[1])
min_lat = float(lat_string.split(':')[0]) - delta_lat
max_lat = float(lat_string.split(':')[1]) + delta_lat
min_lon = float(lon_string.split(':')[0]) - delta_lat/2
max_lon = float(lon_string.split(':')[1]) + delta_lat/2

intersects_string = '--intersectsWith=\'Polygon(({:.2f} {:.2f}, {:.2f} {:.2f}, {:.2f} {:.2f}, {:.2f} {:.2f}, ' \
'{:.2f} {:.2f}))\''.format(min_lon, min_lat, min_lon, max_lat, max_lon, max_lat, max_lon, min_lat, min_lon, min_lat)
Expand Down Expand Up @@ -119,36 +118,75 @@ def convert_bounding_box_to_intersects_string(string_bbox, delta_lat):

return intersects_string

###############################################
def convert_intersects_string_to_extent_string(intersects_string):
""" Converts a intersectsWith string to an extent string."""

match = re.search(r"Polygon\(\((.*?)\)\)", intersects_string)
if match:
polygon_str = match.group(1)
else:
polygon_str = None

lon_list = []
lat_list = []
bbox_list = polygon_str.split(',')
for bbox in bbox_list:
lon, lat = map(float, bbox.split())
lon_list.append(lon)
lat_list.append(lat)
lon_list.sort()
lat_list.sort()

extent_list = [lon_list[0], lat_list[0], lon_list [-1], lat_list[-1]]
extent_str = ' '.join(map(str, extent_list))

return extent_str, extent_list

###############################################
def generate_download_command(template):
""" generate ssara download options to use """

dataset_template = Template(template)
dataset_template.options.update(pathObj.correct_for_ssara_date_format(dataset_template.options))

ssaraopt_string = dataset_template.generate_ssaraopt_string()
ssaraopt_string, ssaraopt_dict = dataset_template.generate_ssaraopt_string()
ssaraopt = ssaraopt_string.split(' ')
if 'end' not in ssaraopt_dict:
ssaraopt_dict['end'] = '2099-12-31'

if not any(option.startswith('ssaraopt.intersectsWith') for option in dataset_template.get_options()):
# Your code here
# if 'ssaraopt.intersectsWith' not in dataset_template.get_options():
intersects_string = generate_intersects_string(dataset_template)
intersects_string = generate_intersects_string(dataset_template, delta_lat=0.1)
ssaraopt.insert(2, intersects_string)
ssaraopt.append('--maxResults=20000')

extent_str, extent_list = convert_intersects_string_to_extent_string(intersects_string)
print('QQ0 New intersectsWith sting using delta_lat=0.1: ', intersects_string)
print('QQ0 New extent sting using delta_lat=0.1: ', extent_str)

ssara_cmd = ['ssara_federated_query.bash'] + ssaraopt
ssara_cmd_python = ['ssara_federated_query.py'] + ssaraopt + ['--asfResponseTimeout=300', '--kml', '--print', '--download']
asf_cmd = ['asf_search_args.py'] + ssaraopt + ['--Product=SLC', '--print', '--download']
ssara_cmd_slc_download_bash = ['ssara_federated_query.bash'] + ssaraopt
ssara_cmd_kml_download_python = ['ssara_federated_query.py'] + ssaraopt + ['--maxResults=20000','--asfResponseTimeout=300', '--kml']
ssara_cmd_slc_download_python = ['ssara_federated_query.py'] + ssaraopt + ['--maxResults=20000','--asfResponseTimeout=300', '--kml', '--print','--download']

#asf_cmd = [item for item in asf_cmd if "SENTINEL" not in item]
asf_cmd = [item for item in asf_cmd if "maxResults" not in item]
asf_cmd_slc_download = ['asf_search_args.py', '--product=SLC'] + ssaraopt + ['--print', '--download']
asf_cmd_burst_download = ['asf_search_args.py', '--product=BURST'] + ssaraopt + ['--print', '--download']
asf_cmd_burst2safe = ['burst2stack','--rel-orbit',ssaraopt_dict['relativeOrbit'],'--start-date',ssaraopt_dict['start'],'--end-date',ssaraopt_dict['end'],'--extent',extent_str]
asf_cmd_burst2safe = ['burst2stack','--rel-orbit',ssaraopt_dict['relativeOrbit'],'--start-date',ssaraopt_dict['start'],'--end-date',ssaraopt_dict['end'],'--extent',extent_str]
asf_cmd_burst2safe.insert(0, "srun -n1 -N1 -A $JOBSHEDULER_PROJECTNAME -p $QUEUENAME -t 00:25:00 ") #FA 8/24: it should create a burst2safe.job

with open('ssara_command.txt', 'w') as f:
f.write(' '.join(ssara_cmd) + '\n')
# with open('ssara_command_python.txt', 'w') as f:
# f.write(' '.join(ssara_cmd_python) + '\n')
with open('asf_command.txt', 'w') as f:
f.write(' '.join(asf_cmd) + '\n')
f.write(' '.join(ssara_cmd_slc_download_bash) + '\n')

with open('asf_slc_download.txt', 'w') as f:
f.write(' '.join(asf_cmd_slc_download) + '\n')

with open('asf_burst_download.txt', 'w') as f:
f.write(' '.join(ssara_cmd_kml_download_python) + '\n')
f.write(' '.join(asf_cmd_burst_download) + '\n')
f.write(' '.join(asf_cmd_burst2safe ) + '\n')

ssara_cmd_python = ['ssara_federated_query.py'] + ssaraopt + ['--maxResults=20000','--asfResponseTimeout=300', '--kml', '--print']
with open('ssara_command_python.txt', 'w') as f:
f.write(' '.join(ssara_cmd_python) + '\n')

return

Expand Down
Loading

0 comments on commit f2bf795

Please sign in to comment.