diff --git a/minsar/create_insarmaps_jobfile.py b/minsar/create_insarmaps_jobfile.py index fc568705..a6688ad3 100755 --- a/minsar/create_insarmaps_jobfile.py +++ b/minsar/create_insarmaps_jobfile.py @@ -7,7 +7,7 @@ import sys import glob import argparse -import re +import h5py from pathlib import Path from minsar.objects import message_rsmas from minsar.objects.auto_defaults import PathFind @@ -17,6 +17,9 @@ sys.path.insert(0, os.getenv('SSARAHOME')) import password_config as password +REMOTEHOST_INSARMAPS = os.getenv('REMOTEHOST_INSARMAPS') +REMOTEHOST_INSARMAPS2 = os.getenv('REMOTEHOST_INSARMAPS2') + pathObj = PathFind() ############################################################ EXAMPLE = """example: @@ -30,7 +33,7 @@ def create_parser(): epilog = EXAMPLE parser = argparse.ArgumentParser(description=synopsis, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('data_dir', nargs=1, help='Directory with hdf5eos file.\n') - parser.add_argument('--dataset', dest='dataset', choices=['PS', 'DS', 'PSDS', 'geo', 'all'], default='PS', help='Plot data as image or scatter (default: %(default)s).') + parser.add_argument('--dataset', dest='dataset', choices=['PS', 'DS', 'PSDS', 'geo', 'filt*DS','all'], default='geo', help='Plot data as image or scatter (default: %(default)s).') parser.add_argument("--queue", dest="queue", metavar="QUEUE", default=os.getenv('QUEUENAME'), help="Name of queue to submit job to") parser.add_argument('--walltime', dest='wall_time', metavar="WALLTIME (HH:MM)", default='1:00', help='job walltime (default=1:00)') @@ -48,21 +51,20 @@ def main(iargs=None): inps.num_data = 1 job_obj = JOB_SUBMIT(inps) - files = glob.glob(inps.work_dir + '/' + inps.data_dir[0] + '/*.he5') + all_files = glob.glob(inps.work_dir + '/' + inps.data_dir[0] + '/*.he5') - file_DS = None - file_PS = None + file_geo = [file for file in all_files if 'DS' not in file and 'PS' not in file] + file_PS = [file for file in all_files if 'PS' in file] + file_DS = [file for file in all_files if 'DS' in file and 'filt' not in file] + file_filtDS = [file for file in all_files if 'DS' in file and 'filt' in file] - for file in files: - if re.search(r'filt.*DS', file): - file_DS = file - elif 'PS' in file: - file_PS = file - else: - file_geo = file - + job_name = f"insarmaps" job_file_name = job_name + + with h5py.File(file_geo[0], 'r') as f: + ref_lat = float(f.attrs['REF_LAT']) + ref_lon = float(f.attrs['REF_LON']) files = [] suffixes = [] @@ -75,6 +77,9 @@ def main(iargs=None): if inps.dataset == "DS": files.append(file_DS) suffixes.append("_DS") + if inps.dataset == "filt*DS": + files.append(file_filtDS) + suffixes.append("_filtDS") if inps.dataset == "PSDS" or inps.dataset == "DSPS": files.append(file_PS) files.append(file_DS) @@ -90,29 +95,33 @@ def main(iargs=None): command = [] for file, suffix in zip(files, suffixes): - command.append( f'rm -r {inps.data_dir[0]}/JSON{suffix}\n' ) - command.append( f'hdfeos5_2json_mbtiles.py {file} {inps.work_dir}/{inps.data_dir[0]}/JSON{suffix} --num-workers 8\n' ) + command.append( f'rm -rf {inps.data_dir[0]}/JSON{suffix}' ) + command.append( f'hdfeos5_2json_mbtiles.py {file[0]} {inps.work_dir}/{inps.data_dir[0]}/JSON{suffix} --num-workers 8' ) - command.append('wait\n\n') + command.append('wait\n') for file, suffix in zip(files, suffixes): - path_obj = Path(file) + path_obj = Path(file[0]) mbtiles_file = f"{path_obj.parent}/JSON{suffix}/{path_obj.name}" mbtiles_file = mbtiles_file.replace('he5','mbtiles') - command.append( f'json_mbtiles2insarmaps.py --num-workers 8 -u {password.insaruser} -p {password.insarpass} --host insarmaps.miami.edu -P rsmastest -U rsmas\@gmail.com --json_folder {inps.work_dir}/{inps.data_dir[0]}/JSON{suffix} --mbtiles_file {mbtiles_file} &\n' ) - command.append( f'json_mbtiles2insarmaps.py --num-workers 8 -u {password.docker_insaruser} -p {password.docker_insarpass} --host 149.165.174.11 -P insarmaps -U insarmaps@insarmaps.com --json_folder {inps.work_dir}/{inps.data_dir[0]}/JSON{suffix} --mbtiles_file {mbtiles_file} &\n' ) + command.append( f'json_mbtiles2insarmaps.py --num-workers 8 -u {password.insaruser} -p {password.insarpass} --host {REMOTEHOST_INSARMAPS} -P {password.databasepass} -U {password.databaseuser} --json_folder {inps.work_dir}/{inps.data_dir[0]}/JSON{suffix} --mbtiles_file {mbtiles_file} &' ) + command.append( f'json_mbtiles2insarmaps.py --num-workers 8 -u {password.docker_insaruser} -p {password.docker_insarpass} --host {REMOTEHOST_INSARMAPS2} -P {password.docker_databasepass} -U {password.docker_databaseuser} --json_folder {inps.work_dir}/{inps.data_dir[0]}/JSON{suffix} --mbtiles_file {mbtiles_file} &' ) - command.append('wait\n\n') - str = [f'cat >> insarmaps.log<> insarmaps.log< default: tops (available options: tops, stripmap)') @@ -65,28 +64,28 @@ def generate_intersects_string(dataset_template, delta_lat=0.0): if 'miaplpy.subset.lalo' in dataset_template.get_options(): - print("Creating intersectsWith string using miaplpy.subset.lalo") - intersects_string = convert_subset_lalo_to_intersects_string(dataset_template.get_options()['miaplpy.subset.lalo']) + print("QQ0 Creating intersectsWith string using miaplpy.subset.lalo: ", dataset_template.get_options()['miaplpy.subset.lalo']) + intersects_string = convert_subset_lalo_to_intersects_string(dataset_template.get_options()['miaplpy.subset.lalo'], delta_lat) elif 'mintpy.subset.lalo' in dataset_template.get_options(): - print("Creating intersectsWith string using mintpy.subset.lalo") - intersects_string = convert_subset_lalo_to_intersects_string(dataset_template.get_options()['mintpy.subset.lalo']) + print("QQ0 Creating intersectsWith string using mintpy.subset.lalo: dataset_template.get_options()['mintpy.subset.lalo']") + intersects_string = convert_subset_lalo_to_intersects_string(dataset_template.get_options()['mintpy.subset.lalo'], delta_lat) else: - print("Creating intersectsWith string using *Stack.boundingBox") + print("QQ0 Creating intersectsWith string using *Stack.boundingBox: ", dataset_template.get_options()[prefix + 'Stack.boundingBox']) intersects_string = convert_bounding_box_to_intersects_string(dataset_template.get_options()[prefix + 'Stack.boundingBox'], delta_lat) return intersects_string ############################################### -def convert_subset_lalo_to_intersects_string(subset_lalo): +def convert_subset_lalo_to_intersects_string(subset_lalo, delta_lat=0): """ Converts a subset.lalo string in S:N,E:W format (e.g., "2.7:2.8,125.3:125.4") to an intersectsWith polygon string.""" lat_string = subset_lalo.split(',')[0] lon_string = subset_lalo.split(',')[1] - min_lat = float(lat_string.split(':')[0]) - max_lat = float(lat_string.split(':')[1]) - min_lon = float(lon_string.split(':')[0]) - max_lon = float(lon_string.split(':')[1]) + min_lat = float(lat_string.split(':')[0]) - delta_lat + max_lat = float(lat_string.split(':')[1]) + delta_lat + min_lon = float(lon_string.split(':')[0]) - delta_lat/2 + max_lon = float(lon_string.split(':')[1]) + delta_lat/2 intersects_string = '--intersectsWith=\'Polygon(({:.2f} {:.2f}, {:.2f} {:.2f}, {:.2f} {:.2f}, {:.2f} {:.2f}, ' \ '{:.2f} {:.2f}))\''.format(min_lon, min_lat, min_lon, max_lat, max_lon, max_lat, max_lon, min_lat, min_lon, min_lat) @@ -119,6 +118,31 @@ def convert_bounding_box_to_intersects_string(string_bbox, delta_lat): return intersects_string +############################################### +def convert_intersects_string_to_extent_string(intersects_string): + """ Converts a intersectsWith string to an extent string.""" + + match = re.search(r"Polygon\(\((.*?)\)\)", intersects_string) + if match: + polygon_str = match.group(1) + else: + polygon_str = None + + lon_list = [] + lat_list = [] + bbox_list = polygon_str.split(',') + for bbox in bbox_list: + lon, lat = map(float, bbox.split()) + lon_list.append(lon) + lat_list.append(lat) + lon_list.sort() + lat_list.sort() + + extent_list = [lon_list[0], lat_list[0], lon_list [-1], lat_list[-1]] + extent_str = ' '.join(map(str, extent_list)) + + return extent_str, extent_list + ############################################### def generate_download_command(template): """ generate ssara download options to use """ @@ -126,29 +150,43 @@ def generate_download_command(template): dataset_template = Template(template) dataset_template.options.update(pathObj.correct_for_ssara_date_format(dataset_template.options)) - ssaraopt_string = dataset_template.generate_ssaraopt_string() + ssaraopt_string, ssaraopt_dict = dataset_template.generate_ssaraopt_string() ssaraopt = ssaraopt_string.split(' ') + if 'end' not in ssaraopt_dict: + ssaraopt_dict['end'] = '2099-12-31' if not any(option.startswith('ssaraopt.intersectsWith') for option in dataset_template.get_options()): - # Your code here - # if 'ssaraopt.intersectsWith' not in dataset_template.get_options(): - intersects_string = generate_intersects_string(dataset_template) + intersects_string = generate_intersects_string(dataset_template, delta_lat=0.1) ssaraopt.insert(2, intersects_string) - ssaraopt.append('--maxResults=20000') + + extent_str, extent_list = convert_intersects_string_to_extent_string(intersects_string) + print('QQ0 New intersectsWith sting using delta_lat=0.1: ', intersects_string) + print('QQ0 New extent sting using delta_lat=0.1: ', extent_str) - ssara_cmd = ['ssara_federated_query.bash'] + ssaraopt - ssara_cmd_python = ['ssara_federated_query.py'] + ssaraopt + ['--asfResponseTimeout=300', '--kml', '--print', '--download'] - asf_cmd = ['asf_search_args.py'] + ssaraopt + ['--Product=SLC', '--print', '--download'] + ssara_cmd_slc_download_bash = ['ssara_federated_query.bash'] + ssaraopt + ssara_cmd_kml_download_python = ['ssara_federated_query.py'] + ssaraopt + ['--maxResults=20000','--asfResponseTimeout=300', '--kml'] + ssara_cmd_slc_download_python = ['ssara_federated_query.py'] + ssaraopt + ['--maxResults=20000','--asfResponseTimeout=300', '--kml', '--print','--download'] - #asf_cmd = [item for item in asf_cmd if "SENTINEL" not in item] - asf_cmd = [item for item in asf_cmd if "maxResults" not in item] + asf_cmd_slc_download = ['asf_search_args.py', '--product=SLC'] + ssaraopt + ['--print', '--download'] + asf_cmd_burst_download = ['asf_search_args.py', '--product=BURST'] + ssaraopt + ['--print', '--download'] + asf_cmd_burst2safe = ['burst2stack','--rel-orbit',ssaraopt_dict['relativeOrbit'],'--start-date',ssaraopt_dict['start'],'--end-date',ssaraopt_dict['end'],'--extent',extent_str] + asf_cmd_burst2safe = ['burst2stack','--rel-orbit',ssaraopt_dict['relativeOrbit'],'--start-date',ssaraopt_dict['start'],'--end-date',ssaraopt_dict['end'],'--extent',extent_str] + asf_cmd_burst2safe.insert(0, "srun -n1 -N1 -A $JOBSHEDULER_PROJECTNAME -p $QUEUENAME -t 00:25:00 ") #FA 8/24: it should create a burst2safe.job with open('ssara_command.txt', 'w') as f: - f.write(' '.join(ssara_cmd) + '\n') -# with open('ssara_command_python.txt', 'w') as f: -# f.write(' '.join(ssara_cmd_python) + '\n') - with open('asf_command.txt', 'w') as f: - f.write(' '.join(asf_cmd) + '\n') + f.write(' '.join(ssara_cmd_slc_download_bash) + '\n') + + with open('asf_slc_download.txt', 'w') as f: + f.write(' '.join(asf_cmd_slc_download) + '\n') + + with open('asf_burst_download.txt', 'w') as f: + f.write(' '.join(ssara_cmd_kml_download_python) + '\n') + f.write(' '.join(asf_cmd_burst_download) + '\n') + f.write(' '.join(asf_cmd_burst2safe ) + '\n') + + ssara_cmd_python = ['ssara_federated_query.py'] + ssaraopt + ['--maxResults=20000','--asfResponseTimeout=300', '--kml', '--print'] + with open('ssara_command_python.txt', 'w') as f: + f.write(' '.join(ssara_cmd_python) + '\n') return diff --git a/minsar/minsarApp.bash b/minsar/minsarApp.bash index a226cb3e..a4e7dd56 100755 --- a/minsar/minsarApp.bash +++ b/minsar/minsarApp.bash @@ -42,12 +42,12 @@ function run_command() { function get_date_str() { # get string with start and end date if [ ! -z ${template[miaplpy.load.startDate]} ] && [ ! ${template[miaplpy.load.startDate]} == "auto" ]; then - start_date=${template[miaplpy.load.startDate]} + start_date=${template[miaplpy.load.startDate]} else start_date=$(ls merged/SLC | head -1) fi if [ ! -z ${template[miaplpy.load.endDate]} ] && [ ! ${template[miaplpy.load.endDate]} == "auto" ]; then - end_date=${template[miaplpy.load.endDate]} + end_date=${template[miaplpy.load.endDate]} else end_date=$(ls merged/SLC | tail -1) fi @@ -156,13 +156,14 @@ helptext=" --chunks process in form of multiple chunks. \n\ --tmp copy code and data to local /tmp [default]. \n\ --no-tmp no copying to local /tmp. \n\ + --debug \n\ Coding To Do: \n\ - clean up run_workflow (remove smallbaseline.job insarmaps.job) \n\ - move bash functions into minsarApp_functions.bash \n\ - change flags from 0/1 to False/True for reading from template file \n\ - create a command execution function (cmd_exec) \n\ - - create .minsarrc for defaults \n + - create .minsarrc for defaults \n " printf "$helptext" exit 0; @@ -205,8 +206,9 @@ jobfiles_flag=1 orbit_download_flag=1 select_reference_flag=1 new_reference_flag=0 +debug_flag=0 download_ECMWF_flag=1 -download_ECMWF_before_mintpy_flag=0 +download_ECMWgF_before_mintpy_flag=0 copy_to_tmp="--tmp" runfiles_dir="run_files_tmp" @@ -278,7 +280,7 @@ do orbit_download_flag=0 shift ;; - --sleep) + --sleep) sleep_time="$2" shift shift @@ -307,6 +309,14 @@ do chunks_flag=1 shift ;; + --debug) + debug_flag=1 + shift + ;; + --burst-download) + burst_download_flag=1 + shift + ;; *) POSITIONAL+=("$1") # save it in an array for later shift # past argument @@ -315,12 +325,15 @@ esac done set -- "${POSITIONAL[@]}" # restore positional parameters -# FA 5/2025 commented out as we now have --insarmaps option. It can be removed, I think. if [[ ${#POSITIONAL[@]} -gt 1 ]]; then - echo "Unknown parameters provided." + echo "Unknown parameters provided: ${POSITIONAL[-1]}" exit 1; fi +if [[ $debug_flag == "1" ]]; then + set -x +fi + # adjust switches according to template options if insarmaps_flag is not set # first test if insarmaps_flag is set if [[ ! -v insarmaps_flag ]]; then @@ -338,7 +351,7 @@ if [[ ! -v insarmaps_flag ]]; then fi fi -# adjust switches according to template options if upload_flag is not set +# adjust switches according to template options if upload_flag is not given on command line if [[ ! -v upload_flag ]]; then if [[ -n ${template[minsar.upload_flag]+_} ]]; then if [[ ${template[minsar.upload_flag]} == "True" ]]; then @@ -351,6 +364,19 @@ if [[ ! -v upload_flag ]]; then fi fi +# adjust switches according to template options if burst_download_flag is not given on command line +if [[ ! -v burst_download_flag ]]; then + if [[ -n ${template[minsar.burst_download_flag]+_} ]]; then + if [[ ${template[minsar.burst_download_flag]} == "True" ]]; then + burst_download_flag=1 + else + burst_download_flag=0 + fi + else + burst_download_flag=0 + fi +fi + ### always use --no-tmp on stampede3 if [[ $HOSTNAME == *"stampede3"* ]] && [[ $copy_to_tmp == "--tmp" ]]; then copy_to_tmp="--no-tmp" @@ -358,7 +384,7 @@ if [[ $HOSTNAME == *"stampede3"* ]] && [[ $copy_to_tmp == "--tmp" ]]; then configs_dir="configs" echo "Running on stampede3: switched from --tmp to --no-tmp because of too slow copying to /tmp" fi -miaplpy_tmp_flag=$copy_to_tmp +miaplpy_tmp_flag=$copy_to_tmp if [ ! -z ${sleep_time+x} ]; then echo "sleeping $sleep_time secs before starting ..." @@ -374,12 +400,12 @@ fi #if [[ -v mintpy_flag ]]; then lock_mintpy_flag=1; fi #mintpy_flag=1 -#if [[ -v miaplpy_flag ]]; then +#if [[ -v miaplpy_flag ]]; then # miaplpy_flag=1; # if [[ -v lock_mintpy_flag ]]; then -# mintpy_flag=1; +# mintpy_flag=1; # else -# mintpy_flag=0; +# mintpy_flag=0; # fi #else # miaplpy_flag=0; @@ -434,7 +460,7 @@ elif [[ $startstep == "finishup" ]]; then upload_flag=0 insarmaps_flag=0 elif [[ $startstep != "" ]]; then - echo "startstep received value of "${startstep}". Exiting." + echo "USER ERROR: startstep received value of "${startstep}". Exiting." exit 1 fi @@ -492,12 +518,12 @@ if [[ $copy_to_tmp == "--tmp" ]]; then else echo "copy_to_tmp is OFF" fi -echo "Switches: select_reference: $select_reference_flag download_ECMWF: $download_ECMWF_flag chunks: $chunks_flag" +echo "Switches: select_reference: $select_reference_flag burst_download: $burst_download_flag chunks: $chunks_flag" echo "Flags for processing steps:" echo "download dem jobfiles ifgram mintpy miaplpy upload insarmaps finishup" echo " $download_flag $dem_flag $jobfiles_flag $ifgram_flag $mintpy_flag $miaplpy_flag $upload_flag $insarmaps_flag $finishup_flag" -sleep 2 +sleep 4 ############################################################# # check weather python can load matplotlib.pyplot which occasionaly does not work for unknown reasons @@ -535,8 +561,17 @@ if [[ $download_flag == "1" ]]; then mkdir -p $download_dir cd $download_dir - cmd=$(cat ../ssara_command.txt) - run_command "$cmd" + echo "QQQQQQQ Download is starting...." + if [[ $burst_download_flag == "1" ]]; then + cmd=$(cat ../asf_burst_download.txt) + run_command "$cmd" + #while IFS= read -r cmd; do + # run_command "$cmd" + #done < ssara_command.txt + else + cmd=$(cat ../ssara_command.txt) + run_command "$cmd" + fi # FA 6/2024: The checking for HTTP Error 502 is now done in ssara_federated_query.bash which I think is better # FA 8/2024: I don't think we need the loop over several downloads anymore @@ -576,7 +611,7 @@ if [[ $download_flag == "1" ]]; then date_string=$(grep ^minsar.excludeDates $template_file | awk -F = '{printf "%s\n",$2}') date_array=($(echo $date_string | tr ',' "\n")) echo "${date_array[@]}" - + for date in "${date_array[@]}"; do echo "Remove $date if exist" files="RAW_data/*$date*" @@ -591,7 +626,7 @@ if [[ $dem_flag == "1" ]]; then # copy DEM if given demDir=$(grep -E "^stripmapStack.demDir|^topsStack.demDir" $template_file | awk -F = '{printf "%s\n",$2}' | sed 's/ //') rm -rf DEM; eval "cp -r $demDir DEM" - else + else # download DEM run_command "dem_rsmas.py $template_file --ssara_kml 2>out_dem_rsmas.e 1>out_dem_rsmas.o" fi @@ -638,7 +673,7 @@ if [[ $chunks_flag == "1" ]]; then # generate chunk template files run_command "generate_chunk_template_files.py $template_file $options 2>out_generate_chunk_template_files.e 1>out_generate_chunk_template_files.o" - + echo "Submitting chunk minsar jobs:" | tee -a log cat $WORK_DIR/minsar_commands.txt | tee -a log bash $WORK_DIR/minsar_commands.txt @@ -654,8 +689,8 @@ fi if [[ $jobfiles_flag == "1" ]]; then ############################################################# # download latest orbits from ASF mirror - - if [[ $orbit_download_flag == "1" && $template_file == *"Sen"* ]]; then + + if [[ $orbit_download_flag == "1" && $template_file == *"Sen"* ]]; then echo "Preparing to download latest poe and res orbits from ASF..." year=$(date +%Y) current_month=$(date +%Y%m) @@ -673,16 +708,16 @@ if [[ $jobfiles_flag == "1" ]]; then bash ASF_resorb_latest.txt cd - fi - + # clean directory for processing and create jobfiles pwd=`pwd`; echo "DIR: $pwd" run_command "clean_dir.bash $PWD --runfiles --ifgram --mintpy --miaplpy" run_command "create_runfiles.py $template_file --jobfiles --queue $QUEUENAME $copy_to_tmp 2>out_create_jobfiles.e 1>out_create_jobfiles.o" fi - + if [[ $ifgram_flag == "1" ]]; then - if [[ $template_file != *"Sen"* || $select_reference_flag == "0" ]]; then + if [[ $template_file != *"Sen"* || $select_reference_flag == "0" ]]; then run_command "run_workflow.bash $template_file --dostep ifgram $copy_to_tmp" else @@ -732,7 +767,7 @@ if [[ $ifgram_flag == "1" ]]; then # clean directory for processing and create jobfiles run_command "clean_dir.bash $PWD --runfiles --ifgram" run_command "create_runfiles.py $template_file --jobfiles --queue $QUEUENAME $copy_to_tmp 2>create_jobfiles.e 1>out_create_jobfiles.o" - + # rerun steps 1 to 5 with new reference echo "### Re-running step 1 to 5 with reference $new_reference_date" run_command "run_workflow.bash $template_file --start 1 --stop 5 $copy_to_tmp --append" @@ -743,11 +778,11 @@ if [[ $ifgram_flag == "1" ]]; then # continue running starting step 6 run_command "run_workflow.bash $template_file --start 6 --stop 11 $copy_to_tmp --append" - + fi # correct *xml and *vrt files - #sed -i "s|/tmp|$PWD|g" */*.xml */*/*.xml */*/*/*.xml - #sed -i "s|/tmp|$PWD|g" */*.vrt */*/*.vrt */*/*/*.vrt + #sed -i "s|/tmp|$PWD|g" */*.xml */*/*.xml */*/*/*.xml + #sed -i "s|/tmp|$PWD|g" */*.vrt */*/*.vrt */*/*/*.vrt sed -i "s|/tmp|$PWD|g" merged/geom_reference/*.vrt merged/SLC/*/*.vrt merged/interferograms/*/*vrt sed -i "s|/tmp|$PWD|g" merged/geom_reference/*.xml merged/SLC/*/*.xml merged/interferograms/*/*xml fi @@ -757,10 +792,10 @@ fi ######################## if [[ $mintpy_flag == "1" ]]; then - # run MintPy + # run MintPy run_command "run_workflow.bash $template_file --append --dostep mintpy $copy_to_tmp" - # upload mintpy directory + # upload mintpy directory if [[ $upload_flag == "1" ]]; then run_command "upload_data_products.py mintpy ${template[minsar.upload_option]}" fi @@ -779,10 +814,10 @@ fi ######################## if [[ $miaplpy_flag == "1" ]]; then # correct *xml and *vrt files (if skipped in ifgram step because of unwrap problems) (skipping merged/interferograms because it takes long) - sed -i "s|/tmp|$PWD|g" merged/geom_reference/*.vrt merged/SLC/*/*.vrt - sed -i "s|/tmp|$PWD|g" merged/geom_reference/*.xml merged/SLC/*/*.xml + sed -i "s|/tmp|$PWD|g" merged/geom_reference/*.vrt merged/SLC/*/*.vrt + sed -i "s|/tmp|$PWD|g" merged/geom_reference/*.xml merged/SLC/*/*.xml - # unset $miaplpy_tmp_flag for --no-tmp as miaplpyApp.py does not understand --no-tmp option + # unset $miaplpy_tmp_flag for --no-tmp as miaplpyApp.py does not understand --no-tmp option if [[ $miaplpy_tmp_flag == "--no-tmp" ]]; then unset miaplpy_tmp_flag fi @@ -800,7 +835,7 @@ if [[ $miaplpy_flag == "1" ]]; then # create save_hdf5 jobfile run_command "create_save_hdf5_jobfile.py $template_file $network_dir --outdir $network_dir/run_files --outfile run_10_save_hdfeos5_radar_0 --queue $QUEUENAME --walltime 0:30" - # run save_hdfeos5_radar jobfile + # run save_hdfeos5_radar jobfile run_command "run_workflow.bash $template_file --dir $miaplpy_dir_name --start 10" # create index.html with all images @@ -828,48 +863,48 @@ if [[ $finishup_flag == "1" ]]; then fi run_command "summarize_job_run_times.py $template_file $copy_to_tmp $miaplpy_opt" - IFS="," - last_file=($(tail -1 $download_dir/ssara_listing.txt)) - last_date=${last_file[3]} - echo "Last file: $last_file" - echo "Last processed image date: $last_date" - unset IFS + # IFS="," + # last_file=($(tail -1 $download_dir/ssara_listing.txt)) + # last_date=${last_file[3]} + # echo "Last file: $last_file" + # echo "Last processed image date: $last_date" + # unset IFS fi echo -echo "network_dir: <$network_dir>" +if test -f mintpy/*he5; then + echo "hdfeos5 files produced:" + ls -sh mintpy/*he5 +fi +if test -f $network_dir/*he5; then + echo " hdf5files in network_dir: <$network_dir>" + ls -sh $network_dir/*he5 +fi + +# Summarize results echo -echo "hdfeos5 files produced:" -if test -f mintpy/*he5; then ls -sh mintpy/*he5; fi -if test -f $network_dir/*he5; then ls -sh $network_dir/*he5; fi +echo "Done: $minsarApp_command" echo -echo "Done: $minsarApp_command" + +echo +echo "Yup! That's all from minsarApp.bash." echo -# Summarize results -if [[ "$insarmaps_dataset" == "PS" || "$insarmaps_dataset" == "DS" || "$insarmaps_dataset" == "geo" ]]; then - num=1 +echo "Data products uploaded to:" +if [ -f "upload.log" ]; then + tail -n -1 upload.log fi + +lines=1 if [[ "$insarmaps_dataset" == "PSDS" ]]; then - num=2 + lines=2 fi if [[ "$insarmaps_dataset" == "all" ]]; then - num=3 + lines=4 fi -echo "Data products uploaded to:" -if [ -f "upload.log" ]; then - tail -n -1 upload.log -else - echo "upload.log does not exist." -fi +lines=$((lines * 2)) # multiply as long as we ingestinto two servers if [ -f "insarmaps.log" ]; then - tail -n $num insarmaps.log -else - echo "insarmaps.log does not exist." + tail -n $lines insarmaps.log fi -echo -echo "Yup! That's all from minsarApp.bash." -echo - diff --git a/minsar/objects/dataset_template.py b/minsar/objects/dataset_template.py index 41af34e4..2f2012e6 100755 --- a/minsar/objects/dataset_template.py +++ b/minsar/objects/dataset_template.py @@ -159,10 +159,15 @@ def generate_ssaraopt_string(self): parallel = 6 ssaraopt += ' --parallel={}'.format(parallel) - #this was in Josh's code (above yje ssaraopt assignment) but unclear what it does + #this was in Josh's code (above the ssaraopt assignment) but unclear what it does #parallel_download = self.options.get("ssaraopt.parallelDownload", '30') + ssaraopt_dict = {} + for item in ssaraopt.split(' '): + if item: # Skip empty strings + key, value = item.lstrip('-').split('=', 1) # Split at the first '=' and remove leading '--' + ssaraopt_dict[key] = value - return ssaraopt + return ssaraopt, ssaraopt_dict def correct_keyvalue_quotes(self, options_in): """ quote-independent reformatting of sentinel.subswath key-value: diff --git a/minsar/requirements.txt b/minsar/requirements.txt index 80eaebec..f04dae01 100644 --- a/minsar/requirements.txt +++ b/minsar/requirements.txt @@ -13,3 +13,4 @@ pulp # needed for ISCE but was missing pdf2image # for create_html.py netCDF4 # for nc4 files asf_search +burst2safe diff --git a/minsar/upload_data_products.py b/minsar/upload_data_products.py index e19396f7..e4dc7cd4 100755 --- a/minsar/upload_data_products.py +++ b/minsar/upload_data_products.py @@ -3,6 +3,7 @@ # Author: Falk Amelung ####################### + import os import subprocess import sys @@ -123,7 +124,7 @@ def main(iargs=None): '/'+ data_dir +'/geo/geo_*.dbf', '/'+ data_dir +'/geo/geo_*.prj', '/'+ data_dir +'/geo/geo_*.shp', - '/'+ data_dir +'/geo/geo_*.shx' + '/'+ data_dir +'/geo/geo_*.shx', ]) if inps.triplets_flag: @@ -162,6 +163,21 @@ def main(iargs=None): '/'+ network_dir +'/geo/geo_*.shp', '/'+ network_dir +'/geo/geo_*.shx' ]) + + timeseries_path = 'timeseries_demErr.h5' + if os.path.exists(network_dir + '/' + 'timeseries_ERA5_demErr.h5'): + timeseries_path = 'timeseries_ERA5_demErr.h5' + + # FA 8/24: This section for edgar to have the high-res files + if inps.geo_flag: + scp_list.extend([ + '/'+ data_dir +'../maskPS.h5', + '/'+ data_dir +'/inputs/geometryRadar.h5', + '/'+ data_dir +'/temporalCoherence_lowpass_gaussian.h5', + '/'+ data_dir +'/maskTempCoh__lowpass_gaussian.h5', + '/'+ data_dir + '/' + timeseries_path + ]) + if inps.triplets_flag: scp_list.extend([ '/'+ network_dir +'/numTriNonzeroIntAmbiguity.h5', diff --git a/minsar/utils/create_commands.bash b/minsar/utils/create_commands.bash new file mode 100755 index 00000000..adbb5809 --- /dev/null +++ b/minsar/utils/create_commands.bash @@ -0,0 +1,22 @@ +#!/bin/bash + +# Check if at least one argument is provided +if [ "$#" -lt 1 ]; then + echo "Usage: $0 ... " + exit 1 +fi + +# Create or clear the command.bash file +> command.bash + +# Loop through each argument +for template in "$@"; do + # Remove trailing slash if present + template=${template%/} + + # Write the commands to command.bash + echo "create_save_hdf5_jobfile.py \$TE/${template}.template ${template}/miaplpy_*/network_* --queue skx-dev --outdir ${template}" >> command.bash + echo "run_workflow.bash \$TE/${template}.template --jobfile save_hdfeos5_radar.job" >> command.bash +done + +echo "Commands written to command.bash" diff --git a/minsar/utils/create_commands2.bash b/minsar/utils/create_commands2.bash new file mode 100755 index 00000000..2f5fc1f1 --- /dev/null +++ b/minsar/utils/create_commands2.bash @@ -0,0 +1,26 @@ +#!/bin/bash + +# Check if at least one argument is provided +if [ "$#" -lt 1 ]; then + echo "Usage: $0 ... " + exit 1 +fi + +# Create or clear the command.bash file +> commands2.bash + +# Loop through each argument +for template in "$@"; do + # Remove trailing slash if present + template=${template%/} + + # Write the commands to command.bash + #echo "create_save_hdf5_jobfile.py \$TE/${template}.template ${template}/miaplpy_*/network_* --queue skx-dev --outdir ${template}" >> command2.bash + echo "create_insarmaps_jobfile.py ${template}/miaplpy_*/network_* --dataset filt*DS" --queue skx-dev >> commands2.bash + echo "mv insarmaps.job ${template}" >> commands2.bash + echo "cd ${template}" >> commands2.bash + echo "run_workflow.bash \$TE/${template}.template --jobfile insarmaps.job" >> commands2.bash + echo "cd .." >> commands2.bash +done + +echo "Commands written to commands2.bash" diff --git a/minsar/utils/update_minTempCoh.bash b/minsar/utils/update_minTempCoh.bash new file mode 100755 index 00000000..3c12ca8c --- /dev/null +++ b/minsar/utils/update_minTempCoh.bash @@ -0,0 +1,158 @@ +#!/bin/bash + +show_help() { + echo "Usage: update_minTempCoh.bash template_path [min_temp_coh] data_dir" + echo "" + echo "Options:" + echo " --help Show this help message and exit" + echo "" + echo "Examples:" + echo " update_minTempCoh.bash \$TE/unittestGalapagosSenDT128.template mintpy" + echo " update_minTempCoh.bash \$TE/unittestGalapagosSenDT128.template 0.9 mintpy" + echo " update_minTempCoh.bash \$TE/unittestGalapagosSenDT128.template miaplpy/network_single_reference" + echo " update_minTempCoh.bash \$TE/unittestGalapagosSenDT128.template 0.9 miaplpy/network_single_reference" +} +################################################################################### +function create_template_array() { +mapfile -t array < <(grep -e ^minsar -e ^mintpy -e ^miaplpy $1) +declare -gA template +for item in "${array[@]}"; do + #echo "item: <$item>" + IFS='=' ; read -a arr1 <<< "$item" + item="${arr1[1]}" + IFS='#' ; read -a arr2 <<< "$item" + key="${arr1[0]}" + key=$(echo $key | tr -d ' ') + value="${arr2[0]}" + shopt -s extglob + value="${value##*( )}" # Trim leading whitespaces + value="${value%%*( )}" # Trim trailing whitespaces + shopt -u extglob + #echo "key, value: <$key> <$value>" + if [ ! -z "$key" ]; then + template[$key]="$value" + fi +unset IFS +done +} + +get_project_dir() { + local template_path="$1" + echo "$(basename "$template_path" .template)" +} + +replace_min_temp_coh() { + local template_path="$1" + local min_temp_coh="$2" + + sed -i.bak -E "s/(mintpy\.networkInversion\.minTempCoh\s*=\s*)([0-9.]+|auto)/\1${min_temp_coh}/" "$template_path" + sed -i.bak -E "s/(miaplpy\.timeseries\.minTempCoh\s*=\s*)([0-9.]+|auto)/\1${min_temp_coh}/" "$template_path" +} + +display_job_status() { + local job_ids=("$@") + local job_ids_str=$(IFS=,; echo "${job_ids[*]}") + while true; do + echo "Job status at $(date):" + squeue -j "$job_ids_str" + sleep 10 + done +} + +run_update() { + local template_path="$1" + local min_temp_coh="$2" + local data_dir="$3" + local project_dir + + create_template_array $template_path + + project_dir=$(get_project_dir "$template_path") + cd "${SCRATCHDIR}/${project_dir}" || { echo "Failed to change directory to $project_dir"; exit 1; } + touch "${data_dir}/temporalCoherence.h5" + touch "${data_dir}/geo/geo_temporalCoherence.h5" + if [[ -n "$min_temp_coh" ]]; then + echo "Updating $project_name using minimum temporal coherence (mintpy.timeseries.minTempCoh) of ${min_temp_coh}" + replace_min_temp_coh "$template_path" "$min_temp_coh" + fi + + if [[ "$data_dir" == *"mintpy"* ]]; then + job1_output=$(sbatch -p $QUEUE_DEV smallbaseline_wrapper.job) + job1_id=$(echo "$job1_output" | awk 'END {print $NF}') + echo "First job submitted with ID: $job1_id" + + joblast_output=$(sbatch -p $QUEUE_DEV --dependency=afterok:$job1_id insarmaps.job) + joblast_id=$(echo "$joblast_output" | awk 'END {print $NF}') + echo "Second job submitted with ID: $joblast_id" + else + + # FA 8/24 This need to be in jobfile + cmd="generate_mask.py ${data_dir}/temporalCoherence.h5 -m ${template[mintpy.networkInversion.minTempCoh]} --nonzero -o ${data_dir}/maskTempCoh.h5" + eval "$cmd" + + sed -i "s|SBATCH -t .:..:..|SBATCH -t 1:59:00|g" ${data_dir}/run_files/run_09_mintpy_timeseries_correction_0.job + job1_output=$(sbatch -p $QUEUE_DEV ${data_dir}/run_files/run_09_mintpy_timeseries_correction_0.job) + job1_id=$(echo "$job1_output" | awk 'END {print $NF}') + echo "First job submitted with ID: $job1_id" + + job2_output=$(sbatch -p $QUEUE_DEV --dependency=afterok:$job1_id ${data_dir}/run_files/run_10_save_hdfeos5_radar_0.job) + job2_id=$(echo "$job2_output" | awk 'END {print $NF}') + echo "Second job submitted with ID: $job2_id" + + joblast_output=$(sbatch -p $QUEUE_DEV --dependency=afterok:$job2_id insarmaps.job) + joblast_id=$(echo "$joblast_output" | awk 'END {print $NF}') + echo "Third job submitted with ID: $joblast_id" + fi + + display_job_status $job1_id $job2_id $joblast_id & + + echo "Waiting for job $joblast_id to complete..." + while squeue -j $joblast_id | grep -q "$joblast_id"; do + sleep 10 + done + + # Kill the background job status display + kill %1 + tail -2 insarmaps.log +} + +# Initialize variables +template_path="" +min_temp_coh="" +data_dir="" + +# Parse command-line options +while [[ "$1" != "" ]]; do + case $1 in + --help ) + show_help + exit 0 + ;; + * ) + if [[ -z "$template_path" ]]; then + template_path=$1 + elif [[ -z "$min_temp_coh" && "$1" =~ ^[0-9]*\.?[0-9]+$ ]]; then + min_temp_coh=$1 + elif [[ -z "$data_dir" ]]; then + data_dir=$1 + fi + ;; + esac + shift +done + +# Check if template_path and data_dir are provided +if [[ -z "$template_path" ]]; then + echo "Error: template_path is required." + show_help + exit 1 +fi + +if [[ -z "$data_dir" ]]; then + echo "Error: data_dir is required." + show_help + exit 1 +fi + +# Main script execution +run_update "$template_path" "$min_temp_coh" "$data_dir" diff --git a/setup/environment.bash b/setup/environment.bash index 2bd3ed3c..b15eba87 100644 --- a/setup/environment.bash +++ b/setup/environment.bash @@ -10,6 +10,8 @@ echo "sourcing ${RSMASINSAR_HOME}/setup/environment.bash ..." [ -z $JOBSCHEDULER ] && export JOBSCHEDULER=SLURM [ -z $QUEUENAME ] && export QUEUENAME=normal +[ -f ~/accounts/remote_hosts.bash ] && source ~/accounts/remote_hosts.bash + # set customizable variables to defaults if not given [ -z ${WORKDIR} ] && export WORKDIR=~/insarlab [ -z ${USER_PREFERRED} ] && export USER_PREFERRED=$USER diff --git a/setup/platforms_defaults.bash b/setup/platforms_defaults.bash index 9ef98070..5b700afe 100755 --- a/setup/platforms_defaults.bash +++ b/setup/platforms_defaults.bash @@ -8,7 +8,6 @@ export JOBSCHEDULER=NONE export QUEUENAME=NONE export WORKDIR=~/insarlab export SCRATCHDIR=${WORKDIR}/scratch -export REMOTE_SERVER=centos@129.114.104.223 ############################################# ########### known platforms ##################