Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

initial burst metadata reader #1

Merged
merged 25 commits into from
Dec 16, 2021
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
6267ec8
initial commit
LiangJYu Nov 30, 2021
3ca226a
added center burst to burst class, swapped center lat/lon to lon/lat,…
LiangJYu Dec 1, 2021
37b3fc6
setup fix
LiangJYu Dec 1, 2021
defbf54
add burst border
LiangJYu Dec 6, 2021
9f4325c
updates for clarity in variable names and comments
LiangJYu Dec 8, 2021
cefe0a4
bump python minor version for dataclass
LiangJYu Dec 8, 2021
e1c9052
more comment/documentation updates
LiangJYu Dec 9, 2021
bae201b
save burst border as shapely polygon
LiangJYu Dec 9, 2021
c95b03b
added HV to polarization options
LiangJYu Dec 9, 2021
656f47a
burst center and boundary as shapely objects and check/split boundari…
LiangJYu Dec 9, 2021
f02f602
orbit changes
LiangJYu Dec 10, 2021
6571089
number of centers match number of polys and updated documentation
LiangJYu Dec 10, 2021
36a9faf
new centroid calculation and prepend burst number with b
LiangJYu Dec 10, 2021
074190c
add ignore file, moved source files, updated setup accordingly
LiangJYu Dec 10, 2021
90362f2
add meaningful content to readme
LiangJYu Dec 11, 2021
0e59280
replace orbit ET.Tree with orbit file
LiangJYu Dec 13, 2021
93446d8
more sensible method names
LiangJYu Dec 13, 2021
7de0f9a
stringent file name format enforcement
LiangJYu Dec 13, 2021
c6cf242
documentation fixes and orbit state vector parse cleanup
LiangJYu Dec 13, 2021
20bc7ad
fix VRT length/width off by one and removed unused code
LiangJYu Dec 13, 2021
9aed622
remove unused imports and variables
LiangJYu Dec 14, 2021
acaa997
codacy compliance formatting
LiangJYu Dec 15, 2021
fe4df29
codacy formatting
yunjunz Dec 15, 2021
42426e4
Update README.md
yunjunz Dec 15, 2021
277c29b
addresaddress PR comments
LiangJYu Dec 15, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions bin/s1_read.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
import sys

from sentinel1_reader import sentinel1_reader

if __name__ == "__main__":
# TODO replace with argparse
zip_path = sys.argv[1]
i_subswath = int(sys.argv[2])
if i_subswath < 1 or i_subswath > 3:
raise ValueError("i_subswath not <1 or >3")
pol = sys.argv[3]
if pol not in ['vv', 'vh']:
yunjunz marked this conversation as resolved.
Show resolved Hide resolved
raise ValueError("polarization not 'vv' or 'vh'")
bursts = sentinel1_reader.zip2bursts(zip_path, i_subswath, pol)
for i, burst in enumerate(bursts):
print(burst.id_str, burst.center)
5 changes: 5 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[build-system]
requires = [
"setuptools>=42"
]
build-backend = "setuptools.build_meta"
Empty file added sentinel1_reader/__init__.py
Empty file.
190 changes: 190 additions & 0 deletions sentinel1_reader/sentinel1_burst_slc.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
from dataclasses import dataclass
import datetime
import xml.etree.ElementTree as ET
import os

from osgeo import gdal

import isce3

@dataclass(frozen=True)
class Doppler:
poly1d: isce3.core.Poly1d
lut2d: isce3.core.LUT2d

@dataclass(frozen=True)
class Sentinel1BurstSlc:
'''
Raw values extracted from SAFE XML.
'''
sensing_start: datetime.datetime# *
radar_frequency: float
yunjunz marked this conversation as resolved.
Show resolved Hide resolved
wavelength: float
azimuth_steer_rate: float
azimuth_time_interval: float
slant_range_time: float
starting_range: float
range_sampling_rate: float
range_pxl_spacing: float
yunjunz marked this conversation as resolved.
Show resolved Hide resolved
shape: tuple()
azimuth_fm_rate: isce3.core.Poly1d
doppler: Doppler
range_processing_bandwidth: float
yunjunz marked this conversation as resolved.
Show resolved Hide resolved
pol: str # {VV, VH}
yunjunz marked this conversation as resolved.
Show resolved Hide resolved
id_str: str # t{track_number}_iw{1,2,3}_{burst_index}
yunjunz marked this conversation as resolved.
Show resolved Hide resolved
platform_id: str # S1{A,B}
center: tuple # {center lon, center lat} in degrees
border: list
yunjunz marked this conversation as resolved.
Show resolved Hide resolved
# VRT params
# TODO maybe make these own dataclass?
yunjunz marked this conversation as resolved.
Show resolved Hide resolved
tiff_path: str
i_burst: int
first_valid_sample: int
last_valid_sample: int
first_valid_line: int
last_valid_line: int

def as_isce3_radargrid(self):
'''
Init and return isce3.product.RadarGridParameters.

Returns:
--------
_ : RadarGridParameters
RadarGridParameters constructed from class members.
'''

prf = 1 / self.azimuth_time_interval

length, width = self.shape

time_delta = datetime.timedelta(days=2)
ref_epoch = isce3.core.DateTime(self.sensing_start - time_delta)
# sensing start with respect to reference epoch
sensing_start = time_delta.total_seconds()

# init radar grid
return isce3.product.RadarGridParameters(sensing_start,
self.wavelength,
prf,
self.starting_range,
self.range_pxl_spacing,
isce3.core.LookSide.Right,
length,
width,
ref_epoch)

def to_vrt_file(self, out_path):
'''
Write burst to VRT file.

Parameters:
-----------
out_path : string
Path of output VRT file.
'''
line_offset = self.i_burst * self.shape[0]

inwidth = self.last_valid_sample - self.first_valid_sample
inlength = self.last_valid_line - self.first_valid_line
outlength, outwidth = self.shape
yoffset = line_offset + self.first_valid_line
localyoffset = self.first_valid_line
xoffset = self.first_valid_sample
gdal_obj = gdal.Open(self.tiff_path, gdal.GA_ReadOnly)
fullwidth = gdal_obj.RasterXSize
fulllength = gdal_obj.RasterYSize

# TODO maybe cleaner to write with ElementTree
tmpl = f'''<VRTDataset rasterXSize="{outwidth}" rasterYSize="{outlength}">
<VRTRasterBand dataType="CFloat32" band="1">
<NoDataValue>0.0</NoDataValue>
<SimpleSource>
<SourceFilename relativeToVRT="1">{self.tiff_path}</SourceFilename>
<SourceBand>1</SourceBand>
<SourceProperties RasterXSize="{fullwidth}" RasterYSize="{fulllength}" DataType="CInt16"/>
<SrcRect xOff="{xoffset}" yOff="{yoffset}" xSize="{inwidth}" ySize="{inlength}"/>
<DstRect xOff="{xoffset}" yOff="{localyoffset}" xSize="{inwidth}" ySize="{inlength}"/>
</SimpleSource>
</VRTRasterBand>
</VRTDataset>'''

with open(out_path, 'w') as fid:
fid.write(tmpl)

def get_sensing_mid(self):
'''
Returns sensing mid as datetime object.

Returns:
--------
_ : datetime
Sensing mid as datetime object.
'''
d_seconds = 0.5 * (self.shape[0] - 1) * self.azimuth_time_interval
return self.sensing_start + datetime.timedelta(seconds=d_seconds)

def get_isce3_orbit(self, orbit_dir: str):
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it makes sense to load orbit while the XML is read and to make it a member of Sentinel1BurstSlc. It'll be highly unlikely that a swath will span multiple orbit files, so repeated calls current function is doing a lot repeated/redundant file reading.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree that orbit may be a member of sentinel-1 burst SLC. But still you need a method to return the orbit as we will need orbit for workflows as you know.
The extra call to get_orbit did not bother me. actually it's more clear what is going on.

'''
Init and return ISCE3 orbit.

Parameters:
-----------
orbit_dir : string
Path to directory containing orbit files.

Returns:
--------
_ : datetime
Sensing mid as datetime object.
'''
if not os.path.isdir(orbit_dir):
raise NotADirectoryError

# determine start and end time from metadata
pulse_length = (self.shape[0] - 1) * self.azimuth_time_interval
t_pulse_end = self.sensing_start + datetime.timedelta(seconds=pulse_length)

# find files with self.platform_id
item_valid = lambda item, sat_id: os.path.isfile(item) and sat_id in item
orbit_files = [item for item in os.listdir(orbit_dir)
if item_valid(f'{orbit_dir}/{item}', self.platform_id)]
if not orbit_files:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just for my curiosity. At the moment the reader is assuming to find the orbit file allocated somewhere in a directory. Would it make sense to introduce functionality to download the orbits in case these files are missing? We do have prototypes for that so it should not be too much work.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point @vbrancat I was actually thinking the same. But then I was hesitant if we want to have the reader to be isolated as a pure reader and let workflows at higher level be worried about downloading orbits and all the junk around (connection failure etc). I don't have a strong opinion really but I just wanted to make sure you also note that point.
Anyways I think the reader should accept external orbit and that should be the default behavior.
A middle ground would be to have a flag for orbit_download which is off by default. So if the orbit is not found and if the user specifically asks to download, then the reader attempts to download.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree. The higher-level workflow should properly handle this (e.g. check the internet connection and all this stuff). However, I do not see any harm in providing an optional functionality to download the orbit (if not present). Maybe the functionality (it can be a simple flag) should not have too much logic in it (e.g. not checking the internet connection). I will let @LiangJYu illuminate us if this solution does not violate the "separation of functionalities" principle (https://en.wikipedia.org/wiki/Single-responsibility_principle)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds good to me as far as it is optional to use.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Especially when running in production, I also agree the higher-level workflow should download the orbits. For other use cases, I'm not opposed to including a download option. Perhaps something like the following?

def get_local_orbit_file(t_start, t_stop, orbit_dir):
    # try to find and return path file in orbit_dir else return empty string

def get_online_orbit_file(t_start, t_stop):
    # try to find and return file from online source else return empty string

def get_isce3_orbit(..., online_ok=False):
    ...
    # try local first
    orbit_file = get_local_orbit_file(...)

    # try online if nothing local and ok to go online
    if not orbit_file and online_ok:
        orbit_file = get_online_orbit_file(...)

        # check if nothing found online
        if not orbit_file:
            raise ValueError("no orbit file found")
    ...

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This looks good to me.

err_str = f"No orbit files found for {self.platform_id} in f{orbit_dir}"
raise RuntimeError(err_str)

fmt = "%Y%m%dT%H%M%S"
# parse start and end time of files
for orbit_file in orbit_files:
_, tail = os.path.split(orbit_file)
t_orbit_start, t_orbit_end = tail.split('_')[-2:]
t_orbit_start = datetime.datetime.strptime(t_orbit_start[1:], fmt)
t_orbit_end = datetime.datetime.strptime(t_orbit_end[:-4], fmt)
if t_orbit_start < self.sensing_start and t_orbit_end > t_pulse_end:
break

# find 'Data_Block/List_of_OSVs'
tree = ET.parse(f'{orbit_dir}/{orbit_file}')
osv_list = tree.find('Data_Block/List_of_OSVs')
# TODO turn into generator?
# loop thru elements
# while OSV/UTC < burst_end
# UTC, pos, vel to list of isce3.core.stateVectors
fmt = "UTC=%Y-%m-%dT%H:%M:%S.%f"
orbit_sv = []
# add start & end padding to ensure sufficient number of orbit points
pad = datetime.timedelta(seconds=60)
for osv in osv_list:
t_orbit = datetime.datetime.strptime(osv[1].text, fmt)
pos = [float(osv[i].text) for i in range(4,7)]
vel = [float(osv[i].text) for i in range(7,10)]
if t_orbit > self.sensing_start - pad:
orbit_sv.append(isce3.core.StateVector(isce3.core.DateTime(t_orbit),
pos, vel))
if t_orbit > t_pulse_end + pad:
break

# use list of stateVectors to init and return isce3.core.Orbit
time_delta = datetime.timedelta(days=2)
ref_epoch = isce3.core.DateTime(self.sensing_start - time_delta)
return isce3.core.Orbit(orbit_sv, ref_epoch)
Loading