fix: (7369f3b) in pf_dev, allow projector_focus of different dimensio…

…ns than volume_xyz

tal/reconstruct/__init__.py

@@ -91,7 +91,7 @@ def compensate_i_j(nlx, nly):
         if data.is_laser_paired_to_sensor():
             compensate_i_j(*data.sensor_grid_xyz.shape[:2])
         else:
-            assert len(data.laser_grid_xyz) == 3
+            assert data.laser_grid_xyz.size == 3
             compensate(data.H,
                        data.laser_grid_xyz.reshape(3),
                        data.laser_grid_normals.reshape(3))
@@ -101,7 +101,7 @@ def compensate_i_j(nlx, nly):
         if data.is_laser_paired_to_sensor():
             compensate_i(data.sensor_grid_xyz.shape[0])
         else:
-            assert len(data.laser_grid_xyz) == 3
+            assert data.laser_grid_xyz.size == 3
             compensate(data.H,
                        data.laser_grid_xyz.reshape(3),
                        data.laser_grid_normals.reshape(3))

tal/reconstruct/bp/backprojection.py

@@ -43,7 +43,7 @@ def backproject(H_0, laser_grid_xyz, sensor_grid_xyz, volume_xyz, volume_xyz_sha
 
     if camera_system.implements_projector():
         assert projector_focus is not None, 'projector_focus is required for this camera system'
-        assert len(projector_focus) == 3, \
+        assert projector_focus.size == 3, \
             'When using tal.reconstruct.bp, projector_focus must be a single 3D point. ' \
             'If you want to focus the illumination aperture at multiple points, ' \
             'please use tal.reconstruct.pf_dev instead or call tal.reconstruct.bp once per projector_focus.'

tal/reconstruct/pf_dev/__init__.py

@@ -87,6 +87,8 @@ def solve(data: NLOSCaptureData,
         You can generate these depth-slices with tal.reconstruct.get_volumeXXX functions.
     """
     from tal.reconstruct.util import convert_to_N_3, convert_reconstruction_from_N_3
+    if projector_focus is not None:
+        projector_focus = np.array(projector_focus)
     H, laser_grid_xyz, sensor_grid_xyz, volume_xyz_n3, \
         optimize_projector_convolutions, optimize_camera_convolutions = \
         convert_to_N_3(data, volume_xyz, volume_format,
@@ -105,9 +107,6 @@ def solve(data: NLOSCaptureData,
         compensate_invsq=compensate_invsq,
         progress=progress)
 
-    mutliple_projector_points = \
-        camera_system.implements_projector() \
-        and projector_focus is not None and len(projector_focus) > 3
-
-    return convert_reconstruction_from_N_3(data, reconstructed_volume_n3, volume_xyz, volume_format, camera_system,
-                                           is_exhaustive_reconstruction=mutliple_projector_points)
+    return convert_reconstruction_from_N_3(data, reconstructed_volume_n3,
+                                           volume_xyz, volume_format,
+                                           camera_system, projector_focus)

tal/reconstruct/pf_dev/phasor_fields.py

@@ -27,24 +27,36 @@ def backproject_pf_multi_frequency(
 
     nt, nl, ns = H_0.shape
     nv = np.prod(volume_xyz.shape[:-1])  # N or X * Y or X * Y * Z
-    if optimize_projector_convolutions or optimize_camera_convolutions:
-        if volume_xyz.ndim == 3:
-            nvx, nvy, _ = volume_xyz.shape
-            volume_xyz = volume_xyz.reshape((nvx, nvy, 1, 3))
-        assert volume_xyz.ndim == 4, 'Expecting X_Y_Z_3 format'
-        nvx, nvy, nvz, _ = volume_xyz.shape
+    if projector_focus is None:
+        npf = 0
+    elif projector_focus.size == 3:
+        npf = 1
     else:
-        nvz = 1
+        npf = np.prod(projector_focus.shape[:-1])
     if optimize_projector_convolutions:
         assert laser_grid_xyz.ndim == 3, 'Expecting X_Y_3 format'
         nlx, nly, _ = laser_grid_xyz.shape
+
+        if projector_focus.ndim == 3:
+            npfx, npfy, _ = projector_focus.shape
+            projector_focus = projector_focus.reshape((npfx, npfy, 1, 3))
+        assert projector_focus.ndim == 4, 'Expecting X_Y_Z_3 format'
+        npfx, npfy, npfz, _ = projector_focus.shape
     else:
         nlx, nly = 1, 1
+        npfz = 1
     if optimize_camera_convolutions:
         assert sensor_grid_xyz.ndim == 3, 'Expecting X_Y_3 format'
         nsx, nsy, _ = sensor_grid_xyz.shape
+
+        if volume_xyz.ndim == 3:
+            nvx, nvy, _ = volume_xyz.shape
+            volume_xyz = volume_xyz.reshape((nvx, nvy, 1, 3))
+        assert volume_xyz.ndim == 4, 'Expecting X_Y_Z_3 format'
+        nvx, nvy, nvz, _ = volume_xyz.shape
     else:
         nsx, nsy = 1, 1
+        nvz = 1
 
     """ Phasor fields filter """
 
@@ -89,7 +101,7 @@ def backproject_pf_multi_frequency(
     if camera_system.implements_projector():
         assert projector_focus is not None, \
             'projector_focus is required for this camera system'
-        if len(projector_focus) == 3:
+        if projector_focus.size == 3:
             projector_focus = np.array(projector_focus).reshape(
                 (1, 1, 1, 3))
             projector_focus_mode = 'single'
@@ -98,11 +110,11 @@ def backproject_pf_multi_frequency(
                     'Falling back to default method.')
             optimize_projector_convolutions = False
         else:
-            assert nvz == 1, \
-                'When projector_focus=volume_xyz, the volume must be a single Z slice. ' \
+            assert npfz == 1, \
+                'When projector_focus is a volume, it must be a single Z slice. ' \
                 f'In your case, your volume has {nvz} Z slices. ' \
                 'You can call this same function for each individual point in the volume.'
-            projector_focus = projector_focus.reshape((1, nv, 1, 3))
+            projector_focus = projector_focus.reshape((1, npf, 1, 3))
             projector_focus_mode = 'exhaustive'
     else:
         assert projector_focus is None, \
@@ -151,7 +163,7 @@ def invsq(d):
         d_4 = np.float32(0.0)
 
     if projector_focus_mode == 'exhaustive':
-        n_projector_points = nv
+        n_projector_points = npf
     else:
         n_projector_points = 1
 
@@ -187,9 +199,8 @@ def invsq(d):
             dtype=np.complex64)
 
     for i_z, nvzi in range_z:
-        if optimize_projector_convolutions or nl == 1:
-            projector_focus_i = projector_focus.reshape(
-                (nvx, nvy, nvz, 3))[..., nvzi, :]
+        if optimize_projector_convolutions:
+            projector_focus_i = projector_focus.reshape((npfx, npfy, 3))
         else:
             projector_focus_i = projector_focus
         if optimize_camera_convolutions:
@@ -201,8 +212,8 @@ def invsq(d):
         if camera_system.bp_accounts_for_d_2():
             if optimize_projector_convolutions:
                 assert projector_focus_mode in ['exhaustive', 'confocal']
-                rlx = nvx + nlx - 1
-                rly = nvy + nly - 1
+                rlx = npfx + nlx - 1
+                rly = npfy + nly - 1
                 laser_grid_xyz = laser_grid_xyz.reshape((nlx, nly, 3))
                 l_dx = laser_grid_xyz[1, 0, ...] - \
                     laser_grid_xyz[0, 0, ...]
@@ -300,15 +311,15 @@ def work_zslice_freq(range_w):
                 rsd_3 = np.exp(np.complex64(2j * np.pi) * d_3 * frequency)
                 rsd_3 *= invsq_3
                 if optimize_camera_convolutions:
-                    H_0_w = H_0_w.reshape((nlx, nly, nsx, nsy))
-                    rsd_3 = rsd_3.reshape((1, 1, rsx, rsy))
+                    H_0_w = H_0_w.reshape((nl, nsx, nsy))
+                    rsd_3 = rsd_3.reshape((1, rsx, rsy))
                     H_0_w_fft = np.fft.fft2(
-                        H_0_w, axes=(2, 3), s=(rsx, rsy))
+                        H_0_w, axes=(1, 2), s=(rsx, rsy))
                     rsd_3_fft = np.fft.fft2(
-                        rsd_3, axes=(2, 3), s=(rsx, rsy))
+                        rsd_3, axes=(1, 2), s=(rsx, rsy))
                     H_0_w_fft *= rsd_3_fft
-                    H_0_w = np.fft.ifft2(H_0_w_fft, axes=(2, 3))
-                    H_0_w = H_0_w[:, :, :nvx, :nvy]
+                    H_0_w = np.fft.ifft2(H_0_w_fft, axes=(1, 2))
+                    H_0_w = H_0_w[:, :nvx, :nvy]
                     H_0_w = H_0_w.reshape((nl, nvi))
                 else:
                     H_0_w = H_0_w.reshape((nl, 1, ns))
@@ -323,15 +334,15 @@ def work_zslice_freq(range_w):
                         assert optimize_projector_convolutions, \
                             'You must use the convolutions optimization when projector_focus=volume_xyz. ' \
                             'Check the documentation for tal.reconstruct.pf_dev for more information.'
-                        H_0_w = H_0_w.reshape((nlx, nly, nvx, nvy))
+                        H_0_w = H_0_w.reshape((nlx, nly, nsx, nsy))
                         rsd_2 = rsd_2.reshape((rlx, rly, 1, 1))
                         H_0_w_fft = np.fft.fft2(
                             H_0_w, axes=(0, 1), s=(rlx, rly))
                         rsd_2_fft = np.fft.fft2(
                             rsd_2, axes=(0, 1), s=(rlx, rly))
                         H_0_w_fft *= rsd_2_fft
                         H_0_w = np.fft.ifft2(H_0_w_fft, axes=(0, 1))
-                        H_0_w = H_0_w[:nvx, :nvy, :, :]
+                        H_0_w = H_0_w[:npfx, :npfy, :, :]
                     else:
                         assert projector_focus_mode in ['single', 'confocal']
                         H_0_w = H_0_w.reshape((nl, nvi))

tal/reconstruct/upf_dev/__init__.py

@@ -69,6 +69,8 @@ def solve(data: NLOSCaptureData,
         this optimization, tal.reconstruct.pf_dev will fall back to the default implementation.
         You can generate these depth-slices with tal.reconstruct.get_volumeXXX functions.
     """
+    raise NotImplementedError('This version of upf_dev is deprecated. '
+                              'If you want to use it, use a previous version of TAL or contact the authors.')
     from tal.reconstruct.util import convert_to_N_3, convert_reconstruction_from_N_3
     H, laser_grid_xyz, sensor_grid_xyz, volume_xyz_n3, \
         optimize_projector_convolutions, optimize_camera_convolutions = \
@@ -88,7 +90,7 @@ def solve(data: NLOSCaptureData,
 
     mutliple_projector_points = \
         camera_system.implements_projector() \
-        and projector_focus is not None and len(projector_focus) > 3
+        and projector_focus is not None and projector_focus.size > 3
 
     return convert_reconstruction_from_N_3(data, reconstructed_volume_n3, volume_xyz, volume_format, camera_system,
                                            is_exhaustive_reconstruction=mutliple_projector_points)

tal/reconstruct/upf_dev/phasor_fields.py

@@ -63,7 +63,7 @@ def backproject_pf_multi_frequency(
     if camera_system.implements_projector():
         assert projector_focus is not None, \
             'projector_focus is required for this camera system'
-        if len(projector_focus) == 3:
+        if projector_focus.size == 3:
             projector_focus = np.array(projector_focus).reshape(
                 (1, 1, 1, 3))
             projector_focus_mode = 'single'

tal/reconstruct/util.py

@@ -44,22 +44,24 @@ def convert_to_N_3(data: NLOSCaptureData,
         - The laser_grid_xyz's slices are parallel to the projector_focus' slices
         - The sensor_grid_xyz's slices are parallel to the volume_xyz's slices
         - The points in the {laser|sensor}_grid_xyz's slices are sampled at the same rate
+            as {projector_focus|volume_xyz}'s points
     """
 
     volume_format = _infer_volume_format(
         volume_xyz, volume_format, do_log=True)
     try:
         assert projector_focus is not None
+        projector_focus = np.array(projector_focus)
         projector_focus_format = _infer_volume_format(
             projector_focus, VolumeFormat.UNKNOWN, do_log=False)
-    except AttributeError:
+    except AssertionError:
         projector_focus_format = VolumeFormat.UNKNOWN
 
     # this variable is set to false during the conversion
     optimize_projector_convolutions = try_optimize_convolutions and \
-        volume_format in [VolumeFormat.X_Y_3, VolumeFormat.X_Y_Z_3]
-    optimize_camera_convolutions = optimize_projector_convolutions and \
         projector_focus_format in [VolumeFormat.X_Y_3, VolumeFormat.X_Y_Z_3]
+    optimize_camera_convolutions = try_optimize_convolutions and \
+        volume_format in [VolumeFormat.X_Y_3, VolumeFormat.X_Y_Z_3]
 
     is_laser_paired_to_sensor = data.is_laser_paired_to_sensor()
 
@@ -134,7 +136,7 @@ def convert_to_N_3(data: NLOSCaptureData,
         assert projector_focus is not None
         assert projector_focus_format in [
             VolumeFormat.X_Y_3, VolumeFormat.X_Y_Z_3]
-        npx, npy = projector_focus_format.shape[:2]
+        npx, npy = projector_focus.shape[:2]
         assert npx > 1 and npy > 1
 
         # list of (Z, 3) normals of all Z positions in the plane
@@ -153,7 +155,7 @@ def convert_to_N_3(data: NLOSCaptureData,
         p_dy = np.linalg.norm(
             projector_focus[0, 1, z_index, :] - projector_focus[0, 0, z_index, :])
     except AssertionError:
-        optimize_camera_convolutions = False
+        optimize_projector_convolutions = False
 
     assert volume_format.xyz_dim_is_last(), 'Unexpected volume_format'
     try:
@@ -241,7 +243,8 @@ def convert_reconstruction_from_N_3(data: NLOSCaptureData,
                                     volume_xyz: NLOSCaptureData.VolumeXYZType,
                                     volume_format: VolumeFormat,
                                     camera_system: CameraSystem,
-                                    is_exhaustive_reconstruction: bool = False):
+                                    projector_focus:  Union[NLOSCaptureData.Array3,
+                                                            NLOSCaptureData.VolumeXYZType] = None):
 
     volume_format = _infer_volume_format(
         volume_xyz, volume_format, do_log=False)
@@ -253,9 +256,14 @@ def convert_reconstruction_from_N_3(data: NLOSCaptureData,
 
     assert volume_format.xyz_dim_is_last(), 'Unexpected volume_format'
 
+    is_exhaustive_reconstruction = \
+        camera_system.implements_projector() \
+        and projector_focus is not None and projector_focus.size > 3
+
     if is_exhaustive_reconstruction:
         # add an additional shape dimension for the exhaustive reconstruction
-        shape += volume_xyz.shape[:-1]
+        shape += projector_focus.shape[:-1]
+
     shape += volume_xyz.shape[:-1]
 
     return reconstructed_volume_n3.reshape(shape)

tal/render/render.py

@@ -133,8 +133,8 @@ def render_nlos_scene(config_path, args, num_retries=0):
         def get_grid_xyz(nx, ny, rw_scale_x, rw_scale_y, ax0=0, ax1=1, ay0=0, ay1=1):
             px0 = -rw_scale_x + 2 * rw_scale_x * ax0
             px1 = rw_scale_x - 2 * rw_scale_x * (1 - ax1)
-            py0 = rw_scale_y - 2 * rw_scale_y * ay0
-            py1 = -rw_scale_y + 2 * rw_scale_y * (1 - ay1)
+            py0 = -rw_scale_y + 2 * rw_scale_y * ay0
+            py1 = rw_scale_y - 2 * rw_scale_y * (1 - ay1)
             xg = np.stack(
                 (np.linspace(px0, px1, num=2*nx + 1)[1::2],)*ny, axis=1)
             yg = np.stack(
@@ -203,8 +203,8 @@ def expand(vec, x, y):
         else:
             laser_grid_xyz = get_grid_xyz(
                 laser_width, laser_height, relay_wall['scale_x'], relay_wall['scale_y'],
-                ax0=laser_aperture_start_x, ax1=laser_aperture_start_x,
-                ay0=laser_aperture_start_y, ay1=laser_aperture_start_y)
+                ax0=laser_aperture_start_x, ax1=laser_aperture_end_x,
+                ay0=laser_aperture_start_y, ay1=laser_aperture_end_y)
         laser_grid_xyz += displacement
         # TODO(diego): rotate [0, 0, 1] by rot_degrees_x (assmes RW is a plane)
         # or use a more generalist approach

tal/render/scene_defaults.yaml

@@ -56,7 +56,7 @@ geometry:
     filename: ./Z.obj
   displacement_x: 0.0
   displacement_y: 0.0
-  displacement_z: 1.0  # using default RW settings this corresponds to depth
+  displacement_z: 0.0  # using default RW settings this corresponds to depth
   rot_degrees_x: 0.0
   rot_degrees_y: 0.0
   rot_degrees_z: 0.0