Problems encountered when implementing official tutorial documents for module DeLF

[VergilWang15]

TF hub DeLF
It seems to be no available output from the module.
When I try to print delf_outputs

{'scales': <tf.Tensor 'module_apply_default/NonMaxSuppression/Gather/GatherV2_3:0' shape=(None,) dtype=float32>,
 'scores': <tf.Tensor 'module_apply_default/ExpandDims:0' shape=(None, 1) dtype=float32>,
 'descriptors': <tf.Tensor 'module_apply_default/postprocess_1/pca_l2_normalization:0' shape=(None, 40) dtype=float32>, 'locations': <tf.Tensor 'module_apply_default/truediv_2:0' shape=(None, 2) dtype=float32>, 
'attention': <tf.Tensor 'module_apply_default/Reshape_4:0' shape=(None,) dtype=float32>, 
'boxes': <tf.Tensor 'module_apply_default/NonMaxSuppression/Gather/GatherV2:0' shape=(None, 4) dtype=float32>,
 'features': <tf.Tensor 'module_apply_default/NonMaxSuppression/Gather/GatherV2_1:0' shape=(None, 1024) dtype=float32>}

It seems that every label's shape is None.
Thus, in the part
Use the locations and description vectors to match the images

num_features_1 = result1['locations'].shape[0]
print("Loaded image 1's %d features" % num_features_1)

This code does not work cause shape[0] is none

I'm using python 3.6 with tensorflow 2.5.0
The complete code is at the end. I‘m new to the TF-hub. Any help is grateful.

from absl import logging

import matplotlib.pyplot as plt
import numpy as np
from PIL import Image, ImageOps
from scipy.spatial import cKDTree
from skimage.feature import plot_matches
from skimage.measure import ransac
from skimage.transform import AffineTransform
from six import BytesIO
# import tensorflow as tf
import tensorflow_hub as hub
from six.moves.urllib.request import urlopen
import tensorflow.compat.v1 as tf  # 兼容性设置
# To make tf 2.0 compatible with tf1.0 code, we disable the tf2.0 functionalities
from cv2 import cv2 as cv
import glob

tf.disable_eager_execution()  # 关闭急切执行
data_path = "../Data/*.jpg"  # 用于比对的指纹数据位置
delf_module = hub.Module("https://tfhub.dev/google/delf/1")


def run_delf(image):
    np_image = np.array(image)
    float_image = tf.image.convert_image_dtype(np_image, tf.float32)
    print("------------------\n")
    print(float_image)
    print("\n------------------\n")
    delf_inputs = {
        # An image tensor with dtype float32 and shape [height, width, 3], where
        # height and width are positive integers:
        'image': float_image,
        # Scaling factors for building the image pyramid as described in the paper:
        'image_scales': [0.25, 0.3536, 0.5, 0.7071, 1.0, 1.4142, 2.0],
        # Image features whose attention score exceeds this threshold will be
        # returned:
        'score_threshold': 100.0,
        # The maximum number of features that should be returned:
        'max_feature_num': 1000,
    }

    # Apply the DELF module to the inputs to get the outputs.
    delf_outputs = delf_module(delf_inputs, as_dict=True)
    return delf_outputs


def match_images(image1, image2, result1, result2):
    distance_threshold = 0.8

    # Read features.
    num_features_1 = result1['locations'].shape[0]
    print("Loaded image 1's %d features" % num_features_1)

    num_features_2 = result2['locations'].shape[0]
    print("Loaded image 2's %d features" % num_features_2)

    # Find nearest-neighbor matches using a KD tree.
    d1_tree = cKDTree(result1['descriptors'])
    _, indices = d1_tree.query(
        result2['descriptors'],
        distance_upper_bound=distance_threshold)

    # Select feature locations for putative matches.
    locations_2_to_use = np.array([
        result2['locations'][i,]
        for i in range(num_features_2)
        if indices[i] != num_features_1
    ])
    locations_1_to_use = np.array([
        result1['locations'][indices[i],]
        for i in range(num_features_2)
        if indices[i] != num_features_1
    ])

    # Perform geometric verification using RANSAC.
    _, inliers = ransac(
        (locations_1_to_use, locations_2_to_use),
        AffineTransform,
        min_samples=3,
        residual_threshold=20,
        max_trials=1000)

    print('Found %d inliers' % sum(inliers))

    # Visualize correspondences.
    _, ax = plt.subplots()
    inlier_idxs = np.nonzero(inliers)[0]
    plot_matches(
        ax,
        image1,
        image2,
        locations_1_to_use,
        locations_2_to_use,
        np.column_stack((inlier_idxs, inlier_idxs)),
        matches_color='b')
    ax.axis('off')
    ax.set_title('DELF correspondences')


if __name__ == '__main__':
    files_path = glob.glob(data_path)
    image1 = Image.open(files_path[0])
    image2 = Image.open(files_path[1])
    image1 = ImageOps.fit(image1, (256, 256), Image.ANTIALIAS)
    image2 = ImageOps.fit(image2, (256, 256), Image.ANTIALIAS)

    result1 = run_delf(image1)
    result2 = run_delf(image2)
    #print(result1)
    #print(result2)

    shape = result1['locations'].shape.as_list()
    #print(shape)
    match_images(image1, image2, result1, result2)

[AseiSugiyama]

Hi @VergilWang15 I guess your code uses sample code of this page.

delf_module = hub.Module("https://tfhub.dev/google/delf/1")

    delf_inputs = {
        # An image tensor with dtype float32 and shape [height, width, 3], where
        # height and width are positive integers:
        'image': float_image,
        # Scaling factors for building the image pyramid as described in the paper:
        'image_scales': [0.25, 0.3536, 0.5, 0.7071, 1.0, 1.4142, 2.0],
        # Image features whose attention score exceeds this threshold will be
        # returned:
        'score_threshold': 100.0,
        # The maximum number of features that should be returned:
        'max_feature_num': 1000,
    }

    # Apply the DELF module to the inputs to get the outputs.
    delf_outputs = delf_module(delf_inputs, as_dict=True)

However, hub.Module is deprecated and it is recommended to use hub.load to load tf.hub models. Check this document.

delf = hub.load('https://tfhub.dev/google/delf/1').signatures['default']

def run_delf(image):
  np_image = np.array(image)
  float_image = tf.image.convert_image_dtype(np_image, tf.float32)
  return delf(
      image=float_image,
      score_threshold=tf.constant(100.0),
      image_scales=tf.constant([0.25, 0.3536, 0.5, 0.7071, 1.0, 1.4142, 2.0]),
      max_feature_num=tf.constant(1000),
      )

I checked codes above to run the Jupyter notebook of TensorFlow Hub on Google Colab (Python 3.7.10 and TensorFlow 2.4.1, 2.5.0). Output should be like this.

[AseiSugiyama]

The migrated code should be like this. The output should be the same image.

from absl import logging

import matplotlib.pyplot as plt
import numpy as np
from PIL import Image, ImageOps
from scipy.spatial import cKDTree
from skimage.feature import plot_matches
from skimage.measure import ransac
from skimage.transform import AffineTransform
from six import BytesIO
import tensorflow as tf
import tensorflow_hub as hub
from six.moves.urllib.request import urlopen
from cv2 import cv2 as cv
import glob

# I located two images here because the default location of tf.keras.utils.get_file is this.
# This can be changed for your location.
data_path = '/root/.keras/datasets/*.jpg'

delf = hub.load('https://tfhub.dev/google/delf/1').signatures['default']

def run_delf(image):
  np_image = np.array(image)
  float_image = tf.image.convert_image_dtype(np_image, tf.float32)
  return delf(
      image=float_image,
      score_threshold=tf.constant(100.0),
      image_scales=tf.constant([0.25, 0.3536, 0.5, 0.7071, 1.0, 1.4142, 2.0]),
      max_feature_num=tf.constant(1000),
      )

def match_images(image1, image2, result1, result2):
  distance_threshold = 0.8

  # Read features.
  num_features_1 = result1['locations'].shape[0]
  print("Loaded image 1's %d features" % num_features_1)
  
  num_features_2 = result2['locations'].shape[0]
  print("Loaded image 2's %d features" % num_features_2)

  # Find nearest-neighbor matches using a KD tree.
  d1_tree = cKDTree(result1['descriptors'])
  _, indices = d1_tree.query(
      result2['descriptors'],
      distance_upper_bound=distance_threshold)

  # Select feature locations for putative matches.
  locations_2_to_use = np.array([
      result2['locations'][i,]
      for i in range(num_features_2)
      if indices[i] != num_features_1
  ])
  locations_1_to_use = np.array([
      result1['locations'][indices[i],]
      for i in range(num_features_2)
      if indices[i] != num_features_1
  ])

  # Perform geometric verification using RANSAC.
  _, inliers = ransac(
      (locations_1_to_use, locations_2_to_use),
      AffineTransform,
      min_samples=3,
      residual_threshold=20,
      max_trials=1000)

  print('Found %d inliers' % sum(inliers))

  # Visualize correspondences.
  _, ax = plt.subplots()
  inlier_idxs = np.nonzero(inliers)[0]
  plot_matches(
      ax,
      image1,
      image2,
      locations_1_to_use,
      locations_2_to_use,
      np.column_stack((inlier_idxs, inlier_idxs)),
      matches_color='b')
  ax.axis('off')
  ax.set_title('DELF correspondences')

if __name__ == '__main__':
  files_path = glob.glob(data_path)
  image1 = Image.open(files_path[0])
  image2 = Image.open(files_path[1])
  image1 = ImageOps.fit(image1, (256, 256), Image.ANTIALIAS)
  image2 = ImageOps.fit(image2, (256, 256), Image.ANTIALIAS)

  result1 = run_delf(image1)
  result2 = run_delf(image2)

  match_images(image1, image2, result1, result2)

[VergilWang15]

Thank you so much for your reply!
It does work after I changed to hub.load()
Now I could get on well with my own dataset, thank you🎈