From 34fc5f903b9d5401ad1f85ce5991bdd426f9fd33 Mon Sep 17 00:00:00 2001 From: liqikai Date: Tue, 8 Mar 2022 17:43:24 +0800 Subject: [PATCH 01/19] [Feature] Add shelf and campus datasets --- configs/_base_/datasets/campus.py | 151 +++++ configs/_base_/datasets/shelf.py | 151 +++++ ...ose_prn64x64x64_cpn80x80x20_campus_cam3.py | 216 ++++++ ...pose_prn64x64x64_cpn80x80x20_shelf_cam5.py | 213 ++++++ mmpose/core/evaluation/eval_hooks.py | 11 +- mmpose/datasets/__init__.py | 8 +- mmpose/datasets/datasets/__init__.py | 7 +- .../kpt_3d_mview_rgb_img_direct_dataset.py | 2 +- mmpose/datasets/datasets/body3d/__init__.py | 5 +- .../body3d_mview_direct_campus_dataset.py | 632 ++++++++++++++++++ .../body3d_mview_direct_shelf_dataset.py | 629 +++++++++++++++++ mmpose/datasets/pipelines/pose3d_transform.py | 211 +++++- 12 files changed, 2224 insertions(+), 12 deletions(-) create mode 100644 configs/_base_/datasets/campus.py create mode 100644 configs/_base_/datasets/shelf.py create mode 100644 configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py create mode 100644 configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py create mode 100644 mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py create mode 100644 mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py diff --git a/configs/_base_/datasets/campus.py b/configs/_base_/datasets/campus.py new file mode 100644 index 0000000000..334316e9c2 --- /dev/null +++ b/configs/_base_/datasets/campus.py @@ -0,0 +1,151 @@ +dataset_info = dict( + dataset_name='campus', + paper_info=dict( + author='Belagiannis, Vasileios and Amin, Sikandar and Andriluka, ' + 'Mykhaylo and Schiele, Bernt and Navab, Nassir and Ilic, Slobodan', + title='3D Pictorial Structures for Multiple Human Pose Estimation', + container='IEEE Computer Society Conference on Computer Vision and ' + 'Pattern Recognition (CVPR)', + year='2014', + homepage='http://campar.in.tum.de/Chair/MultiHumanPose', + ), + keypoint_info={ + 0: + dict( + name='right_ankle', + id=0, + color=[255, 128, 0], + type='lower', + swap='left_ankle'), + 1: + dict( + name='right_knee', + id=1, + color=[255, 128, 0], + type='lower', + swap='left_knee'), + 2: + dict( + name='right_hip', + id=2, + color=[255, 128, 0], + type='lower', + swap='left_hip'), + 3: + dict( + name='left_hip', + id=3, + color=[0, 255, 0], + type='lower', + swap='right_hip'), + 4: + dict( + name='left_knee', + id=4, + color=[0, 255, 0], + type='lower', + swap='right_knee'), + 5: + dict( + name='left_ankle', + id=5, + color=[0, 255, 0], + type='lower', + swap='right_ankle'), + 6: + dict( + name='right_wrist', + id=6, + color=[255, 128, 0], + type='upper', + swap='left_wrist'), + 7: + dict( + name='right_elbow', + id=7, + color=[255, 128, 0], + type='upper', + swap='left_elbow'), + 8: + dict( + name='right_shoulder', + id=8, + color=[255, 128, 0], + type='upper', + swap='left_shoulder'), + 9: + dict( + name='left_shoulder', + id=9, + color=[0, 255, 0], + type='upper', + swap='right_shoulder'), + 10: + dict( + name='left_elbow', + id=10, + color=[0, 255, 0], + type='upper', + swap='right_elbow'), + 11: + dict( + name='left_wrist', + id=11, + color=[0, 255, 0], + type='upper', + swap='right_wrist'), + 12: + dict( + name='bottom_head', + id=12, + color=[51, 153, 255], + type='upper', + swap=''), + 13: + dict( + name='top_head', + id=13, + color=[51, 153, 255], + type='upper', + swap=''), + }, + skeleton_info={ + 0: + dict(link=('right_ankle', 'right_knee'), id=0, color=[255, 128, 0]), + 1: + dict(link=('right_knee', 'right_hip'), id=1, color=[255, 128, 0]), + 2: + dict(link=('left_hip', 'left_knee'), id=2, color=[0, 255, 0]), + 3: + dict(link=('left_knee', 'left_ankle'), id=3, color=[0, 255, 0]), + 4: + dict(link=('right_hip', 'left_hip'), id=4, color=[51, 153, 255]), + 5: + dict(link=('right_wrist', 'right_elbow'), id=5, color=[255, 128, 0]), + 6: + dict( + link=('right_elbow', 'right_shoulder'), id=6, color=[255, 128, 0]), + 7: + dict(link=('left_shoulder', 'left_elbow'), id=7, color=[0, 255, 0]), + 8: + dict(link=('left_elbow', 'left_wrist'), id=8, color=[0, 255, 0]), + 9: + dict(link=('right_hip', 'right_shoulder'), id=9, color=[255, 128, 0]), + 10: + dict(link=('left_hip', 'left_shoulder'), id=10, color=[0, 255, 0]), + 11: + dict( + link=('right_shoulder', 'bottom_head'), id=11, color=[255, 128, + 0]), + 12: + dict(link=('left_shoulder', 'bottom_head'), id=12, color=[0, 255, 0]), + 13: + dict(link=('bottom_head', 'top_head'), id=13, color=[51, 153, 255]), + }, + joint_weights=[ + 1.5, 1.2, 1.0, 1.0, 1.2, 1.5, 1.5, 1.2, 1.0, 1.0, 1.2, 1.5, 1.0, 1.0 + ], + sigmas=[ + 0.089, 0.087, 0.107, 0.107, 0.087, 0.089, 0.062, 0.072, 0.079, 0.079, + 0.072, 0.062, 0.026, 0.026 + ]) diff --git a/configs/_base_/datasets/shelf.py b/configs/_base_/datasets/shelf.py new file mode 100644 index 0000000000..5fe6e42b3b --- /dev/null +++ b/configs/_base_/datasets/shelf.py @@ -0,0 +1,151 @@ +dataset_info = dict( + dataset_name='shelf', + paper_info=dict( + author='Belagiannis, Vasileios and Amin, Sikandar and Andriluka, ' + 'Mykhaylo and Schiele, Bernt and Navab, Nassir and Ilic, Slobodan', + title='3D Pictorial Structures for Multiple Human Pose Estimation', + container='IEEE Computer Society Conference on Computer Vision and ' + 'Pattern Recognition (CVPR)', + year='2014', + homepage='http://campar.in.tum.de/Chair/MultiHumanPose', + ), + keypoint_info={ + 0: + dict( + name='right_ankle', + id=0, + color=[255, 128, 0], + type='lower', + swap='left_ankle'), + 1: + dict( + name='right_knee', + id=1, + color=[255, 128, 0], + type='lower', + swap='left_knee'), + 2: + dict( + name='right_hip', + id=2, + color=[255, 128, 0], + type='lower', + swap='left_hip'), + 3: + dict( + name='left_hip', + id=3, + color=[0, 255, 0], + type='lower', + swap='right_hip'), + 4: + dict( + name='left_knee', + id=4, + color=[0, 255, 0], + type='lower', + swap='right_knee'), + 5: + dict( + name='left_ankle', + id=5, + color=[0, 255, 0], + type='lower', + swap='right_ankle'), + 6: + dict( + name='right_wrist', + id=6, + color=[255, 128, 0], + type='upper', + swap='left_wrist'), + 7: + dict( + name='right_elbow', + id=7, + color=[255, 128, 0], + type='upper', + swap='left_elbow'), + 8: + dict( + name='right_shoulder', + id=8, + color=[255, 128, 0], + type='upper', + swap='left_shoulder'), + 9: + dict( + name='left_shoulder', + id=9, + color=[0, 255, 0], + type='upper', + swap='right_shoulder'), + 10: + dict( + name='left_elbow', + id=10, + color=[0, 255, 0], + type='upper', + swap='right_elbow'), + 11: + dict( + name='left_wrist', + id=11, + color=[0, 255, 0], + type='upper', + swap='right_wrist'), + 12: + dict( + name='bottom_head', + id=12, + color=[51, 153, 255], + type='upper', + swap=''), + 13: + dict( + name='top_head', + id=13, + color=[51, 153, 255], + type='upper', + swap=''), + }, + skeleton_info={ + 0: + dict(link=('right_ankle', 'right_knee'), id=0, color=[255, 128, 0]), + 1: + dict(link=('right_knee', 'right_hip'), id=1, color=[255, 128, 0]), + 2: + dict(link=('left_hip', 'left_knee'), id=2, color=[0, 255, 0]), + 3: + dict(link=('left_knee', 'left_ankle'), id=3, color=[0, 255, 0]), + 4: + dict(link=('right_hip', 'left_hip'), id=4, color=[51, 153, 255]), + 5: + dict(link=('right_wrist', 'right_elbow'), id=5, color=[255, 128, 0]), + 6: + dict( + link=('right_elbow', 'right_shoulder'), id=6, color=[255, 128, 0]), + 7: + dict(link=('left_shoulder', 'left_elbow'), id=7, color=[0, 255, 0]), + 8: + dict(link=('left_elbow', 'left_wrist'), id=8, color=[0, 255, 0]), + 9: + dict(link=('right_hip', 'right_shoulder'), id=9, color=[255, 128, 0]), + 10: + dict(link=('left_hip', 'left_shoulder'), id=10, color=[0, 255, 0]), + 11: + dict( + link=('right_shoulder', 'bottom_head'), id=11, color=[255, 128, + 0]), + 12: + dict(link=('left_shoulder', 'bottom_head'), id=12, color=[0, 255, 0]), + 13: + dict(link=('bottom_head', 'top_head'), id=13, color=[51, 153, 255]), + }, + joint_weights=[ + 1.5, 1.2, 1.0, 1.0, 1.2, 1.5, 1.5, 1.2, 1.0, 1.0, 1.2, 1.5, 1.0, 1.0 + ], + sigmas=[ + 0.089, 0.087, 0.107, 0.107, 0.087, 0.089, 0.062, 0.072, 0.079, 0.079, + 0.072, 0.062, 0.026, 0.026 + ]) diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py new file mode 100644 index 0000000000..688c2f4661 --- /dev/null +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py @@ -0,0 +1,216 @@ +_base_ = [ + '../../../../_base_/default_runtime.py', + '../../../../_base_/datasets/campus.py' +] +checkpoint_config = dict(interval=1) +evaluation = dict( + interval=1, metric='pcp', save_best='pcp', recall_threshold=500) + +optimizer = dict( + type='Adam', + lr=0.0001, +) +optimizer_config = dict(grad_clip=None) + +# learning policy +lr_config = dict( + policy='step', + warmup='linear', + warmup_iters=500, + warmup_ratio=0.001, + step=[10, 20]) +total_epochs = 30 +log_config = dict( + interval=50, hooks=[ + dict(type='TextLoggerHook'), + ]) + +space_size = [12000.0, 12000.0, 2000.0] +space_center = [3000.0, 4500.0, 1000.0] +cube_size = [80, 80, 20] +sub_space_size = [2000.0, 2000.0, 2000.0] +sub_cube_size = [64, 64, 64] +image_size = [800, 640] +heatmap_size = [200, 160] + +num_joints = 17 + +data_root = 'data/campus' +train_data_cfg = dict( + space_size=space_size, + space_center=space_center, + cube_size=cube_size, + image_size=image_size, + heatmap_size=[heatmap_size], + num_joints=num_joints, + cam_list=[0, 1, 2], + num_cameras=3, + subset='train', + width=360, + height=288, + root_id=[2, 3], + coco_lhip_idx=11, + coco_rhip_idx=12, + max_nposes=10, + min_nposes=1, + num_train_sampels=3000, + maximum_person=10, + cam_file=f'{data_root}/calibration_campus.json', + test_pose_db_file=f'{data_root}/pred_campus_maskrcnn_hrnet_coco.pkl', + train_pose_db_file=f'{data_root}/panoptic_training_pose.pkl', + gt_pose_db_file=f'{data_root}/actorsGT.mat', +) + +test_data_cfg = train_data_cfg.copy() +test_data_cfg.update(dict(subset='test')) + +# model settings +model = dict( + type='DetectAndRegress', + backbone=None, + pretrained=None, + human_detector=dict( + type='VoxelCenterDetector', + image_size=image_size, + heatmap_size=heatmap_size, + space_size=space_size, + cube_size=cube_size, + space_center=space_center, + center_net=dict( + type='V2VNet', input_channels=num_joints, output_channels=1), + center_head=dict( + type='CuboidCenterHead', + space_size=space_size, + space_center=space_center, + cube_size=cube_size, + max_num=10, + max_pool_kernel=3), + train_cfg=dict(dist_threshold=500.0), + test_cfg=dict(center_threshold=0.1), + ), + pose_regressor=dict( + type='VoxelSinglePose', + image_size=image_size, + heatmap_size=heatmap_size, + sub_space_size=sub_space_size, + sub_cube_size=sub_cube_size, + num_joints=num_joints, + pose_net=dict( + type='V2VNet', + input_channels=num_joints, + output_channels=num_joints), + pose_head=dict(type='CuboidPoseHead', beta=100.0))) + +train_pipeline = [ + dict( + type='MultiItemProcess', + pipeline=[ + dict( + type='AffineJoints', + item='joints', + visible_item='joints_visible'), + dict( + type='GenerateInputHeatmaps', + item='joints', + visible_item='joints_visible', + obscured=0.0, + from_pred=False, + sigma=3, + scale=1.0, + base_size=96, + target_type='gaussian', + heatmap_cfg=dict( + base_scale=0.9, + offset=0.03, + threshold=0.6, + extra=[ + dict( + joint_ids=[7, 8], scale_factor=0.5, threshold=0.1), + dict( + joint_ids=[9, 10], + scale_factor=0.2, + threshold=0.1, + ), + dict( + joint_ids=[ + 0, 1, 2, 3, 4, 5, 6, 11, 12, 13, 14, 15, 16 + ], + scale_factor=0.5, + threshold=0.05) + ])) + ]), + dict( + type='DiscardDuplicatedItems', + keys_list=[ + 'joints_3d', 'joints_3d_visible', 'ann_info', 'roots_3d', + 'num_persons', 'sample_id' + ]), + dict( + type='GenerateVoxel3DHeatmapTarget', + sigma=200.0, + joint_indices=[[11, 12]]), + dict( + type='Collect', + keys=['sample_id', 'input_heatmaps', 'targets_3d'], + meta_keys=[ + 'num_persons', 'joints_3d', 'camera', 'center', 'scale', + 'joints_3d_visible', 'roots_3d', 'sample_id' + ]), +] + +val_pipeline = [ + dict( + type='MultiItemProcess', + pipeline=[ + dict(type='AffineJoints', item='joints'), + dict( + type='GenerateInputHeatmaps', + item='joints', + from_pred=True, + scale=1.0, + sigma=3, + base_size=96, + target_type='gaussian'), + ]), + dict( + type='DiscardDuplicatedItems', + keys_list=[ + 'joints_3d', 'joints_3d_visible', 'joints_2d', 'joints_2d_visible', + 'ann_info', 'sample_id' + ]), + dict( + type='Collect', + keys=['sample_id', 'input_heatmaps'], + meta_keys=['sample_id', 'camera', 'center', 'scale']), +] + +test_pipeline = val_pipeline + +data_root = 'data/campus' +data = dict( + samples_per_gpu=1, + workers_per_gpu=4, + val_dataloader=dict(samples_per_gpu=1), + test_dataloader=dict(samples_per_gpu=1), + train=dict( + type='Body3DMviewDirectCampusDataset', + ann_file=None, + img_prefix=data_root, + data_cfg=train_data_cfg, + pipeline=train_pipeline, + dataset_info={{_base_.dataset_info}}), + val=dict( + type='Body3DMviewDirectCampusDataset', + ann_file=None, + img_prefix=data_root, + data_cfg=test_data_cfg, + pipeline=val_pipeline, + dataset_info={{_base_.dataset_info}}), + test=dict( + type='Body3DMviewDirectCampusDataset', + ann_file=None, + img_prefix=data_root, + data_cfg=test_data_cfg, + pipeline=test_pipeline, + dataset_info={{_base_.dataset_info}}), +) diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py new file mode 100644 index 0000000000..d1b55a1ce0 --- /dev/null +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py @@ -0,0 +1,213 @@ +_base_ = [ + '../../../../_base_/default_runtime.py', + '../../../../_base_/datasets/shelf.py' +] +checkpoint_config = dict(interval=1) +evaluation = dict( + interval=1, metric='pcp', save_best='pcp', recall_threshold=500) + +optimizer = dict( + type='Adam', + lr=0.0001, +) +optimizer_config = dict(grad_clip=None) + +# learning rate policy +lr_config = dict( + policy='step', + warmup='linear', + warmup_iters=500, + warmup_ratio=0.001, + step=[10, 20]) +total_epochs = 30 +log_config = dict( + interval=50, hooks=[ + dict(type='TextLoggerHook'), + ]) + +space_size = [8000, 8000, 2000] +space_center = [450, -320, 800] +cube_size = [80, 80, 20] +sub_space_size = [2000, 2000, 2000] +sub_cube_size = [64, 64, 64] +image_size = [800, 608] +heatmap_size = [200, 152] + +num_joints = 17 + +data_root = 'data/shelf' +train_data_cfg = dict( + space_size=space_size, + space_center=space_center, + cube_size=cube_size, + image_size=image_size, + heatmap_size=[heatmap_size], + num_joints=num_joints, + cam_list=[0, 1, 2, 3, 4], + num_cameras=5, + subset='train', + width=1032, + height=776, + root_id=[11, 12], + max_nposes=6, + min_nposes=1, + num_train_samples=3000, + maximum_person=10, + cam_file=f'{data_root}/calibration_shelf.json', + test_pose_db_file=f'{data_root}/pred_shelf_maskrcnn_hrnet_coco.pkl', + train_pose_db_file=f'{data_root}/panoptic_training_pose.pkl', + gt_pose_db_file=f'{data_root}/actorsGT.mat', +) + +test_data_cfg = train_data_cfg.copy() +test_data_cfg.update(dict(subset='test')) + +# model settings +model = dict( + type='DetectAndRegress', + backbone=None, + pretrained=None, + human_detector=dict( + type='VoxelCenterDetector', + image_size=image_size, + heatmap_size=heatmap_size, + space_size=space_size, + cube_size=cube_size, + space_center=space_center, + center_net=dict( + type='V2VNet', input_channels=num_joints, output_channels=1), + center_head=dict( + type='CuboidCenterHead', + space_size=space_size, + space_center=space_center, + cube_size=cube_size, + max_num=10, + max_pool_kernel=3), + train_cfg=dict(dist_threshold=500.0), + test_cfg=dict(center_threshold=0.1), + ), + pose_regressor=dict( + type='VoxelSinglePose', + image_size=image_size, + heatmap_size=heatmap_size, + sub_space_size=sub_space_size, + sub_cube_size=sub_cube_size, + num_joints=num_joints, + pose_net=dict( + type='V2VNet', + input_channels=num_joints, + output_channels=num_joints), + pose_head=dict(type='CuboidPoseHead', beta=100.0))) + +train_pipeline = [ + dict( + type='MultiItemProcess', + pipeline=[ + dict( + type='AffineJoints', + item='joints', + visible_item='joints_visible'), + dict( + type='GenerateInputHeatmaps', + item='joints', + visible_item='joints_visible', + obscured=0.05, + from_pred=False, + sigma=3, + scale=1.0, + base_size=96, + target_type='gaussian', + heatmap_cfg=dict( + base_scale=0.9, + offset=0.03, + threshold=0.6, + extra=[ + dict( + joint_ids=[7, 8, 13, 14], + scale_factor=0.5, + threshold=0.1), + dict( + joint_ids=[9, 10, 15, 16], + scale_factor=0.2, + threshold=0.1, + ), + dict( + joint_ids=[0, 1, 2, 3, 4, 5, 6, 11, 12], + scale_factor=0.5, + threshold=0.05) + ])), + ]), + dict( + type='DiscardDuplicatedItems', + keys_list=[ + 'joints_3d', 'joints_3d_visible', 'ann_info', 'roots_3d', + 'num_persons', 'sample_id' + ]), + dict( + type='GenerateVoxel3DHeatmapTarget', + sigma=200.0, + joint_indices=[[11, 12]]), + dict( + type='Collect', + keys=['sample_id', 'input_heatmaps', 'targets_3d'], + meta_keys=[ + 'num_persons', 'joints_3d', 'camera', 'center', 'scale', + 'joints_3d_visible', 'roots_3d', 'sample_id' + ]), +] + +val_pipeline = [ + dict( + type='MultiItemProcess', + pipeline=[ + dict(type='AffineJoints', item='joints'), + dict( + type='GenerateInputHeatmaps', + item='joints', + from_pred=True, + sigma=3, + base_size=96, + target_type='gaussian'), + ]), + dict( + type='DiscardDuplicatedItems', + keys_list=[ + 'joints_3d', 'joints_3d_visible', 'joints_2d', 'joints_2d_visible', + 'ann_info', 'sample_id' + ]), + dict( + type='Collect', + keys=['sample_id', 'input_heatmaps'], + meta_keys=['sample_id', 'camera', 'center', 'scale']), +] + +test_pipeline = val_pipeline + +data_root = 'data/shelf' +data = dict( + samples_per_gpu=1, + workers_per_gpu=4, + val_dataloader=dict(samples_per_gpu=4), + test_dataloader=dict(samples_per_gpu=4), + train=dict( + type='Body3DMviewDirectShelfDataset', + ann_file=None, + img_prefix=data_root, + data_cfg=train_data_cfg, + pipeline=train_pipeline, + dataset_info={{_base_.dataset_info}}), + val=dict( + type='Body3DMviewDirectShelfDataset', + ann_file=None, + img_prefix=data_root, + data_cfg=test_data_cfg, + pipeline=val_pipeline, + dataset_info={{_base_.dataset_info}}), + test=dict( + type='Body3DMviewDirectShelfDataset', + ann_file=None, + img_prefix=data_root, + data_cfg=test_data_cfg, + pipeline=test_pipeline, + dataset_info={{_base_.dataset_info}}), +) diff --git a/mmpose/core/evaluation/eval_hooks.py b/mmpose/core/evaluation/eval_hooks.py index cf36a03885..1cebeab65a 100644 --- a/mmpose/core/evaluation/eval_hooks.py +++ b/mmpose/core/evaluation/eval_hooks.py @@ -5,7 +5,16 @@ from mmcv.runner import EvalHook as _EvalHook MMPOSE_GREATER_KEYS = [ - 'acc', 'ap', 'ar', 'pck', 'auc', '3dpck', 'p-3dpck', '3dauc', 'p-3dauc' + 'acc', + 'ap', + 'ar', + 'pck', + 'auc', + '3dpck', + 'p-3dpck', + '3dauc', + 'p-3dauc', + 'pcp', ] MMPOSE_LESS_KEYS = ['loss', 'epe', 'nme', 'mpjpe', 'p-mpjpe', 'n-mpjpe'] diff --git a/mmpose/datasets/__init__.py b/mmpose/datasets/__init__.py index 1b9e7cf035..650d7e9480 100644 --- a/mmpose/datasets/__init__.py +++ b/mmpose/datasets/__init__.py @@ -19,7 +19,9 @@ TopDownFreiHandDataset, TopDownH36MDataset, TopDownJhmdbDataset, TopDownMhpDataset, TopDownMpiiDataset, TopDownMpiiTrbDataset, TopDownOCHumanDataset, TopDownOneHand10KDataset, TopDownPanopticDataset, - TopDownPoseTrack18Dataset, TopDownPoseTrack18VideoDataset) + TopDownPoseTrack18Dataset, TopDownPoseTrack18VideoDataset, + Body3DMviewDirectPanopticDataset, Body3DMviewDirectShelfDataset, + Body3DMviewDirectCampusDataset) __all__ = [ 'TopDownCocoDataset', 'BottomUpCocoDataset', 'BottomUpMhpDataset', @@ -38,5 +40,7 @@ 'AnimalFlyDataset', 'AnimalLocustDataset', 'AnimalZebraDataset', 'AnimalATRWDataset', 'AnimalPoseDataset', 'TopDownH36MDataset', 'TopDownPoseTrack18VideoDataset', 'build_dataloader', 'build_dataset', - 'Compose', 'DistributedSampler', 'DATASETS', 'PIPELINES', 'DatasetInfo' + 'Compose', 'DistributedSampler', 'DATASETS', 'PIPELINES', 'DatasetInfo', + 'Body3DMviewDirectPanopticDataset', 'Body3DMviewDirectShelfDataset', + 'Body3DMviewDirectCampusDataset' ] diff --git a/mmpose/datasets/datasets/__init__.py b/mmpose/datasets/datasets/__init__.py index f3839e5eaa..603f840206 100644 --- a/mmpose/datasets/datasets/__init__.py +++ b/mmpose/datasets/datasets/__init__.py @@ -4,7 +4,9 @@ from .animal import (AnimalATRWDataset, AnimalFlyDataset, AnimalHorse10Dataset, AnimalLocustDataset, AnimalMacaqueDataset, AnimalPoseDataset, AnimalZebraDataset) -from .body3d import Body3DH36MDataset, Body3DMviewDirectPanopticDataset +from .body3d import (Body3DH36MDataset, Body3DMviewDirectCampusDataset, + Body3DMviewDirectPanopticDataset, + Body3DMviewDirectShelfDataset) from .bottom_up import (BottomUpAicDataset, BottomUpCocoDataset, BottomUpCocoWholeBodyDataset, BottomUpCrowdPoseDataset, BottomUpMhpDataset) @@ -41,5 +43,6 @@ 'AnimalFlyDataset', 'AnimalLocustDataset', 'AnimalZebraDataset', 'AnimalATRWDataset', 'AnimalPoseDataset', 'TopDownH36MDataset', 'TopDownHalpeDataset', 'TopDownPoseTrack18VideoDataset', - 'Body3DMviewDirectPanopticDataset' + 'Body3DMviewDirectPanopticDataset', 'Body3DMviewDirectShelfDataset', + 'Body3DMviewDirectCampusDataset' ] diff --git a/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py b/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py index 94cc1c22e9..5e79be1139 100644 --- a/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py +++ b/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py @@ -61,7 +61,7 @@ def __init__(self, dataset_info = DatasetInfo(dataset_info) - assert self.ann_info['num_joints'] <= dataset_info.keypoint_num + # assert self.ann_info['num_joints'] <= dataset_info.keypoint_num self.ann_info['flip_pairs'] = dataset_info.flip_pairs self.ann_info['num_scales'] = 1 self.ann_info['flip_index'] = dataset_info.flip_index diff --git a/mmpose/datasets/datasets/body3d/__init__.py b/mmpose/datasets/datasets/body3d/__init__.py index 5bc25a9ebb..e17444661f 100644 --- a/mmpose/datasets/datasets/body3d/__init__.py +++ b/mmpose/datasets/datasets/body3d/__init__.py @@ -1,11 +1,14 @@ # Copyright (c) OpenMMLab. All rights reserved. from .body3d_h36m_dataset import Body3DH36MDataset from .body3d_mpi_inf_3dhp_dataset import Body3DMpiInf3dhpDataset +from .body3d_mview_direct_campus_dataset import Body3DMviewDirectCampusDataset from .body3d_mview_direct_panoptic_dataset import \ Body3DMviewDirectPanopticDataset +from .body3d_mview_direct_shelf_dataset import Body3DMviewDirectShelfDataset from .body3d_semi_supervision_dataset import Body3DSemiSupervisionDataset __all__ = [ 'Body3DH36MDataset', 'Body3DSemiSupervisionDataset', - 'Body3DMpiInf3dhpDataset', 'Body3DMviewDirectPanopticDataset' + 'Body3DMpiInf3dhpDataset', 'Body3DMviewDirectPanopticDataset', + 'Body3DMviewDirectShelfDataset', 'Body3DMviewDirectCampusDataset' ] diff --git a/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py b/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py new file mode 100644 index 0000000000..be9b9e3690 --- /dev/null +++ b/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py @@ -0,0 +1,632 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy +# import glob +import json +import os.path as osp +import pickle +import random +import tempfile +import warnings +from collections import OrderedDict + +import mmcv +import numpy as np +import scipy.io as scio +from mmcv import Config + +from mmpose.core.camera import SimpleCamera +from mmpose.datasets.builder import DATASETS +from mmpose.datasets.datasets.base import Kpt3dMviewRgbImgDirectDataset + + +@DATASETS.register_module() +class Body3DMviewDirectCampusDataset(Kpt3dMviewRgbImgDirectDataset): + """Campus dataset for direct multi-view human pose estimation. + + `3D Pictorial Structures for Multiple Human Pose Estimation' CVPR'2014 + More details can be found in the paper + ` + + The dataset loads both 2D and 3D annotations as well as camera parameters. + + Campus keypoint indices:: + + 'Right-Ankle': 0, + 'Right-Knee': 1, + 'Right-Hip': 2, + 'Left-Hip': 3, + 'Left-Knee': 4, + 'Left-Ankle': 5, + 'Right-Wrist': 6, + 'Right-Elbow': 7, + 'Right-Shoulder': 8, + 'Left-Shoulder': 9, + 'Left-Elbow': 10, + 'Left-Wrist': 11, + 'Bottom-Head': 12, + 'Top-Head': 13, + + Args: + ann_file (str): Path to the annotation file. + img_prefix (str): Path to a directory where images are held. + Default: None. + data_cfg (dict): config + pipeline (list[dict | callable]): A sequence of data transforms. + dataset_info (DatasetInfo): A class containing all dataset info. + test_mode (bool): Store True when building test or + validation dataset. Default: False. + """ + ALLOWED_METRICS = {'pcp', '3dpcp'} + LIMBS = [[0, 1], [1, 2], [3, 4], [4, 5], [6, 7], [7, 8], [9, 10], [10, 11], + [12, 13]] + BONE_GROUP = OrderedDict([('Head', [8]), ('Torso', [9]), + ('Upper arms', [5, 6]), ('Lower arms', [4, 7]), + ('Upper legs', [1, 2]), ('Lower legs', [0, 3])]) + + def __init__(self, + ann_file, + img_prefix, + data_cfg, + pipeline, + dataset_info=None, + test_mode=False): + + if dataset_info is None: + warnings.warn( + 'dataset_info is missing. ' + 'Check https://github.com/open-mmlab/mmpose/pull/663 ' + 'for details.', DeprecationWarning) + cfg = Config.fromfile('configs/_base_/datasets/campus.py') + dataset_info = cfg._cfg_dict['dataset_info'] + + super().__init__( + ann_file, + img_prefix, + data_cfg, + pipeline, + dataset_info=dataset_info, + test_mode=test_mode) + + self.load_config(data_cfg) + self.ann_info['use_different_joint_weights'] = data_cfg.get( + 'use_different_joint_weights', False) + + self.db_size = self.num_cameras * len( + self.frame_range + ) if self.test_mode else self.num_cameras * self.num_train_sampels + print(f'=> load {self.db_size} samples') + + def load_config(self, data_cfg): + """Initialize dataset attributes according to the config. + + Override this method to set dataset specific attributes. + """ + self.num_joints = data_cfg['num_joints'] + self.cam_list = data_cfg['cam_list'] + self.num_cameras = data_cfg['num_cameras'] + assert self.num_cameras == len(self.cam_list) + self.need_camera_param = True + + self.subset = data_cfg.get('subset', 'train') + if self.subset == 'test': + self.frame_range = list(range(350, 471)) + list(range(650, 751)) + else: + self.frame_range = list(range(0, 350)) + list(range( + 471, 650)) + list(range(751, 2000)) + + self.width = data_cfg.get('width', 360) + self.height = data_cfg.get('height', 288) + self.center = np.array((self.width / 2, self.height / 2), + dtype=np.float32) + self.scale = self._get_scale((self.width, self.height)) + + root_id = data_cfg.get('root_id', [11, 12]) + self.root_id = [root_id] if isinstance(root_id, int) else root_id + + self.max_nposes = data_cfg.get('max_nposes', 10) + self.min_nposes = data_cfg.get('min_nposes', 1) + self.num_train_sampels = data_cfg.get('num_train_sampels', 3000) + self.maximum_person = data_cfg.get('maximum_person', 10) + + self.cam_file = data_cfg.get( + 'cam_file', osp.join(self.img_prefix, 'calibration_campus.json')) + + self.test_pose_db_file = data_cfg.get( + 'test_pose_db_file', + osp.join(self.img_prefix, 'pred_campus_maskrcnn_hrnet_coco.pkl')) + + self.train_pose_db_file = data_cfg.get( + 'train_pose_db_file', + osp.join(self.img_prefix, 'panoptic_training_pose.pkl')) + + self.gt_pose_db_file = data_cfg.get( + 'gt_pose_db_file', osp.join(self.img_prefix, 'actorsGT.mat')) + + self._get_db() + + def _get_scale(self, raw_image_size): + heatmap_size = self.ann_info['heatmap_size'] + image_size = self.ann_info['image_size'] + assert heatmap_size[0][0] / heatmap_size[0][1] \ + == image_size[0] / image_size[1] + w, h = raw_image_size + w_resized, h_resized = image_size + if w / w_resized < h / h_resized: + w_pad = h / h_resized * w_resized + h_pad = h + else: + w_pad = w + h_pad = w / w_resized * h_resized + + scale = np.array([w_pad, h_pad], dtype=np.float32) + + return scale + + def _get_db(self): + """load related db files.""" + assert osp.exists(self.cam_file), f'camera calibration file ' \ + f"{self.cam_file} doesn't exist, please check again" + with open(self.cam_file) as cfile: + calib = json.load(cfile) + self.cameras = self._get_cam(calib) + + assert osp.exists(self.train_pose_db_file), f'train_pose_db_file ' \ + f"{self.train_pose_db_file} doesn't exist, please check again" + with open(self.train_pose_db_file, 'rb') as pfile: + self.train_pose_db = pickle.load(pfile) + + assert osp.exists(self.test_pose_db_file), f'test_pose_db_file ' \ + f"{self.test_pose_db_file} doesn't exist, please check again" + with open(self.test_pose_db_file, 'rb') as pfile: + self.test_pose_db = pickle.load(pfile) + + assert osp.exists(self.gt_pose_db_file), f'gt_pose_db_file ' \ + f"{self.gt_pose_db_file} doesn't exist, please check again" + gt = scio.loadmat(self.gt_pose_db_file) + self.gt_pose_db = np.array(np.array( + gt['actor3D'].tolist()).tolist()).squeeze() + + self.num_persons = len(self.gt_pose_db) + + def _get_cam(self, calib): + """Get camera parameters. + + Returns: Camera parameters. + """ + cameras = {} + for id, cam in calib.items(): + sel_cam = {} + # note the transpose operation different from from VoxelPose + sel_cam['R'] = np.array(cam['R'], dtype=np.float32).T + sel_cam['T'] = np.array(cam['T'], dtype=np.float32) + + sel_cam['k'] = np.array(cam['k'], dtype=np.float32) + sel_cam['p'] = np.array(cam['p'], dtype=np.float32) + + sel_cam['f'] = [[cam['fx']], [cam['fy']]] + sel_cam['c'] = [[cam['cx']], [cam['cy']]] + + cameras[id] = sel_cam + + return cameras + + def __getitem__(self, idx): + """Get the sample given index.""" + + if self.test_mode: + results = self._prepare_test_sample(idx) + else: + results = self._prepare_train_sample(idx) + + return self.pipeline(results) + + def _prepare_test_sample(self, idx): + results = {} + fid = self.frame_range[idx] + + for cam_id, cam_param in self.cameras.items(): + image_file = osp.join( + self.img_prefix, 'Camera' + cam_id, + 'campus4-c{0}-{1:05d}.png'.format(cam_id, fid)) + assert osp.exists(image_file), f'the image file {image_file}' \ + f'does not exist, please check it again' + + all_poses_3d = [] + all_poses_3d_vis = [] + all_poses_2d = [] + all_poses_2d_vis = [] + single_view_camera = SimpleCamera(cam_param) + + for person in range(self.num_persons): + pose3d = self.gt_pose_db[person][fid] * 1000.0 + if len(pose3d[0]) > 0: + all_poses_3d.append(pose3d) + all_poses_3d_vis.append(np.ones((self.num_joints, 3))) + + pose2d = single_view_camera.world_to_pixel(pose3d) + x_check = np.bitwise_and(pose2d[:, 0] >= 0, + pose2d[:, 0] <= self.width - 1) + y_check = np.bitwise_and(pose2d[:, 1] >= 0, + pose2d[:, 1] <= self.height - 1) + check = np.bitwise_and(x_check, y_check) + + joints_vis = np.ones((len(pose2d), 1)) + joints_vis[np.logical_not(check)] = 0 + all_poses_2d.append(pose2d) + all_poses_2d_vis.append( + np.repeat(np.reshape(joints_vis, (-1, 1)), 2, axis=1)) + + pred_index = '{}_{}'.format(cam_id, fid) + pred_poses = self.test_pose_db[pred_index] + preds = [] + for pose in pred_poses: + preds.append(np.array(pose['pred'])) + preds = np.array(preds) + + results[int(cam_id)] = { + 'image_file': image_file, + 'joints_3d': all_poses_3d, + 'joints_3d_visible': all_poses_3d_vis, + 'joints_2d': all_poses_2d, + 'joints_2d_visible': all_poses_2d_vis, + 'camera': cam_param, + 'joints': preds, + 'sample_id': idx * self.num_cameras + int(cam_id), + 'center': self.center, + 'scale': self.scale, + 'rotation': 0.0, + 'ann_info': self.ann_info + } + + return results + + def _prepare_train_sample(self, idx): + results = {} + nposes_ori = np.random.choice(range(self.min_nposes, self.max_nposes)) + + select_poses = np.random.choice(self.train_pose_db, nposes_ori) + + joints_3d = np.array([p['pose'] for p in select_poses]) + joints_3d_vis = np.array([p['vis'] for p in select_poses]) + + bbox_list = [] + center_list = [] + for n in range(nposes_ori): + points = joints_3d[n][:, :2].copy() + center = np.mean(points[self.root_id, :2], axis=0) + rot_rad = np.random.uniform(-180, 180) + + new_center = self.get_new_center(center_list) + new_xy = self.rotate_points(points, center, + rot_rad) - center + new_center + + loop_count = 0 + # here n will be at least 1 + while not self.isvalid(new_center, + self.calc_bbox(new_xy, joints_3d_vis[n]), + bbox_list): + loop_count += 1 + if loop_count >= 100: + break + new_center = self.get_new_center(center_list) + new_xy = self.rotate_points(points, center, + rot_rad) - center + new_center + + if loop_count >= 100: + nposes = n + joints_3d = joints_3d[:n] + joints_3d_vis = joints_3d_vis[:n] + break + else: + nposes = nposes_ori + center_list.append(new_center) + bbox_list.append(self.calc_bbox(new_xy, joints_3d_vis[n])) + joints_3d[n][:, :2] = new_xy + + joints_3d_u = np.zeros((self.maximum_person, len(joints_3d[0]), 3)) + joints_3d_vis_u = np.zeros((self.maximum_person, len(joints_3d[0]), 3)) + for i in range(nposes): + joints_3d_u[i] = joints_3d[i][:, 0:3] + joints_3d_vis_u[i] = joints_3d_vis[i][:, 0:3] + + roots_3d = np.mean(joints_3d_u[:, self.root_id], axis=1) + + for cam_id, cam_param in self.cameras.items(): + joints = [] + joints_vis = [] + single_view_camera = SimpleCamera(cam_param) + for n in range(nposes): + pose2d = single_view_camera.world_to_pixel(joints_3d[n]) + x_check = np.bitwise_and(pose2d[:, 0] >= 0, + pose2d[:, 0] <= self.width - 1) + y_check = np.bitwise_and(pose2d[:, 1] >= 0, + pose2d[:, 1] <= self.height - 1) + check = np.bitwise_and(x_check, y_check) + vis = joints_3d_vis[n][:, 0] > 0 + vis[np.logical_not(check)] = 0 + + joints.append(pose2d) + joints_vis.append( + np.repeat(np.reshape(vis, (-1, 1)), 2, axis=1)) + + # make joints and joints_vis having same shape + joints_u = np.zeros((self.maximum_person, len(joints[0]), 2)) + joints_vis_u = np.zeros((self.maximum_person, len(joints[0]), 2)) + for i in range(nposes): + joints_u[i] = joints[i] + joints_vis_u[i] = joints_vis[i] + + results[int(cam_id)] = { + 'joints_3d': joints_3d_u, + 'joints_3d_visible': joints_3d_vis_u, + 'roots_3d': roots_3d, + 'joints': joints_u, + 'joints_visible': joints_vis_u, + 'camera': cam_param, + 'sample_id': idx * self.num_cameras + int(cam_id), + 'center': self.center, + 'scale': self.scale, + 'rotation': 0.0, + 'num_persons': nposes, + 'ann_info': self.ann_info + } + + return results + + def __len__(self): + """Get the size of the dataset.""" + if self.test_mode: + return len(self.frame_range) + else: + return self.num_train_sampels + + @staticmethod + def get_new_center(center_list): + if len(center_list) == 0 or random.random() < 0.7: + new_center = np.array([ + np.random.uniform(-2500.0, 8500.0), + np.random.uniform(-1000.0, 10000.0) + ]) + else: + xy = center_list[np.random.choice(range(len(center_list)))] + new_center = xy + np.random.normal(500, 50, 2) * np.random.choice( + [1, -1], 2) + + return new_center + + def isvalid(self, new_center, bbox, bbox_list): + new_center_us = new_center.reshape(1, -1) + vis = 0 + for _, cam_param in self.cameras.items(): + single_view_camera = SimpleCamera(cam_param) + loc_2d = single_view_camera.world_to_pixel( + np.hstack((new_center_us, [[1000.0]]))) + if 10 < loc_2d[0, 0] < self.width - 10 and 10 < loc_2d[ + 0, 1] < self.height - 10: + vis += 1 + + if len(bbox_list) == 0: + return vis >= 2 + + bbox_list = np.array(bbox_list) + x0 = np.maximum(bbox[0], bbox_list[:, 0]) + y0 = np.maximum(bbox[1], bbox_list[:, 1]) + x1 = np.minimum(bbox[2], bbox_list[:, 2]) + y1 = np.minimum(bbox[3], bbox_list[:, 3]) + + intersection = np.maximum(0, (x1 - x0) * (y1 - y0)) + area = (bbox[2] - bbox[0]) * (bbox[3] - bbox[1]) + area_list = (bbox_list[:, 2] - bbox_list[:, 0]) * ( + bbox_list[:, 3] - bbox_list[:, 1]) + iou_list = intersection / (area + area_list - intersection) + + return vis >= 2 and np.max(iou_list) < 0.01 + + @staticmethod + def calc_bbox(pose, pose_vis): + index = pose_vis[:, 0] > 0 + bbox = [ + np.min(pose[index, 0]), + np.min(pose[index, 1]), + np.max(pose[index, 0]), + np.max(pose[index, 1]) + ] + + return np.array(bbox) + + @staticmethod + def rotate_points(points, center, rot_rad): + """ + :param points: N*2 + :param center: 2 + :param rot_rad: scalar + :return: N*2 + """ + rot_rad = rot_rad * np.pi / 180.0 + rotate_mat = np.array([[np.cos(rot_rad), -np.sin(rot_rad)], + [np.sin(rot_rad), + np.cos(rot_rad)]]) + center = center.reshape(2, 1) + points = points.T + points = rotate_mat.dot(points - center) + center + + return points.T + + def evaluate(self, + results, + res_folder=None, + metric='pcp', + recall_threshold=500, + alpha_error=0.5, + **kwargs): + """ + Args: + results (list[dict]): Testing results containing the following + items: + - pose_3d (np.ndarray): predicted 3D human pose + - sample_id (np.ndarray): sample id of a frame. + res_folder (str, optional): The folder to save the testing + results. If not specified, a temp folder will be created. + Default: None. + metric (str | list[str]): Metric to be performed. + Defaults: 'pcp'. + recall_threshold: threshold for calculating recall. + alpha_error: coefficient when calculating error for correct parts. + **kwargs: + + Returns: + + """ + pose_3ds = np.concatenate([result['pose_3d'] for result in results], + axis=0) + sample_ids = [] + for result in results: + sample_ids.extend(result['sample_id']) + _results = [ + dict(sample_id=sample_id, pose_3d=pose_3d) + for (sample_id, pose_3d) in zip(sample_ids, pose_3ds) + ] + _results = self._sort_and_unique_outputs(_results, key='sample_id') + + metrics = metric if isinstance(metric, list) else [metric] + for _metric in metrics: + if _metric not in self.ALLOWED_METRICS: + raise ValueError( + f'Unsupported metric "{_metric}"' + f'Supported metrics are {self.ALLOWED_METRICS}') + + if res_folder is not None: + tmp_folder = None + res_file = osp.join(res_folder, 'result_keypoints.json') + else: + tmp_folder = tempfile.TemporaryDirectory() + res_file = osp.join(tmp_folder.name, 'result_keypoints.json') + mmcv.dump(_results, res_file) + + gt_num = self.db_size // self.num_cameras + assert len( + _results) == gt_num, f'number mismatch: {len(_results)}, {gt_num}' + + match_gt = 0 + total_gt = 0 + + correct_parts = np.zeros(self.num_persons) + total_parts = np.zeros(self.num_persons) + bone_correct_parts = np.zeros((self.num_persons, len(self.LIMBS) + 1)) + + for i, fid in enumerate(self.frame_range): + pred_coco = pose_3ds[i].copy() + pred_coco = pred_coco[pred_coco[:, 0, 3] >= 0, :, :3] + + if len(pred_coco) == 0: + continue + + pred = np.stack([ + self.coco2campus3D(p) + for p in copy.deepcopy(pred_coco[:, :, :3]) + ]) + + for person in range(self.num_persons): + gt = self.gt_pose_db[person][fid] * 1000.0 + if len(gt[0]) == 0: + continue + + mpjpes = np.mean( + np.sqrt(np.sum((gt[np.newaxis] - pred)**2, axis=-1)), + axis=-1) + min_n = np.argmin(mpjpes) + min_mpjpe = np.min(mpjpes) + if min_mpjpe < recall_threshold: + match_gt += 1 + total_gt += 1 + + for j, k in enumerate(self.LIMBS): + total_parts[person] += 1 + error_s = np.linalg.norm(pred[min_n, k[0], 0:3] - gt[k[0]]) + error_e = np.linalg.norm(pred[min_n, k[1], 0:3] - gt[k[1]]) + limb_length = np.linalg.norm(gt[k[0]] - gt[k[1]]) + + if (error_s + error_e) / 2.0 <= alpha_error * limb_length: + correct_parts[person] += 1 + bone_correct_parts[person, j] += 1 + + # an extra limb + total_parts[person] += 1 + # hip position + rhip_idx, lhip_idx = 2, 3 + pred_hip = (pred[min_n, rhip_idx, 0:3] + + pred[min_n, lhip_idx, 0:3]) / 2.0 + gt_hip = (gt[rhip_idx] + gt[lhip_idx]) / 2.0 + error_s = np.linalg.norm(pred_hip - gt_hip) + # bottom-head position + bh_idx = 12 + error_e = np.linalg.norm(pred[min_n, bh_idx, 0:3] - gt[bh_idx]) + limb_length = np.linalg.norm(gt_hip - gt[bh_idx]) + + if (error_e + error_s) / 2.0 <= alpha_error * limb_length: + correct_parts[person] += 1 + bone_correct_parts[person, -1] += 1 + + actor_pcp = correct_parts / (total_parts + 1e-8) * 100.0 + avg_pcp = np.mean(actor_pcp[:3]) + + stats_names = [ + f'Actor {person+1} Total PCP' for person in range(self.num_persons) + ] + ['pcp'] + stats_values = [*actor_pcp, avg_pcp] + + results = OrderedDict() + for name, value in zip(stats_names, stats_values): + results[name] = value + + for k, v in self.BONE_GROUP.items(): + cum_pcp = 0 + for person in range(self.num_persons): + new_k = f'Actor {person+1} ' + k + ' PCP' + pcp = np.sum( + bone_correct_parts[person, v], + axis=-1) / (total_parts[person] / + (len(self.LIMBS) + 1) * len(v) + 1e-8) * 100 + results[new_k] = pcp + cum_pcp += pcp + + new_k = 'Average ' + k + ' PCP' + results[new_k] = cum_pcp / self.num_persons + + return results + + @staticmethod + def coco2campus3D(coco_pose): + """transform coco order(our method output) 3d pose to campus dataset + order with interpolation. + + :param coco_pose: np.array with shape 17x3 + :return: 3D pose in campus order with shape 14x3 + """ + campus_pose = np.zeros((14, 3)) + coco2campus = np.array([16, 14, 12, 11, 13, 15, 10, 8, 6, 5, 7, 9]) + campus_pose[0:12] += coco_pose[coco2campus] + + # L and R shoulder + mid_sho = (coco_pose[5] + coco_pose[6]) / 2 + # middle of two ear + head_center = (coco_pose[3] + coco_pose[4]) / 2 + + # nose and head center + head_bottom = (mid_sho + head_center) / 2 + head_top = head_bottom + (head_center - head_bottom) * 2 + campus_pose[12] += head_bottom + campus_pose[13] += head_top + + return campus_pose + + @staticmethod + def _sort_and_unique_outputs(outputs, key='sample_id'): + """sort outputs and remove the repeated ones.""" + outputs = sorted(outputs, key=lambda x: x[key]) + num_outputs = len(outputs) + for i in range(num_outputs - 1, 0, -1): + if outputs[i][key] == outputs[i - 1][key]: + del outputs[i] + + return outputs diff --git a/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py b/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py new file mode 100644 index 0000000000..91b0072c59 --- /dev/null +++ b/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py @@ -0,0 +1,629 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy +# import glob +import json +import os.path as osp +import pickle +import random +import tempfile +import warnings +from collections import OrderedDict + +import mmcv +import numpy as np +import scipy.io as scio +from mmcv import Config + +from mmpose.core.camera import SimpleCamera +from mmpose.datasets.builder import DATASETS +from mmpose.datasets.datasets.base import Kpt3dMviewRgbImgDirectDataset + + +@DATASETS.register_module() +class Body3DMviewDirectShelfDataset(Kpt3dMviewRgbImgDirectDataset): + """Shelf dataset for direct multi-view human pose estimation. + + `3D Pictorial Structures for Multiple Human Pose Estimation' CVPR'2014 + More details can be found in the paper + ` + + The dataset loads both 2D and 3D annotations as well as camera parameters. + + Shelf keypoint indices:: + + 'Right-Ankle': 0, + 'Right-Knee': 1, + 'Right-Hip': 2, + 'Left-Hip': 3, + 'Left-Knee': 4, + 'Left-Ankle': 5, + 'Right-Wrist': 6, + 'Right-Elbow': 7, + 'Right-Shoulder': 8, + 'Left-Shoulder': 9, + 'Left-Elbow': 10, + 'Left-Wrist': 11, + 'Bottom-Head': 12, + 'Top-Head': 13, + + Args: + ann_file (str): Path to the annotation file. + img_prefix (str): Path to a directory where images are held. + Default: None. + data_cfg (dict): config + pipeline (list[dict | callable]): A sequence of data transforms. + dataset_info (DatasetInfo): A class containing all dataset info. + test_mode (bool): Store True when building test or + validation dataset. Default: False. + """ + ALLOWED_METRICS = {'pcp', '3dpcp'} + LIMBS = [[0, 1], [1, 2], [3, 4], [4, 5], [6, 7], [7, 8], [9, 10], [10, 11], + [12, 13]] + BONE_GROUP = OrderedDict([('Head', [8]), ('Torso', [9]), + ('Upper arms', [5, 6]), ('Lower arms', [4, 7]), + ('Upper legs', [1, 2]), ('Lower legs', [0, 3])]) + + def __init__(self, + ann_file, + img_prefix, + data_cfg, + pipeline, + dataset_info=None, + test_mode=False): + + if dataset_info is None: + warnings.warn( + 'dataset_info is missing. ' + 'Check https://github.com/open-mmlab/mmpose/pull/663 ' + 'for details.', DeprecationWarning) + cfg = Config.fromfile('configs/_base_/datasets/shelf.py') + dataset_info = cfg._cfg_dict['dataset_info'] + + super().__init__( + ann_file, + img_prefix, + data_cfg, + pipeline, + dataset_info=dataset_info, + test_mode=test_mode) + + self.load_config(data_cfg) + self.ann_info['use_different_joint_weights'] = data_cfg.get( + 'use_different_joint_weights', False) + + self.db_size = self.num_cameras * len( + self.frame_range + ) if self.test_mode else self.num_cameras * self.num_train_samples + print(f'=> load {self.db_size} samples') + + def load_config(self, data_cfg): + """Initialize dataset attributes according to the config. + + Override this method to set dataset specific attributes. + """ + self.num_joints = data_cfg['num_joints'] + self.cam_list = data_cfg['cam_list'] + self.num_cameras = data_cfg['num_cameras'] + assert self.num_cameras == len(self.cam_list) + self.need_camera_param = True + + self.subset = data_cfg.get('subset', 'train') + if self.subset == 'test': + self.frame_range = list(range(300, 601)) + else: + self.frame_range = list(range(0, 300)) + list(range(601, 3200)) + + self.width = data_cfg.get('width', 1032) + self.height = data_cfg.get('height', 776) + self.center = np.array((self.width / 2, self.height / 2), + dtype=np.float32) + self.scale = self._get_scale((self.width, self.height)) + + root_id = data_cfg.get('root_id', [11, 12]) + self.root_id = [root_id] if isinstance(root_id, int) else root_id + + self.max_nposes = data_cfg.get('max_nposes', 6) + self.min_nposes = data_cfg.get('min_nposes', 1) + self.num_train_samples = data_cfg.get('num_train_samples', 3000) + self.maximum_person = data_cfg.get('maximum_person', 10) + + self.cam_file = data_cfg.get( + 'cam_file', osp.join(self.img_prefix, 'calibration_shelf.json')) + + self.test_pose_db_file = data_cfg.get( + 'test_pose_db_file', + osp.join(self.img_prefix, 'pred_shelf_maskrcnn_hrnet_coco.pkl')) + + self.train_pose_db_file = data_cfg.get( + 'train_pose_db_file', + osp.join(self.img_prefix, 'panoptic_training_pose.pkl')) + + self.gt_pose_db_file = data_cfg.get( + 'gt_pose_db_file', osp.join(self.img_prefix, 'actorsGT.mat')) + + self._get_db() + + def _get_scale(self, raw_image_size): + heatmap_size = self.ann_info['heatmap_size'] + image_size = self.ann_info['image_size'] + assert heatmap_size[0][0] / heatmap_size[0][1] \ + == image_size[0] / image_size[1] + w, h = raw_image_size + w_resized, h_resized = image_size + if w / w_resized < h / h_resized: + w_pad = h / h_resized * w_resized + h_pad = h + else: + w_pad = w + h_pad = w / w_resized * h_resized + + scale = np.array([w_pad, h_pad], dtype=np.float32) + + return scale + + def _get_db(self): + """load related files.""" + assert osp.exists(self.cam_file), f'camera calibration file ' \ + f"{self.cam_file} doesn't exist, please check again" + with open(self.cam_file) as cfile: + calib = json.load(cfile) + self.cameras = self._get_cam(calib) + + assert osp.exists(self.train_pose_db_file), f'train_pose_db_file ' \ + f"{self.train_pose_db_file} doesn't exist, please check again" + with open(self.train_pose_db_file, 'rb') as pfile: + self.train_pose_db = pickle.load(pfile) + + assert osp.exists(self.test_pose_db_file), f'test_pose_db_file ' \ + f"{self.test_pose_db_file} doesn't exist, please check again" + with open(self.test_pose_db_file, 'rb') as pfile: + self.test_pose_db = pickle.load(pfile) + + assert osp.exists(self.gt_pose_db_file), f'gt_pose_db_file ' \ + f"{self.gt_pose_db_file} doesn't exist, please check again" + gt = scio.loadmat(self.gt_pose_db_file) + self.gt_pose_db = np.array(np.array( + gt['actor3D'].tolist()).tolist()).squeeze() + + self.num_persons = len(self.gt_pose_db) + + def _get_cam(self, calib): + """Get camera parameters. + + Returns: Camera parameters. + """ + cameras = {} + for id, cam in calib.items(): + sel_cam = {} + # note the transpose operation different from from VoxelPose + sel_cam['R'] = np.array(cam['R'], dtype=np.float32).T + sel_cam['T'] = np.array(cam['T'], dtype=np.float32) + + sel_cam['k'] = np.array(cam['k'], dtype=np.float32) + sel_cam['p'] = np.array(cam['p'], dtype=np.float32) + + sel_cam['f'] = [[cam['fx']], [cam['fy']]] + sel_cam['c'] = [[cam['cx']], [cam['cy']]] + + cameras[id] = sel_cam + + return cameras + + def __getitem__(self, idx): + """Get the sample given index.""" + + if self.test_mode: + results = self._prepare_test_sample(idx) + else: + results = self._prepare_train_sample(idx) + + return self.pipeline(results) + + def _prepare_test_sample(self, idx): + results = {} + fid = self.frame_range[idx] + + for cam_id, cam_param in self.cameras.items(): + image_file = osp.join(self.img_prefix, 'Camera' + cam_id, + 'img_{:06d}.png'.format(fid)) + + all_poses_3d = [] + all_poses_3d_vis = [] + all_poses_2d = [] + all_poses_2d_vis = [] + single_view_camera = SimpleCamera(cam_param) + + for person in range(self.num_persons): + pose3d = self.gt_pose_db[person][fid] * 1000.0 + if len(pose3d[0]) > 0: + all_poses_3d.append(pose3d) + all_poses_3d_vis.append(np.ones((self.num_joints, 3))) + + pose2d = single_view_camera.world_to_pixel(pose3d) + x_check = np.bitwise_and(pose2d[:, 0] >= 0, + pose2d[:, 0] <= self.width - 1) + y_check = np.bitwise_and(pose2d[:, 1] >= 0, + pose2d[:, 1] <= self.height - 1) + check = np.bitwise_and(x_check, y_check) + + joints_vis = np.ones((len(pose2d), 1)) + joints_vis[np.logical_not(check)] = 0 + all_poses_2d.append(pose2d) + all_poses_2d_vis.append( + np.repeat(np.reshape(joints_vis, (-1, 1)), 2, axis=1)) + + pred_index = '{}_{}'.format(cam_id, fid) + pred_poses = self.test_pose_db[pred_index] + preds = [] + for pose in pred_poses: + preds.append(np.array(pose['pred'])) + preds = np.array(preds) + + results[int(cam_id)] = { + 'image_file': image_file, + 'joints_3d': all_poses_3d, + 'joints_3d_visible': all_poses_3d_vis, + 'joints_2d': all_poses_2d, + 'joints_2d_visible': all_poses_2d_vis, + 'camera': cam_param, + 'joints': preds, + 'sample_id': idx * self.num_cameras + int(cam_id), + 'center': self.center, + 'scale': self.scale, + 'rotation': 0.0, + 'ann_info': self.ann_info + } + + return results + + def _prepare_train_sample(self, idx): + results = {} + nposes_ori = np.random.choice(range(self.min_nposes, self.max_nposes)) + select_poses = np.random.choice(self.train_pose_db, nposes_ori) + + joints_3d = np.array([p['pose'] for p in select_poses]) + joints_3d_vis = np.array([p['vis'] for p in select_poses]) + + bbox_list = [] + center_list = [] + for n in range(nposes_ori): + points = joints_3d[n][:, :2].copy() + + center = np.mean(points[self.root_id, :2], axis=0) + rot_rad = np.random.uniform(-180, 180) + + new_center = self.get_new_center(center_list) + new_xy = self.rotate_points(points, center, + rot_rad) - center + new_center + + loop_count = 0 + # here n will be at least 1 + while not self.isvalid( + self.calc_bbox(new_xy, joints_3d_vis[n]), bbox_list): + loop_count += 1 + if loop_count >= 100: + break + new_center = self.get_new_center(center_list) + new_xy = self.rotate_points(points, center, + rot_rad) - center + new_center + + if loop_count >= 100: + nposes = n + joints_3d = joints_3d[:n] + joints_3d_vis = joints_3d_vis[:n] + break + else: + nposes = nposes_ori + center_list.append(new_center) + bbox_list.append(self.calc_bbox(new_xy, joints_3d_vis[n])) + joints_3d[n][:, :2] = new_xy + + joints_3d_u = np.zeros((self.maximum_person, len(joints_3d[0]), 3)) + joints_3d_vis_u = np.zeros((self.maximum_person, len(joints_3d[0]), 3)) + for i in range(nposes): + joints_3d_u[i] = joints_3d[i][:, 0:3] + joints_3d_vis_u[i] = joints_3d_vis[i][:, 0:3] + + roots_3d = np.mean(joints_3d_u[:, self.root_id], axis=1) + + for cam_id, cam_param in self.cameras.items(): + joints = [] + joints_vis = [] + single_view_camera = SimpleCamera(cam_param) + for n in range(nposes): + pose2d = single_view_camera.world_to_pixel(joints_3d[n]) + x_check = np.bitwise_and(pose2d[:, 0] >= 0, + pose2d[:, 0] <= self.width - 1) + y_check = np.bitwise_and(pose2d[:, 1] >= 0, + pose2d[:, 1] <= self.height - 1) + check = np.bitwise_and(x_check, y_check) + vis = joints_3d_vis[n][:, 0] > 0 + vis[np.logical_not(check)] = 0 + + joints.append(pose2d) + joints_vis.append( + np.repeat(np.reshape(vis, (-1, 1)), 2, axis=1)) + + # make joints and joints_vis having same shape + joints_u = np.zeros((self.maximum_person, len(joints[0]), 2)) + joints_vis_u = np.zeros((self.maximum_person, len(joints[0]), 2)) + for i in range(nposes): + joints_u[i] = joints[i] + joints_vis_u[i] = joints_vis[i] + + results[int(cam_id)] = { + 'joints_3d': joints_3d_u, + 'joints_3d_visible': joints_3d_vis_u, + 'roots_3d': roots_3d, + 'joints': joints_u, + 'joints_visible': joints_vis_u, + 'camera': cam_param, + 'sample_id': idx * self.num_cameras + int(cam_id), + 'center': self.center, + 'scale': self.scale, + 'rotation': 0.0, + 'num_persons': nposes, + 'ann_info': self.ann_info + } + + return results + + def __len__(self): + """Get the size of the dataset.""" + if self.test_mode: + return len(self.frame_range) + else: + return self.num_train_samples + + @staticmethod + def get_new_center(center_list): + if len(center_list) == 0 or random.random() < 0.7: + new_center = np.array([ + np.random.uniform(-1000.0, 2000.0), + np.random.uniform(-1600.0, 1600.0) + ]) + else: + xy = center_list[np.random.choice(range(len(center_list)))] + new_center = xy + np.random.normal(500, 50, 2) * np.random.choice( + [1, -1], 2) + + return new_center + + @staticmethod + def rotate_points(points, center, rot_rad): + """ + :param points: N*2 + :param center: 2 + :param rot_rad: scalar + :return: N*2 + """ + rot_rad = rot_rad * np.pi / 180.0 + rotate_mat = np.array([[np.cos(rot_rad), -np.sin(rot_rad)], + [np.sin(rot_rad), + np.cos(rot_rad)]]) + center = center.reshape(2, 1) + points = points.T + points = rotate_mat.dot(points - center) + center + + return points.T + + @staticmethod + def isvalid(bbox, bbox_list): + if len(bbox_list) == 0: + return True + + bbox_list = np.array(bbox_list) + x0 = np.maximum(bbox[0], bbox_list[:, 0]) + y0 = np.maximum(bbox[1], bbox_list[:, 1]) + x1 = np.minimum(bbox[2], bbox_list[:, 2]) + y1 = np.minimum(bbox[3], bbox_list[:, 3]) + + intersection = np.maximum(0, (x1 - x0) * (y1 - y0)) + area = (bbox[2] - bbox[0]) * (bbox[3] - bbox[1]) + area_list = (bbox_list[:, 2] - bbox_list[:, 0]) * ( + bbox_list[:, 3] - bbox_list[:, 1]) + iou_list = intersection / (area + area_list - intersection) + + return np.max(iou_list) < 0.01 + + @staticmethod + def calc_bbox(pose, pose_vis): + index = pose_vis[:, 0] > 0 + bbox = [ + np.min(pose[index, 0]), + np.min(pose[index, 1]), + np.max(pose[index, 0]), + np.max(pose[index, 1]) + ] + + return np.array(bbox) + + def evaluate(self, + results, + res_folder=None, + metric='pcp', + recall_threshold=500, + alpha_error=0.5, + alpha_head=0.75, + **kwargs): + """ + Args: + results (list[dict]): Testing results containing the following + items: + - pose_3d (np.ndarray): predicted 3D human pose + - sample_id (np.ndarray): sample id of a frame. + res_folder (str, optional): The folder to save the testing + results. If not specified, a temp folder will be created. + Default: None. + metric (str | list[str]): Metric to be performed. + Defaults: 'pcp'. + recall_threshold: threshold for calculating recall. + alpha_error: coefficient when calculating correct parts. + alpha_head: coefficient for conputing head keypoints position + when converting coco poses to shelf poses + **kwargs: + + Returns: + + """ + pose_3ds = np.concatenate([result['pose_3d'] for result in results], + axis=0) + sample_ids = [] + for result in results: + sample_ids.extend(result['sample_id']) + _results = [ + dict(sample_id=sample_id, pose_3d=pose_3d) + for (sample_id, pose_3d) in zip(sample_ids, pose_3ds) + ] + _results = self._sort_and_unique_outputs(_results, key='sample_id') + + metrics = metric if isinstance(metric, list) else [metric] + for _metric in metrics: + if _metric not in self.ALLOWED_METRICS: + raise ValueError( + f'Unsupported metric "{_metric}"' + f'Supported metrics are {self.ALLOWED_METRICS}') + + if res_folder is not None: + tmp_folder = None + res_file = osp.join(res_folder, 'result_keypoints.json') + else: + tmp_folder = tempfile.TemporaryDirectory() + res_file = osp.join(tmp_folder.name, 'result_keypoints.json') + mmcv.dump(_results, res_file) + + gt_num = self.db_size // self.num_cameras + assert len( + _results) == gt_num, f'number mismatch: {len(_results)}, {gt_num}' + + match_gt = 0 + total_gt = 0 + + correct_parts = np.zeros(self.num_persons) + total_parts = np.zeros(self.num_persons) + bone_correct_parts = np.zeros((self.num_persons, len(self.LIMBS) + 1)) + + for i, fid in enumerate(self.frame_range): + pred_coco = pose_3ds[i].copy() + pred_coco = pred_coco[pred_coco[:, 0, 3] >= 0, :, :3] + + pred = np.stack([ + self.coco2shelf3D(p, alpha_head) + for p in copy.deepcopy(pred_coco[:, :, :3]) + ]) + + for person in range(self.num_persons): + gt = self.gt_pose_db[person][fid] * 1000.0 + if len(gt[0]) == 0: + continue + + mpjpes = np.mean( + np.sqrt(np.sum((gt[np.newaxis] - pred)**2, axis=-1)), + axis=-1) + min_n = np.argmin(mpjpes) + min_mpjpe = np.min(mpjpes) + if min_mpjpe < recall_threshold: + match_gt += 1 + total_gt += 1 + + for j, k in enumerate(self.LIMBS): + total_parts[person] += 1 + error_s = np.linalg.norm(pred[min_n, k[0], 0:3] - gt[k[0]]) + error_e = np.linalg.norm(pred[min_n, k[1], 0:3] - gt[k[1]]) + limb_length = np.linalg.norm(gt[k[0]] - gt[k[1]]) + + if (error_s + error_e) / 2.0 <= alpha_error * limb_length: + correct_parts[person] += 1 + bone_correct_parts[person, j] += 1 + + # an extra limb + total_parts[person] += 1 + # hip position + rhip_idx, lhip_idx = 2, 3 + pred_hip = (pred[min_n, rhip_idx, 0:3] + + pred[min_n, lhip_idx, 0:3]) / 2.0 + gt_hip = (gt[rhip_idx] + gt[lhip_idx]) / 2.0 + error_s = np.linalg.norm(pred_hip - gt_hip) + # bottom-head position + bh_idx = 12 + error_e = np.linalg.norm(pred[min_n, bh_idx, 0:3] - gt[bh_idx]) + limb_length = np.linalg.norm(gt_hip - gt[bh_idx]) + + if (error_e + error_s) / 2.0 <= alpha_error * limb_length: + correct_parts[person] += 1 + bone_correct_parts[person, -1] += 1 + + actor_pcp = correct_parts / (total_parts + 1e-8) * 100.0 + avg_pcp = np.mean(actor_pcp[:3]) + + stats_names = [ + f'Actor {person+1} Total PCP' for person in range(self.num_persons) + ] + ['pcp'] + stats_values = [*actor_pcp, avg_pcp] + + results = OrderedDict() + for name, value in zip(stats_names, stats_values): + results[name] = value + + for k, v in self.BONE_GROUP.items(): + cum_pcp = 0 + for person in range(self.num_persons): + new_k = f'Actor {person+1} ' + k + ' PCP' + pcp = np.sum( + bone_correct_parts[person, v], + axis=-1) / (total_parts[person] / + (len(self.LIMBS) + 1) * len(v) + 1e-8) * 100 + results[new_k] = pcp + cum_pcp += pcp + + new_k = 'Average ' + k + ' PCP' + results[new_k] = cum_pcp / self.num_persons + + return results + + @staticmethod + def coco2shelf3D(coco_pose, alpha): + """transform coco order(our method output) 3d pose to shelf dataset + order with interpolation. + + :param coco_pose: np.array with shape 17x3 + :return: 3D pose in shelf order with shape 14x3 + """ + shelf_pose = np.zeros((14, 3)) + coco2shelf = np.array([16, 14, 12, 11, 13, 15, 10, 8, 6, 5, 7, 9]) + shelf_pose[0:12] += coco_pose[coco2shelf] + + # L and R shoulder + mid_sho = (coco_pose[5] + coco_pose[6]) / 2 + # middle of two ear + head_center = (coco_pose[3] + coco_pose[4]) / 2 + + # nose and head center + head_bottom = (mid_sho + head_center) / 2 + head_top = head_bottom + (head_center - head_bottom) * 2 + + # Use middle of shoulder to init + shelf_pose[12] = (shelf_pose[8] + shelf_pose[9]) / 2 + # use nose to init + shelf_pose[13] = coco_pose[0] + + shelf_pose[13] = shelf_pose[12] + ( + shelf_pose[13] - shelf_pose[12]) * np.array([0.75, 0.75, 1.5]) + shelf_pose[12] = shelf_pose[12] + ( + coco_pose[0] - shelf_pose[12]) * np.array([0.5, 0.5, 0.5]) + + shelf_pose[13] = shelf_pose[13] * alpha + head_top * (1 - alpha) + shelf_pose[12] = shelf_pose[12] * alpha + head_bottom * (1 - alpha) + + return shelf_pose + + @staticmethod + def _sort_and_unique_outputs(outputs, key='sample_id'): + """sort outputs and remove the repeated ones.""" + outputs = sorted(outputs, key=lambda x: x[key]) + num_outputs = len(outputs) + for i in range(num_outputs - 1, 0, -1): + if outputs[i][key] == outputs[i - 1][key]: + del outputs[i] + + return outputs diff --git a/mmpose/datasets/pipelines/pose3d_transform.py b/mmpose/datasets/pipelines/pose3d_transform.py index 124937861f..765442bbfc 100644 --- a/mmpose/datasets/pipelines/pose3d_transform.py +++ b/mmpose/datasets/pipelines/pose3d_transform.py @@ -1,5 +1,6 @@ # Copyright (c) OpenMMLab. All rights reserved. import copy +import random import mmcv import numpy as np @@ -7,7 +8,8 @@ from mmcv.utils import build_from_cfg from mmpose.core.camera import CAMERAS -from mmpose.core.post_processing import fliplr_regression +from mmpose.core.post_processing import (affine_transform, fliplr_regression, + get_affine_transform) from mmpose.datasets.builder import PIPELINES @@ -603,10 +605,12 @@ def __call__(self, results): for n in range(num_people): for idx, joint_id in enumerate(joint_indices): - mu_x = joints_3d[n][joint_id][0] - mu_y = joints_3d[n][joint_id][1] - mu_z = joints_3d[n][joint_id][2] - vis = joints_3d_visible[n][joint_id][0] + assert joints_3d.shape[2] == 3 + + mu_x = np.mean(joints_3d[n][joint_id, 0]) + mu_y = np.mean(joints_3d[n][joint_id, 1]) + mu_z = np.mean(joints_3d[n][joint_id, 2]) + vis = np.mean(joints_3d_visible[n][joint_id, 0]) if vis < 1: continue i_x = [ @@ -641,3 +645,200 @@ def __call__(self, results): results['targets_3d'] = target return results + + +@PIPELINES.register_module() +class AffineJoints: + """Apply affine transformation to joints coordinates. + + Args: + item (str): The name of the joints to apply affine. + visible_item (str): The name of the visibility item. + + Required keys: + item, visible_item(optional) + + Modified keys: + item, visible_item(optional) + """ + + def __init__(self, item='joints', visible_item=None): + self.item = item + self.visible_item = visible_item + + def __call__(self, results): + """Perform random affine transformation to joints coordinates.""" + + c = results['center'] + s = results['scale'] / 200.0 + r = results['rotation'] + image_size = results['ann_info']['image_size'] + + assert self.item in results + joints = results[self.item] + + if self.visible_item is not None: + assert self.visible_item in results + joints_vis = results[self.visible_item] + else: + joints_vis = [np.ones_like(joints[0]) for _ in range(len(joints))] + + trans = get_affine_transform(c, s, r, image_size) + nposes = len(joints) + for n in range(nposes): + for i in range(len(joints[0])): + if joints_vis[n][i, 0] > 0.0: + joints[n][i, + 0:2] = affine_transform(joints[n][i, 0:2], trans) + if (np.min(joints[n][i, :2]) < 0 + or joints[n][i, 0] >= image_size[0] + or joints[n][i, 1] >= image_size[1]): + joints_vis[n][i, :] = 0 + + results[self.item] = joints + if self.visible_item is not None: + results[self.visible_item] = joints_vis + + return results + + +@PIPELINES.register_module() +class GenerateInputHeatmaps: + """Generate 2D input heatmaps for multi-camera heatmaps when the 2D model + is not available. + + Required keys: 'joints' + Modified keys: 'input_heatmaps' + + Args: + sigma (int): Sigma of heatmap gaussian (mm). + base_size (int): the base size of human + target_type (str): type of target heatmap, only support 'gaussian' now + """ + + def __init__(self, + item='joints', + visible_item=None, + obscured=0.0, + from_pred=True, + sigma=3, + scale=None, + base_size=96, + target_type='gaussian', + heatmap_cfg=None): + self.item = item + self.visible_item = visible_item + self.obscured = obscured + self.from_pred = from_pred + self.sigma = sigma + self.scale = scale + self.base_size = base_size + self.target_type = target_type + self.heatmap_cfg = heatmap_cfg + + def _compute_human_scale(self, pose, joints_vis): + idx = joints_vis[:, 0] == 1 + if np.sum(idx) == 0: + return 0 + minx, maxx = np.min(pose[idx, 0]), np.max(pose[idx, 0]) + miny, maxy = np.min(pose[idx, 1]), np.max(pose[idx, 1]) + + return np.clip( + np.maximum(maxy - miny, maxx - minx)**2, (self.base_size / 2)**2, + (self.base_size * 2)**2) + + def __call__(self, results): + assert self.target_type == 'gaussian', 'Only support gaussian map now' + assert results['ann_info'][ + 'num_scales'] == 1, 'Only support one scale now' + heatmap_size = results['ann_info']['heatmap_size'][0] + + num_joints = results['ann_info']['num_joints'] + image_size = results['ann_info']['image_size'] + + joints = results[self.item] + + if self.visible_item is not None: + assert self.visible_item in results + joints_vis = results[self.visible_item] + else: + joints_vis = [np.ones_like(joints[0]) for _ in range(len(joints))] + + nposes = len(joints) + target = np.zeros((num_joints, heatmap_size[1], heatmap_size[0]), + dtype=np.float32) + feat_stride = image_size / heatmap_size + + for n in range(nposes): + if random.random() < self.obscured: + continue + + human_scale = 2 * self._compute_human_scale( + joints[n][:, 0:2] / feat_stride, joints_vis[n]) + if human_scale == 0: + continue + cur_sigma = self.sigma * np.sqrt( + (human_scale / (self.base_size**2))) + tmp_size = cur_sigma * 3 + for joint_id in range(num_joints): + feat_stride = image_size / heatmap_size + mu_x = int(joints[n][joint_id][0] / feat_stride[0]) + mu_y = int(joints[n][joint_id][1] / feat_stride[1]) + + ul = [int(mu_x - tmp_size), int(mu_y - tmp_size)] + br = [int(mu_x + tmp_size + 1), int(mu_y + tmp_size + 1)] + if ul[0] >= heatmap_size[0] or \ + ul[1] >= heatmap_size[1] \ + or br[0] < 0 or br[1] < 0: + continue + + size = 2 * tmp_size + 1 + x = np.arange(0, size, 1, np.float32) + y = x[:, np.newaxis] + x0 = y0 = size // 2 + + # determine the value of scale + if self.from_pred: + if self.scale is None: + scale = joints[n][joint_id][2] if len( + joints[n][joint_id]) == 3 else 1.0 + else: + scale = self.scale + else: + if self.heatmap_cfg is None: + scale = self.scale + else: + base_scale = self.heatmap_cfg['base_scale'] + offset = self.heatmap_cfg['offset'] + thr = self.heatmap_cfg['threshold'] + scale = (base_scale + np.random.randn(1) * offset + ) if random.random() < thr else self.scale + + for cfg in self.heatmap_cfg['extra']: + if joint_id in cfg['joint_ids']: + scale = scale * cfg[ + 'scale_factor'] if random.random( + ) < cfg['threshold'] else scale + + g = np.exp(-((x - x0)**2 + (y - y0)**2) / + (2 * cur_sigma**2)) * scale + + # usable gaussian range + g_x = max(0, 0 - ul[0]), min(br[0], heatmap_size[0]) - ul[0] + g_y = max(0, -ul[1]), min(br[1], heatmap_size[1]) - ul[1] + + # Image range + img_x = max(0, ul[0]), min(br[0], heatmap_size[0]) + img_y = max(0, ul[1]), min(br[1], heatmap_size[1]) + + target[joint_id][img_y[0]:img_y[1], + img_x[0]:img_x[1]] = np.maximum( + target[joint_id][img_y[0]:img_y[1], + img_x[0]:img_x[1]], + g[g_y[0]:g_y[1], g_x[0]:g_x[1]]) + target = np.clip(target, 0, 1) + + # target can be extended to multi-scale, + # if results['ann_info']['num_scales'] > 1 + results['input_heatmaps'] = [target] + return results From 69162b3f8165fb9ee7eae44c71bf483983a9ee7e Mon Sep 17 00:00:00 2001 From: liqikai Date: Tue, 8 Mar 2022 18:07:25 +0800 Subject: [PATCH 02/19] fix config typo --- .../campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py index 688c2f4661..473163259e 100644 --- a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py @@ -48,9 +48,7 @@ subset='train', width=360, height=288, - root_id=[2, 3], - coco_lhip_idx=11, - coco_rhip_idx=12, + root_id=[11, 12], max_nposes=10, min_nposes=1, num_train_sampels=3000, From e4c1290f4fbb0e1b201e971bdec2979b17d3c180 Mon Sep 17 00:00:00 2001 From: liqikai Date: Fri, 11 Mar 2022 15:27:07 +0800 Subject: [PATCH 03/19] Add unittests for dataset and pipeline and modify configs --- ...ose_prn32x32x32_cpn80x80x20_campus_cam3.py | 215 ++++++++++++++ ...ose_prn64x64x64_cpn80x80x20_campus_cam3.py | 12 +- ...pose_prn32x32x32_cpn48x48x12_shelf_cam5.py | 213 ++++++++++++++ ...pose_prn64x64x64_cpn80x80x20_shelf_cam5.py | 4 +- .../kpt_3d_mview_rgb_img_direct_dataset.py | 1 - .../body3d_mview_direct_campus_dataset.py | 21 +- .../body3d_mview_direct_shelf_dataset.py | 18 +- tests/data/campus/actorsGT.mat | Bin 0 -> 4920 bytes tests/data/campus/calibration_campus.json | 161 +++++++++++ tests/data/campus/panoptic_training_pose.pkl | Bin 0 -> 1827 bytes .../pred_campus_maskrcnn_hrnet_coco.pkl | Bin 0 -> 231 bytes tests/data/shelf/actorsGT.mat | Bin 0 -> 13968 bytes tests/data/shelf/calibration_shelf.json | 267 ++++++++++++++++++ tests/data/shelf/panoptic_training_pose.pkl | Bin 0 -> 1827 bytes .../shelf/pred_shelf_maskrcnn_hrnet_coco.pkl | Bin 0 -> 27826 bytes tests/test_datasets/test_body3d_dataset.py | 180 +++++++++++- tests/test_pipelines/test_pose3d_transform.py | 111 ++++++++ 17 files changed, 1171 insertions(+), 32 deletions(-) create mode 100644 configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3.py create mode 100644 configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5.py create mode 100644 tests/data/campus/actorsGT.mat create mode 100644 tests/data/campus/calibration_campus.json create mode 100644 tests/data/campus/panoptic_training_pose.pkl create mode 100644 tests/data/campus/pred_campus_maskrcnn_hrnet_coco.pkl create mode 100644 tests/data/shelf/actorsGT.mat create mode 100644 tests/data/shelf/calibration_shelf.json create mode 100644 tests/data/shelf/panoptic_training_pose.pkl create mode 100644 tests/data/shelf/pred_shelf_maskrcnn_hrnet_coco.pkl diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3.py b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3.py new file mode 100644 index 0000000000..83a4b241f5 --- /dev/null +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3.py @@ -0,0 +1,215 @@ +_base_ = [ + '../../../../_base_/default_runtime.py', + '../../../../_base_/datasets/campus.py' +] +checkpoint_config = dict(interval=1) +evaluation = dict( + interval=1, metric='pcp', save_best='pcp', recall_threshold=500) + +optimizer = dict( + type='Adam', + lr=0.0001, +) +optimizer_config = dict(grad_clip=None) + +# learning policy +lr_config = dict( + policy='step', + warmup='linear', + warmup_iters=500, + warmup_ratio=0.001, + step=[10, 20]) +total_epochs = 30 +log_config = dict( + interval=50, hooks=[ + dict(type='TextLoggerHook'), + ]) + +space_size = [12000.0, 12000.0, 2000.0] +space_center = [3000.0, 4500.0, 1000.0] +cube_size = [80, 80, 20] +sub_space_size = [2000.0, 2000.0, 2000.0] +sub_cube_size = [32, 32, 32] +image_size = [800, 640] +heatmap_size = [200, 160] + +num_joints = 17 + +data_root = 'data/campus' +train_data_cfg = dict( + space_size=space_size, + space_center=space_center, + cube_size=cube_size, + image_size=image_size, + heatmap_size=[heatmap_size], + num_joints=num_joints, + cam_list=[0, 1, 2], + num_cameras=3, + frame_range=list(range(0, 350)) + list(range(471, 650)) + + list(range(751, 2000)), + width=360, + height=288, + root_id=[11, 12], + max_nposes=10, + min_nposes=1, + num_train_samples=3000, + maximum_person=10, + cam_file=f'{data_root}/calibration_campus.json', + test_pose_db_file=f'{data_root}/pred_campus_maskrcnn_hrnet_coco.pkl', + train_pose_db_file=f'{data_root}/panoptic_training_pose.pkl', + gt_pose_db_file=f'{data_root}/actorsGT.mat', +) + +test_data_cfg = train_data_cfg.copy() +test_data_cfg.update( + dict(frame_range=list(range(350, 471)) + list(range(650, 751)))) +# model settings +model = dict( + type='DetectAndRegress', + backbone=None, + pretrained=None, + human_detector=dict( + type='VoxelCenterDetector', + image_size=image_size, + heatmap_size=heatmap_size, + space_size=space_size, + cube_size=cube_size, + space_center=space_center, + center_net=dict( + type='V2VNet', input_channels=num_joints, output_channels=1), + center_head=dict( + type='CuboidCenterHead', + space_size=space_size, + space_center=space_center, + cube_size=cube_size, + max_num=10, + max_pool_kernel=3), + train_cfg=dict(dist_threshold=500.0), + test_cfg=dict(center_threshold=0.1), + ), + pose_regressor=dict( + type='VoxelSinglePose', + image_size=image_size, + heatmap_size=heatmap_size, + sub_space_size=sub_space_size, + sub_cube_size=sub_cube_size, + num_joints=num_joints, + pose_net=dict( + type='V2VNet', + input_channels=num_joints, + output_channels=num_joints), + pose_head=dict(type='CuboidPoseHead', beta=100.0))) + +train_pipeline = [ + dict( + type='MultiItemProcess', + pipeline=[ + dict( + type='AffineJoints', + item='joints', + visible_item='joints_visible'), + dict( + type='GenerateInputHeatmaps', + item='joints', + visible_item='joints_visible', + obscured=0.0, + from_pred=False, + sigma=3, + scale=1.0, + base_size=96, + target_type='gaussian', + heatmap_cfg=dict( + base_scale=0.9, + offset=0.03, + threshold=0.6, + extra=[ + dict( + joint_ids=[7, 8], scale_factor=0.5, threshold=0.1), + dict( + joint_ids=[9, 10], + scale_factor=0.2, + threshold=0.1, + ), + dict( + joint_ids=[ + 0, 1, 2, 3, 4, 5, 6, 11, 12, 13, 14, 15, 16 + ], + scale_factor=0.5, + threshold=0.05) + ])) + ]), + dict( + type='DiscardDuplicatedItems', + keys_list=[ + 'joints_3d', 'joints_3d_visible', 'ann_info', 'roots_3d', + 'num_persons', 'sample_id' + ]), + dict( + type='GenerateVoxel3DHeatmapTarget', + sigma=200.0, + joint_indices=[[11, 12]]), + dict( + type='Collect', + keys=['sample_id', 'input_heatmaps', 'targets_3d'], + meta_keys=[ + 'num_persons', 'joints_3d', 'camera', 'center', 'scale', + 'joints_3d_visible', 'roots_3d', 'sample_id' + ]), +] + +val_pipeline = [ + dict( + type='MultiItemProcess', + pipeline=[ + dict(type='AffineJoints', item='joints'), + dict( + type='GenerateInputHeatmaps', + item='joints', + from_pred=True, + scale=1.0, + sigma=3, + base_size=96, + target_type='gaussian'), + ]), + dict( + type='DiscardDuplicatedItems', + keys_list=[ + 'joints_3d', 'joints_3d_visible', 'joints_2d', 'joints_2d_visible', + 'ann_info', 'sample_id' + ]), + dict( + type='Collect', + keys=['sample_id', 'input_heatmaps'], + meta_keys=['sample_id', 'camera', 'center', 'scale']), +] + +test_pipeline = val_pipeline + +data_root = 'data/campus' +data = dict( + samples_per_gpu=2, + workers_per_gpu=4, + val_dataloader=dict(samples_per_gpu=4), + test_dataloader=dict(samples_per_gpu=4), + train=dict( + type='Body3DMviewDirectCampusDataset', + ann_file=None, + img_prefix=data_root, + data_cfg=train_data_cfg, + pipeline=train_pipeline, + dataset_info={{_base_.dataset_info}}), + val=dict( + type='Body3DMviewDirectCampusDataset', + ann_file=None, + img_prefix=data_root, + data_cfg=test_data_cfg, + pipeline=val_pipeline, + dataset_info={{_base_.dataset_info}}), + test=dict( + type='Body3DMviewDirectCampusDataset', + ann_file=None, + img_prefix=data_root, + data_cfg=test_data_cfg, + pipeline=test_pipeline, + dataset_info={{_base_.dataset_info}}), +) diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py index 473163259e..f44eab55d2 100644 --- a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py @@ -45,13 +45,14 @@ num_joints=num_joints, cam_list=[0, 1, 2], num_cameras=3, - subset='train', + frame_range=list(range(0, 350)) + list(range(471, 650)) + + list(range(751, 2000)), width=360, height=288, root_id=[11, 12], max_nposes=10, min_nposes=1, - num_train_sampels=3000, + num_train_samples=3000, maximum_person=10, cam_file=f'{data_root}/calibration_campus.json', test_pose_db_file=f'{data_root}/pred_campus_maskrcnn_hrnet_coco.pkl', @@ -60,7 +61,8 @@ ) test_data_cfg = train_data_cfg.copy() -test_data_cfg.update(dict(subset='test')) +test_data_cfg.update( + dict(frame_range=list(range(350, 471)) + list(range(650, 751)))) # model settings model = dict( @@ -188,8 +190,8 @@ data = dict( samples_per_gpu=1, workers_per_gpu=4, - val_dataloader=dict(samples_per_gpu=1), - test_dataloader=dict(samples_per_gpu=1), + val_dataloader=dict(samples_per_gpu=4), + test_dataloader=dict(samples_per_gpu=4), train=dict( type='Body3DMviewDirectCampusDataset', ann_file=None, diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5.py b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5.py new file mode 100644 index 0000000000..ea8c8f0672 --- /dev/null +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5.py @@ -0,0 +1,213 @@ +_base_ = [ + '../../../../_base_/default_runtime.py', + '../../../../_base_/datasets/shelf.py' +] +checkpoint_config = dict(interval=1) +evaluation = dict( + interval=1, metric='pcp', save_best='pcp', recall_threshold=500) + +optimizer = dict( + type='Adam', + lr=0.0001, +) +optimizer_config = dict(grad_clip=None) + +# learning rate policy +lr_config = dict( + policy='step', + warmup='linear', + warmup_iters=500, + warmup_ratio=0.001, + step=[10, 20]) +total_epochs = 30 +log_config = dict( + interval=50, hooks=[ + dict(type='TextLoggerHook'), + ]) + +space_size = [8000, 8000, 2000] +space_center = [450, -320, 800] +cube_size = [48, 48, 12] +sub_space_size = [2000, 2000, 2000] +sub_cube_size = [32, 32, 32] +image_size = [800, 608] +heatmap_size = [200, 152] + +num_joints = 17 + +data_root = 'data/shelf' +train_data_cfg = dict( + space_size=space_size, + space_center=space_center, + cube_size=cube_size, + image_size=image_size, + heatmap_size=[heatmap_size], + num_joints=num_joints, + cam_list=[0, 1, 2, 3, 4], + num_cameras=5, + frame_range=list(range(0, 300)) + list(range(601, 3200)), + width=1032, + height=776, + root_id=[11, 12], + max_nposes=6, + min_nposes=1, + num_train_samples=3000, + maximum_person=10, + cam_file=f'{data_root}/calibration_shelf.json', + test_pose_db_file=f'{data_root}/pred_shelf_maskrcnn_hrnet_coco.pkl', + train_pose_db_file=f'{data_root}/panoptic_training_pose.pkl', + gt_pose_db_file=f'{data_root}/actorsGT.mat', +) + +test_data_cfg = train_data_cfg.copy() +test_data_cfg.update(dict(frame_range=list(range(300, 601)))) + +# model settings +model = dict( + type='DetectAndRegress', + backbone=None, + pretrained=None, + human_detector=dict( + type='VoxelCenterDetector', + image_size=image_size, + heatmap_size=heatmap_size, + space_size=space_size, + cube_size=cube_size, + space_center=space_center, + center_net=dict( + type='V2VNet', input_channels=num_joints, output_channels=1), + center_head=dict( + type='CuboidCenterHead', + space_size=space_size, + space_center=space_center, + cube_size=cube_size, + max_num=10, + max_pool_kernel=3), + train_cfg=dict(dist_threshold=500.0), + test_cfg=dict(center_threshold=0.1), + ), + pose_regressor=dict( + type='VoxelSinglePose', + image_size=image_size, + heatmap_size=heatmap_size, + sub_space_size=sub_space_size, + sub_cube_size=sub_cube_size, + num_joints=num_joints, + pose_net=dict( + type='V2VNet', + input_channels=num_joints, + output_channels=num_joints), + pose_head=dict(type='CuboidPoseHead', beta=100.0))) + +train_pipeline = [ + dict( + type='MultiItemProcess', + pipeline=[ + dict( + type='AffineJoints', + item='joints', + visible_item='joints_visible'), + dict( + type='GenerateInputHeatmaps', + item='joints', + visible_item='joints_visible', + obscured=0.05, + from_pred=False, + sigma=3, + scale=1.0, + base_size=96, + target_type='gaussian', + heatmap_cfg=dict( + base_scale=0.9, + offset=0.03, + threshold=0.6, + extra=[ + dict( + joint_ids=[7, 8, 13, 14], + scale_factor=0.5, + threshold=0.1), + dict( + joint_ids=[9, 10, 15, 16], + scale_factor=0.2, + threshold=0.1, + ), + dict( + joint_ids=[0, 1, 2, 3, 4, 5, 6, 11, 12], + scale_factor=0.5, + threshold=0.05) + ])), + ]), + dict( + type='DiscardDuplicatedItems', + keys_list=[ + 'joints_3d', 'joints_3d_visible', 'ann_info', 'roots_3d', + 'num_persons', 'sample_id' + ]), + dict( + type='GenerateVoxel3DHeatmapTarget', + sigma=200.0, + joint_indices=[[11, 12]]), + dict( + type='Collect', + keys=['sample_id', 'input_heatmaps', 'targets_3d'], + meta_keys=[ + 'num_persons', 'joints_3d', 'camera', 'center', 'scale', + 'joints_3d_visible', 'roots_3d', 'sample_id' + ]), +] + +val_pipeline = [ + dict( + type='MultiItemProcess', + pipeline=[ + dict(type='AffineJoints', item='joints'), + dict( + type='GenerateInputHeatmaps', + item='joints', + from_pred=True, + sigma=3, + base_size=96, + target_type='gaussian'), + ]), + dict( + type='DiscardDuplicatedItems', + keys_list=[ + 'joints_3d', 'joints_3d_visible', 'joints_2d', 'joints_2d_visible', + 'ann_info', 'sample_id' + ]), + dict( + type='Collect', + keys=['sample_id', 'input_heatmaps'], + meta_keys=['sample_id', 'camera', 'center', 'scale']), +] + +test_pipeline = val_pipeline + +data_root = 'data/shelf' +data = dict( + samples_per_gpu=2, + workers_per_gpu=4, + val_dataloader=dict(samples_per_gpu=4), + test_dataloader=dict(samples_per_gpu=4), + train=dict( + type='Body3DMviewDirectShelfDataset', + ann_file=None, + img_prefix=data_root, + data_cfg=train_data_cfg, + pipeline=train_pipeline, + dataset_info={{_base_.dataset_info}}), + val=dict( + type='Body3DMviewDirectShelfDataset', + ann_file=None, + img_prefix=data_root, + data_cfg=test_data_cfg, + pipeline=val_pipeline, + dataset_info={{_base_.dataset_info}}), + test=dict( + type='Body3DMviewDirectShelfDataset', + ann_file=None, + img_prefix=data_root, + data_cfg=test_data_cfg, + pipeline=test_pipeline, + dataset_info={{_base_.dataset_info}}), +) diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py index d1b55a1ce0..5b896e2f10 100644 --- a/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py @@ -45,7 +45,7 @@ num_joints=num_joints, cam_list=[0, 1, 2, 3, 4], num_cameras=5, - subset='train', + frame_range=list(range(0, 300)) + list(range(601, 3200)), width=1032, height=776, root_id=[11, 12], @@ -60,7 +60,7 @@ ) test_data_cfg = train_data_cfg.copy() -test_data_cfg.update(dict(subset='test')) +test_data_cfg.update(dict(frame_range=list(range(300, 601)))) # model settings model = dict( diff --git a/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py b/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py index 5e79be1139..ca86f9e092 100644 --- a/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py +++ b/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py @@ -61,7 +61,6 @@ def __init__(self, dataset_info = DatasetInfo(dataset_info) - # assert self.ann_info['num_joints'] <= dataset_info.keypoint_num self.ann_info['flip_pairs'] = dataset_info.flip_pairs self.ann_info['num_scales'] = 1 self.ann_info['flip_index'] = dataset_info.flip_index diff --git a/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py b/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py index be9b9e3690..73d0b0cea6 100644 --- a/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py +++ b/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py @@ -93,7 +93,7 @@ def __init__(self, self.db_size = self.num_cameras * len( self.frame_range - ) if self.test_mode else self.num_cameras * self.num_train_sampels + ) if self.test_mode else self.num_cameras * self.num_train_samples print(f'=> load {self.db_size} samples') def load_config(self, data_cfg): @@ -107,12 +107,7 @@ def load_config(self, data_cfg): assert self.num_cameras == len(self.cam_list) self.need_camera_param = True - self.subset = data_cfg.get('subset', 'train') - if self.subset == 'test': - self.frame_range = list(range(350, 471)) + list(range(650, 751)) - else: - self.frame_range = list(range(0, 350)) + list(range( - 471, 650)) + list(range(751, 2000)) + self.frame_range = data_cfg['frame_range'] self.width = data_cfg.get('width', 360) self.height = data_cfg.get('height', 288) @@ -125,7 +120,7 @@ def load_config(self, data_cfg): self.max_nposes = data_cfg.get('max_nposes', 10) self.min_nposes = data_cfg.get('min_nposes', 1) - self.num_train_sampels = data_cfg.get('num_train_sampels', 3000) + self.num_train_samples = data_cfg.get('num_train_samples', 3000) self.maximum_person = data_cfg.get('maximum_person', 10) self.cam_file = data_cfg.get( @@ -228,8 +223,6 @@ def _prepare_test_sample(self, idx): image_file = osp.join( self.img_prefix, 'Camera' + cam_id, 'campus4-c{0}-{1:05d}.png'.format(cam_id, fid)) - assert osp.exists(image_file), f'the image file {image_file}' \ - f'does not exist, please check it again' all_poses_3d = [] all_poses_3d_vis = [] @@ -378,7 +371,7 @@ def __len__(self): if self.test_mode: return len(self.frame_range) else: - return self.num_train_sampels + return self.num_train_samples @staticmethod def get_new_center(center_list): @@ -600,8 +593,10 @@ def coco2campus3D(coco_pose): """transform coco order(our method output) 3d pose to campus dataset order with interpolation. - :param coco_pose: np.array with shape 17x3 - :return: 3D pose in campus order with shape 14x3 + Args: + coco_pose: np.array with shape 17x3 + + Returns: 3D pose in campus order with shape 14x3 """ campus_pose = np.zeros((14, 3)) coco2campus = np.array([16, 14, 12, 11, 13, 15, 10, 8, 6, 5, 7, 9]) diff --git a/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py b/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py index 91b0072c59..983e477cfb 100644 --- a/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py +++ b/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py @@ -107,12 +107,7 @@ def load_config(self, data_cfg): assert self.num_cameras == len(self.cam_list) self.need_camera_param = True - self.subset = data_cfg.get('subset', 'train') - if self.subset == 'test': - self.frame_range = list(range(300, 601)) - else: - self.frame_range = list(range(0, 300)) + list(range(601, 3200)) - + self.frame_range = data_cfg['frame_range'] self.width = data_cfg.get('width', 1032) self.height = data_cfg.get('height', 776) self.center = np.array((self.width / 2, self.height / 2), @@ -236,8 +231,9 @@ def _prepare_test_sample(self, idx): for person in range(self.num_persons): pose3d = self.gt_pose_db[person][fid] * 1000.0 if len(pose3d[0]) > 0: + # print('len(pose3d[0]): ', len(pose3d[0])) all_poses_3d.append(pose3d) - all_poses_3d_vis.append(np.ones((self.num_joints, 3))) + all_poses_3d_vis.append(np.ones((14, 3))) pose2d = single_view_camera.world_to_pixel(pose3d) x_check = np.bitwise_and(pose2d[:, 0] >= 0, @@ -582,12 +578,14 @@ def evaluate(self, return results @staticmethod - def coco2shelf3D(coco_pose, alpha): + def coco2shelf3D(coco_pose, alpha=0.75): """transform coco order(our method output) 3d pose to shelf dataset order with interpolation. - :param coco_pose: np.array with shape 17x3 - :return: 3D pose in shelf order with shape 14x3 + Args: + coco_pose: np.array with shape 17x3 + + Returns: 3D pose in shelf order with shape 14x3 """ shelf_pose = np.zeros((14, 3)) coco2shelf = np.array([16, 14, 12, 11, 13, 15, 10, 8, 6, 5, 7, 9]) diff --git a/tests/data/campus/actorsGT.mat b/tests/data/campus/actorsGT.mat new file mode 100644 index 0000000000000000000000000000000000000000..c73c8b0acac7fc0e0762302f5a691586501eec5f GIT binary patch literal 4920 zcmeHK2~<RMgim%HwL=iPfw?%VtB zbDy)Vhm-AWy7^R7y0fjvlqJhoap|m8oUkQpc&n}GAz>!;IXo^Wj2lc}6Jka8SQbuq z=J4odrgXEJR%X+z%uVS`Qznymzd|}VYfv-uKOqR}N2pMh{LPcnTlWgT*_El5vp8%G zZ<-w;n?4wD8|C z1hEIBLrl*SI8>vRTCql22+v1w=d@IC^6oN(f3 zQ4xVFv#(tUzbXYoHM90d+5~!;)aWqlUxBzYLU7oVK<&%5k%3wS=2~9dqSsjo?4Ilq zWui8iopNr``HT)Qy3xS+GW8`G&C9J=A$%~YCv$A(7*;4OS+u|21)?F#^!ya%a93XKzcSq7aBPl3u z??`Q&K%&WT^UcK~0)>tW`fa)dj`CqNjxr!{KGj_* zvmVo<5=?_C?x;>{1AQmeMNJ7EAbpXnG@{`plw_XnRuQ*BL2c)wyWRpY>L0jpUA+$$ zXfYbbDT=^Re`YHyTnzup>;H6n_yD*jUUTJN?EydE=drtobwJ%nmUUpH1nA%IXwsy; zKMwqBJLZn_7eQAnH*w)|5pZ+N^B=19!J{oSSHbQ9&@9?`;K)faxE*QdId%=eDE)0@ z&$2GCb}E`w9MTQK)8ne$yL(~LOuNwuC;>@lBAZ6xaO0}?_g4K9c=4%2XIOqeq-inc z9TW7ySDs(xFlG(HU8pRs{#FWs=@XOFrjV$f?^_;dLSW3-uFez36WGPijouVZ;$+ch z*T?Xr(4EKId_IW8&Ci3^3w20j69T;uCV>+ZKuvc(flRlowp)fnP)#P@T$fGYu$zKH zM_+26t!yZw8Gi=`=BBDv{5F_bzc_7pPzU%q#H>qKdkH=vGtI{S&;|z?!NZe<)zI!c zebJC=A9N(_KgHA)!Fds8C%JowpRkl z8e=lkec#}4nbI1$F3I&uu1j)FlG|nb$Mmi7|EXj0aq?ECW3uug=KWpQMX0|UFWGyS zWA=N#V!p{&UKCjW3(hLT-aY>9IC6YhR7dV#j=P*kGW;obzvow3pGMV;%&h#p(xepr zv67NQ`5%>smHH*vy5uxe$`|~s=UiDm68cdss@n0q#Nl$U3Mgw zWRsp$&y})ylHOzm!6y=lEkqhoOoS5a2rj`P!ilYfKjHj3o+h<8c|2JwEANUYTjxWn zrmSzPBG0td9b#lk*RD`QA3=iaf^8<)d^1>MD8~qoaLohSeGG7Er;>T$PHR+d^zLxG zI7(o>;8NHNp&`E$QC5=&=)~=4rSm8zeh4geeNo9gd69 zo`i$DJQhY5O~RGBqdPj2Y>?PnwWyJ&AlS00CUks@F$Q;4M_m;%aGu72M6;sF$hzgH zG}g})-6!73dA?N(lYUOP9T3mJk)vKdnGrl5(R*>#9^hi=hGt?$J_n`SJPi}|{W1EB z((`&h`eBlANm$@P4z`2_4K^xs@w$HG!Lve7)O%tZJ*$O{?hP4w^Ap{1<&pB%Gi7d= z&_1TQG0qEJS58(xs}+C?ZQ5y}`&{wW$JQH6d$%x`jGxwALM+9|5N&Kr|0Lo zqlTyJVrQw%YC1|s@jcXajbLJ8Mq8nJ>gQOVGDDUAQM>P)1ZTvAjcY2Gxvp+heU1`5x!awGCbTnbk#b5nB1Io9^Z4)I=?@1mU1aoZnLkKGT>jdsO*Q9-wL zX20P>*&pN!c}|j_3uJooNBQ}`<(I$1x#Q!y{{1^!&IkE<;X|AgKl1iXUj4WqKg@T^ H`{>%QVREHa literal 0 HcmV?d00001 diff --git a/tests/data/campus/calibration_campus.json b/tests/data/campus/calibration_campus.json new file mode 100644 index 0000000000..910d1eaf88 --- /dev/null +++ b/tests/data/campus/calibration_campus.json @@ -0,0 +1,161 @@ +{ + "0": { + "R": [ + [ + 0.9998819135498813, + -0.007627303394110196, + -0.013341034396255802 + ], + [ + -0.01412240122676837, + -0.11375390190151916, + -0.9934085803866252 + ], + [ + 0.00605943391894462, + 0.9934796797343738, + -0.11384818494586636 + ] + ], + "T": [ + [ + 1774.8953318252247 + ], + [ + -5051.695948238737 + ], + [ + 1923.3559877015355 + ] + ], + "fx": 437.9852173913044, + "fy": 437.9852173913044, + "cx": 185.3596, + "cy": 139.2537, + "k": [ + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ] + ], + "p": [ + [ + 0.0 + ], + [ + 0.0 + ] + ] + }, + "1": { + "R": [ + [ + -0.04633107785835382, + -0.9988140384937536, + 0.014964883303310195 + ], + [ + -0.13065076504992335, + -0.008793265243184023, + -0.9913894573164639 + ], + [ + 0.9903452977706073, + -0.04788731558734052, + -0.1300884168152014 + ] + ], + "T": [ + [ + -6240.579909342256 + ], + [ + 5247.348264374987 + ], + [ + 1947.3802148598609 + ] + ], + "fx": 430.03326086956525, + "fy": 430.03326086956525, + "cx": 184.0583, + "cy": 130.7467, + "k": [ + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ] + ], + "p": [ + [ + 0.0 + ], + [ + 0.0 + ] + ] + }, + "2": { + "R": [ + [ + 0.5386991962445586, + 0.8424723621738047, + -0.006595069276080057 + ], + [ + 0.10782367722838201, + -0.07670471706694504, + -0.9912065581949252 + ], + [ + -0.835570003407504, + 0.5332510715910186, + -0.13215923748499042 + ] + ], + "T": [ + [ + 11943.56106545541 + ], + [ + -1803.8527374133198 + ], + [ + 1973.3939116534714 + ] + ], + "fx": 700.9856521739131, + "fy": 700.9856521739131, + "cx": 167.59475, + "cy": 142.0545, + "k": [ + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ] + ], + "p": [ + [ + 0.0 + ], + [ + 0.0 + ] + ] + } +} \ No newline at end of file diff --git a/tests/data/campus/panoptic_training_pose.pkl b/tests/data/campus/panoptic_training_pose.pkl new file mode 100644 index 0000000000000000000000000000000000000000..b4acc9c586f3810a7d0643f6437e29c76990728d GIT binary patch literal 1827 zcmbW1ZBSHI7{~Vo-xd)F-$euzTqTKjDIlLh$8Z~z1)XISS!ymZ73CWY`4zHV25o`9%xE&0 z3rsR3WvgUmh3qTxQiy8Mj8w4(&8$VUlxVhGAy)V+q;I1+PnH&%D$2DoBvVO9sy&Iw zyY5IWnyW-Ua)neSCUePXUV3`^mVd|)eiR}T^3~M6>LwqxN#AefO6+jkyRqYNn+pDr z=Q{cZkHb5g&W}&=9CUvuy=M>J0c7@fd5#;_(1#2={5JLV{`(hqgKVI^Jf+ADBbD>d zZdN&AETcQ9=29Lg$=}ZsH``amL1Nu2yYKpy!r9GaWw;wom#4@EF7j}5m}E}N1iCyU zq6v&T{(2S*kT-k4ZvhL&bB5}P0{mJBZ!%xHJLB<8}|Ig)trSH1Z0XX6~uQr$c}4P zR!1QRD`*0p5wghk%a((z?nOLo7#-~T>#_=bBoVa>tFzhsMhE<2A%VO0gJM0osO`Cc z{7lGSqmq(p_7xe?b2_O`AR$0UFqp4I3p^Ms)G_5X(P$A>ez6iQnO5F{mi|ZiWtzIw zSn>ZKDA0tKYs^*@B-cr%kF$avCs>I>JjV&u`7pB7g#<0_UW`9Kwdwqc1}rEqX@=Wd zx#MN$^f+dASD;$kfQ6en(+uO3sInqVc#;i#>R{^|n_;7^{5_y7TKq@KrA;PRGcpy|d#4q|~=wG#_u^&Mj*p4@{4f~}1#-taLXQ9_4KSK~9S zgqidPOw+g5x?re&N;{F5#o}ITWO+MXKmm%FG++vvXu^&u&}Xw;@G{N+U?;vrKyh96D0AU?>&~@3RM$h;%<87pnkshA zP$#}ZN&d#l2p=fMuhD!xjkqb5Tuiow#c!#`^cpHj-tbJ)BU`ou;XY73wwKL2j$(o2 zqiR4aO=y)y=OHSLN*Jz0t349RbxWQRku@f?mLd}I6e7>l`0JD?(i8uMX+((N;n8h& zAOd$h{vZPcMCQ1eEy;m3rfIOkCk2ndBWv2$!|fPlA%XtRv4RxrCJW{S&Q ztm)+`425JKl8JX7w>1Fugfut+)Kh9Yco-$++#KV8_VV3C4W=F`yYB?*ks}g2q*J%( zHG&{{CdcYda6;aiFZ{0lx(S9RtBQMnsE6TIgoS>N<Y*FIv#l6%7o6C z46xG+?6ERay>4gkcMvl_??Y_=G9Gr&gnAVaYrA;Dkj?Hd!2y3#lk0sG-tQ;&6S50X elnF&Yg-8sQFjk4;JQBt~Ga?Blv|eM@M*a&%=Ge6W literal 0 HcmV?d00001 diff --git a/tests/data/campus/pred_campus_maskrcnn_hrnet_coco.pkl b/tests/data/campus/pred_campus_maskrcnn_hrnet_coco.pkl new file mode 100644 index 0000000000000000000000000000000000000000..01ef97b33b3bf3f7a46bb84494177e116c287499 GIT binary patch literal 231 zcmXZSxeCHS6a~=HxW;|oO{~*O74XMv?+wGXye`lBPAcC(GaJt*9xA$yTs@=MFr8e4 zKD7^6de8eFnnk{OL^<{@x@B^fFKbIBa=nT|lrXJA46~?llhh{&dy$yz46PdFdQ=j; zaHR${PwpqzoOpnWv$Y6sRIEiltU9-9$h{6}@C=%i-Sq@{xbISZ{Lo!w!(dBi9Y;2z zDH_sgv*spBDqmgVb)^-V85Wn(q4XnphT>I2OFkmfm)`SnQeGp(p5}4xm~>P!vIuhn zJAe!d@kkr1jw8~iqsJdcO(W;3k!T06X{6`koNDL57!rEa=1QaaAaYJr?SygeEYgeH zz|&gv3Hh`9u=)Vw*aulEVzNGHnH07VCC)f|N+g$~`5|1~ZVBb0`X)^h zZ4>Fprz2&$C(hnQ`2uJ`B*E}M><3PlgCj~0s?fN*dmKtJ4Ur=jP-W|7j%AJ;=pz={Pb&VzevJ29Yz^ue(!=XOWVf=e`7%{<3mkBYk9_^_QT6@9DE zp;Z621VxUpZ{MGLAJu;9ke;e?AGIuwX~*1ofN)+o(BHuE=# zp2C-C>!FK_?T4$-9;cJy&C@l=i3)-Vb;k?DC5kVgS@984;PUXA?zebkPqE(8n87PZ ztxB}g>)tdpbJKxE!ueKY@%dW!3j2P*W~XN)=JGH_00V% z-S5t@xHQbSqj(y5xiW0CanA%|oUOX}(qkOif4i`OfJ`FmIWE^Kf1X0Blv@ra9-c)u zuPZRqZjeC`}?hM&m@;%M%FyE0iMIQn0|tjVa`S^8+-`eG1vIJtEb& zwhOh7=GKkHbfRBJVtWUQ-l7W!1v+&mKcI2q_P3AaccBGh^R%Xt9#puodd#J_34P)x zS1hu-1$DQ++RJ777F9mDFaDx=4;mwGV?tW&L#JsmZsrm1(ecG{jO@pDG-GHrXUvT^ zs88Yz+p~V*=)L9L8azg?5#jdwZ-ll1Bzbps`LTroB(leD*Ie6MBzG>DQ%{G1yiPM! zeDGon;j{_#sI;6!8j4c;LUA*Q$a$R8V)_i?Ro37)lRJS#FFW@brH>-R=Ot3}R?Z<$ zb9C{G{=d$J-}+%nxo}IH(KIraliQ(PIDu&C<*z!;97pJ7A%`0JCz08i$b7<+DTJ7K z)l---gU|?6N%rg+WCt$Z+`4@Nq4O$yO2the@fPC@EB0}uZ~vAnhB{M79F^z6ec|8x zVeet_de3WbQ9q~lX=`Q|x?1XDn!0#5T3WG?@wu=MHC1g-VU9JS?rSQ(1#W3ZbCNLy zeClseX8lGE8I2xv{j-dPxLy4y4XI!3*w%^WRu>JVHngLGCu3!eF8>Oj%GgF9u^|tabsTpLk+C0#8#SC&if0vJ4 z)C5AhS-HAGU;@#XZ``f7e+*gSV8M`NpF(<5EpMc}ocwz{xVI&wm+k9BZOX31iWA?W zT~&kEl`Xr_E1GjpIbss*{(bAGM>(K@u|e$Ups{R*^q*g!vlTsyp878%NRrJM&az4oR8U8+%Fls~`Ta^7-PZKJNdOuToga?4RYg@9+7Z>we^`?|Et!>*7cM-{!aP&zJl||L^&&lXcEM9Ip&v zU|~J?Tv;I26W$)Oure|71SEXpj(4e85M$V#VbE}3sH$0dYKc48@}^`;^b-Jor2M?t zCp;_@*?lZa!3&aZYCcIFa{=~Qp%FF?SID^PInO?ggU$_FuXzn{pmI}n;<}9&WJ$KU z_rDGU0nfYGwD(ka5mL!^%p(xyB}~~O_XNOk(+(TA-BhUEsQ9f^I}~1@xIsP9<`2)x zhdf%o5nw^gyu_l@4W_Cp_wD$E0ka+FmJhNKLC``B`w8)bL1%$-X+H;WnH7ukJ!%HO zY8U3Q!wZ{BX>eWdQ2Zc~2A*NTuaj^zaIJ|rAGD7G_k7IEzY0=dAUHcdp^*&tHDWet zyV7CLm$xHEx9PB6YNd1-HysSmu2$fVA%l8$?9xCDGR$fhTjILO5RVUV5K*FnxdOWH zl`0kF6ZckKZKgw3`NIC&A!KmIQ3#Hb{&4DiU)iy^cHYFOCrDpw+;mjXDnQ9edwf>kA;&AgN^fj1UNZVW#g>m3)0z(L!~CJkRDrp zCBoJ2m+i7Pv+uHg1q0&Th%OvS3`pX7%DsIV6I$hh=cPMo(0ol+H&~GdHW_Sgh>fhhEPs>+GRj2jRW%G~ zI}jM!Y{q~w*N-2bxKiQ7UGoaNnv@^e(g zYr6y*$gk`9rpQ4BTB1!Lwwel3(s{$k1{$;li{-s~PXiuBTh1d*G+3Lo@11KH6`Djl z7CDyF;FahJ(N{o)YssOq^BELyHH*0LB03OWWo&=-IE)02>`p3en|wft^h_N!2nIb5 zBiC5gd;4vJ1Rq!sp@c_5Db&ylB#nDcyy=lGdY=<&T^RyV9o|x2^G)f3Y{7?n?<4IYTj*QA;Sp|)I|%Vb z+#?dbz&TQSX6&T}4CD;OhCR~!i(URFa=Iv9>En4h0_Zs(9^Ti9hs43ATK`l}ctz^% zN}YEGjp;~}b8Id^cpfL_K7<9da*@1V9}8~L3dw`Lo`0CrykGPWhOh-dUm8Z4UzZ3m z>BBSQNpA42QS#23N({tIJT)W?5MbM#ua#?d`avHKMGzGSP_K@ReP3hx%k%4xa@yx) z*5*EYI!JBoniv|T!~LX_OKeurA)8@(Yk3kG2&OZ_dHQ5v`mzb~4U>WUZprFU4F}MHE<8=Cwm&uy%8BkPSHb1cre zjc3`-QDOKqOD%6bA0b1@-v=Xt~1v45P?>m}0bgBwXua)82N8tn~ZOq_O55EfFy z3%*EKdciA?n(hngK5(a;x%R{pUl13w|Ipgt250)(SDwlJ^EiEf@{3kB%kS)39)$se zRIt*|oQO4}K|uA3WXrX501=BS4t_cmUG5F%dqM>}HGZ~9H5$Az`OI_6?~il(u=A!WLRjwhc?P zVqhUJt@x?77bH{)2bX17z_o7Gp6Z%EkJG2?$#{$u4y0|rE~t*co*>Xc}N8wG-_X>b1c8JkF+(7esh79?t)yp4_!c< z>%g`9EL}b@vfs`91{O4nm4@^cJt3&hYP@MO49cYkYtw{i@Dhv1pS=?Z5?2nKBSZzj zvm0ebDMml_vt4aGr}9M+5<1qU6oh(_%Cz1%F=G{=o=agkI;{Aentc1kJFq-S#3#}qnFB63Op6T z@ho-zsiz<4UOvAjpANa!-YFcq46r^hue~~s4xQ|tHd5cn!0UMn!<|Y7fnYC=)ov8H z=v#Bj^Ewqm^t(D4QB;`Z-yZk)Jso(Z#M*bT^mpb<{+D{H{!r_#Q*rKtAB+iB?7J2C zQ%_fKdy$*v;0Hoe!=7EsiGVG(EErnELc=pTqk?@nxMyeN&?3W%OXpc4z1kNv6!yzB zRJ%Zwwc`-KjLk3Gr6NaU*uR*}djDvnH8ig;L-8PcKO1zWsG{i&z7 zWun`CgqT3(91{wO3x(UO)9BKC3p6<5vqNAN8yzA8G>O)URIr@bU^{L|gG9M%pLy0g zRFLbv64xCHD8^Thd)WtqS>N_GS4bpSEYiVLN%%l&)WNpm9Y6K-C7Xmv#~dO=p61W5 z4)=nGS2H>^om@dtw9Qg=r5gmC-gNwC79N(oPb$ngOai~7F_MCGdpMV@(&1$G>v|{3 z)$GdU?KEh75M;Yoo(94<3*KqT(g2rAj7*)UgF=5>vyAXhJxzIDSJ}3U3Ot`rxqTa_ zfXxl{EQ#dXzx2<0gJN2mIdwbmpJ-fK>1O} z_)*{Z|JC*3AL<^z?f?Dyw{?>LE&Bj}_TTv*JN|Y5nl)wEzSoU@)&BpnDW-%(``i5>}BQC(fgU#x){(LF=YN0m_VTkvdcFy;4x3a1Jz+BG;lq-B=C`y zy>i-yxmoKgBc36zAP**NQ@Y-kq#Fte?!tp|O`#AecBSH(ECWtOX{O1r^m9w7`7*T& z4A4tIWD@v^4tIhEM1t2*pn5{vEqJpZ%qm=Em!kOrZ_K@;xpEY+l9zT94QD{FimQr* z0TJvfT%#=2@X(XZtufx@4j#!C{7k?=gaDBdu+tm#dvZxRjwCoWMwG|wCP^}T>jSu0+vNA9K_E&}Vy4}vKm~W+ zQtrJ(Xq(O5pUUqIBL3Uf+BANx66GJ*`z&COwE z>&S4LecR@p%>i)6cz=zrmOn5$O9j@j>f#Y!TCdGIl0hvbx9bf{{~p#0%@5s6h81oC z7CiAmAYXQ0t4}5f2sbkou8osmNz0+}kF9u6adhfb7sbFCyUTeZIvDu$%zBxDn>XAj znrpX=_kr=Sg9}BiPH>xXSbTMu4U98H5@@fCfpYZVt}z#5(C3fS%2cz3qfIJKN=z3} z?ovK_CRrcUju`K;)7baRb|GyO*IUYofu1_k7Yu7R&=qjw#s;{-mZGR4Vb)&LMCkbv zjaPV(lZjON67L0aCj@;$4q%|mFKteg&k{uX@eVt+|8K$e2)U(;;zqm=LLbz>9YlH?j+E+k<~5BBEs#L zxFt3p0-(Qso}s#i0w+BgDirXB7wl@cLn1t3Qu$zZOPMR&B(#Nzg}K0tg~g2^XHQ7& z)je9F=MAJf{*lQgfL49G}4l!c?s6VtF^vyLX z@&&d;TE3n#tA19bJu#Q;1NJ8`7B4>%1j5iaD*PY-34k%{hjUTOFy`Y+)-*-5&&(OX66=_UaZNo$dP(RhEs0QeP_!^@NSh{ zG9{4+jt=Q^goq$`Q9Y>^<3R(R`GZU6roG_)2f>?jTpp0V^;C{@v>gNrNvT+e+XAKW zR_$`E2OxrWXErRwgH8Xq?Zyul;L)&)Ir>@sFLwDe>ZQNs^yW0#4tzNSS|rutv@DrG zntq=6{3`?2EqStYcUdUQ9))GCt#t6x6rM2fWq_T6lqQ!x6ACU~3h&t(3R!aIj@af< z5N6BetXt3GQNiukJp&9Vm;dUNjr~2RZ9Y0)?hy2Yth0Hkg3*2;952zbR-OV6Cnm#3 zx*1R+bhAQ0l?Y{>;a?5T<6*bpkv&%T+~M`PBkMBOVxZyO_PNwfqNe}*IQ>vd?}PO)9frE^u76^{gdShl9R_qJ6xD3V$Q25O{UHu3Q&@b) zB45v5J3@yXh3$;|;FYB&pm|M0k9|O?TeQ16Db6>@nv2_i@^~>8VJ2R3H>; zzuLWO8ySWrq$dPo$#4n(-luLW8N!|RG?gp_K&`CwXGl=l7hxBp2Wsql5vw(06*b4`SfaTZzR~ z7FTi)&0#j5^Z_x_DH+2zPC%KE3^TfJ3w85Y-SI&-? zzIehjH^g>t76yXPd|HR+bq4JUAq|N}M<@;M5ap1^0b{S;wN4h_?@IWUM(N-ISGoC` z73=?9mlToFF!_51bU2_3psmv!h_F+eL|t6wpI&olBo!Q>~WH>oVIyeCEhHTg)NQ_K87oxQcsJID`u zL`6hDFe%U;q}iBL&w%R{bW~1{2)XCZ4IU=r!QiuKdkxba^i>0#t7qN7ffU%0=ZA;M z*{7mD8;Bsu`5~aB)fvc}mehx_^5L(3fAB+3%UT3<-(cmVUavxjZ>;?)tv6>)gG^ZK zjwpfYnO973d#7nNEfESiFdS-V#M(!{(9$EMO^1t*ZFLRa(xFl-K_sby2}U~mdz&et z;9SGdT|dEqFK>!zJ_ZaBtf!1>%>Lx`?gu?=VKYJC*0Z_iZHW(19W+D3YJFfNaqDb3 zZ!ipa6n|U(oB|KD=3|fTX3e`W8m;UAOHbdy4ku`Og7vPaEyNrgWacA1Uq!v4|BSAj zat{GiOqT8MKIH-JR-&ZV5msHBwZ8t9?^S;2>FB1z&Zj;GLe0C4Ynt8#!v5VihN5iA zz%HjY5Kuw}5hwN7@LDpcIw?peZw`dbfh|Q_C;Wl5btG)-PypQWbW{(!NrsQbyt)CJ z6xh@d!ee=h3=&!bAD^uahWr#e*XEp`oK9>u6XfM&8OoyZqp^u;Oj+$!HAR-dfr1YUK{ttGLTgS^L&9+nxQE zUB-j@>(d-(Sbj0AE*DHa><%BRxvq&nc7{x9Aw|y68I0AC*1OF(823x7j2Z9*v!G{o z9eH?=OnAmf@5O-now-8S%b36GX<=B*K9dv#cUl(?jVqF1(`Bl*wjU7$L@>FAysYmM zA~G91>&Vc%tZH6qzc+-N?Re9{${QSPl$V}z%Oho-vl ztKV4P>GBSxY$>}o7XUcTI&>vpvN&3xoUMVbR9gfOSP2> ziulCqw;H{mwEDV9hoJ{dzlD2^N_LPQlr*WLV+*@CYM;lv#lXr?`{Tm|);D;KzU=lF kEC99DkBntM{FnLipL*%PRpF literal 0 HcmV?d00001 diff --git a/tests/data/shelf/calibration_shelf.json b/tests/data/shelf/calibration_shelf.json new file mode 100644 index 0000000000..5927139d09 --- /dev/null +++ b/tests/data/shelf/calibration_shelf.json @@ -0,0 +1,267 @@ +{ + "0": { + "k": [ + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ] + ], + "p": [ + [ + 0.0 + ], + [ + 0.0 + ] + ], + "R": [ + [ + 0.650977, + -0.758717, + 0.024027 + ], + [ + -0.018862, + -0.04781, + -0.998678 + ], + [ + 0.758863, + 0.649664, + -0.045434 + ] + ], + "T": [ + [ + -1586.4496077989998 + ], + [ + -2109.46905869 + ], + [ + 1104.209800652 + ] + ], + "fx": 1063.512085, + "fy": 1071.863647, + "cx": 511.738251, + "cy": 350.088287 + }, + "1": { + "k": [ + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ] + ], + "p": [ + [ + 0.0 + ], + [ + 0.0 + ] + ], + "R": [ + [ + -0.016771, + -0.999835, + 0.006926 + ], + [ + -0.029435, + -0.006431, + -0.999546 + ], + [ + 0.999426, + -0.016967, + -0.029322 + ] + ], + "T": [ + [ + -3512.391424833 + ], + [ + 311.47771461800005 + ], + [ + 964.5481307480001 + ] + ], + "fx": 1097.697754, + "fy": 1086.668457, + "cx": 521.652161, + "cy": 376.587067 + }, + "2": { + "k": [ + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ] + ], + "p": [ + [ + 0.0 + ], + [ + 0.0 + ] + ], + "R": [ + [ + -0.789986, + -0.610527, + 0.05638 + ], + [ + -0.370413, + 0.401962, + -0.837389 + ], + [ + 0.488586, + -0.68241, + -0.543691 + ] + ], + "T": [ + [ + -1420.944211509 + ], + [ + 2546.574076866 + ], + [ + 2688.8728944060003 + ] + ], + "fx": 1130.065552, + "fy": 1112.470337, + "cx": 566.884338, + "cy": 375.212708 + }, + "3": { + "k": [ + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ] + ], + "p": [ + [ + 0.0 + ], + [ + 0.0 + ] + ], + "R": [ + [ + -0.970568, + 0.235647, + -0.049676 + ], + [ + 0.09763, + 0.196438, + -0.975644 + ], + [ + -0.22015, + -0.951779, + -0.213663 + ] + ], + "T": [ + [ + 963.489306486 + ], + [ + 3408.674914882 + ], + [ + 1422.035001899 + ] + ], + "fx": 1056.162598, + "fy": 1059.639648, + "cx": 552.43573, + "cy": 393.180389 + }, + "4": { + "k": [ + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ] + ], + "p": [ + [ + 0.0 + ], + [ + 0.0 + ] + ], + "R": [ + [ + -0.194109, + 0.980554, + -0.028888 + ], + [ + 0.233045, + 0.017488, + -0.972309 + ], + [ + -0.952896, + -0.195466, + -0.231908 + ] + ], + "T": [ + [ + 3832.020978729 + ], + [ + 273.55271850000014 + ], + [ + 1439.4616998990002 + ] + ], + "fx": 1089.654175, + "fy": 1080.99939, + "cx": 498.32962, + "cy": 359.514832 + } +} \ No newline at end of file diff --git a/tests/data/shelf/panoptic_training_pose.pkl b/tests/data/shelf/panoptic_training_pose.pkl new file mode 100644 index 0000000000000000000000000000000000000000..c004aacb92dbddc123f6b7ab600f4e3a39cd1e23 GIT binary patch literal 1827 zcmbW1YfMx}6vua2M2di*lnNq(__$V{QbYvHk%wAVp{$lH@~AGXy+lN2xvPL+P(*~w z?%qp5c59R>lE#D@W2{;~;Iy^|Ei^_=t!Zs5)S^Yu1|MnF+L=3rnDkSf4|}sSXU?2+ z{=ai-GBuWY$+)d zOy&Z?4QZH_TS*~1Mb27P9h#7=HXxUJG*O3KIjvgjt<~&7ul;LUC{&ah-H@B!i9EG& zF`Cq{w;s9ckO!yL=+$H|0ePmUr>DIjOZm~N1T@*ukYp!^!N*O!EL(t>sZ>%7T<<7d z>Xu{&-I>6;p%5z!dvr%S8l&*}M_jJ0%?>l`hBD8TTj3s#3rV}03fp~-t+@4t9ir)a zTY`}xa&RI8->M`BN7|SlhXnTyCWKz|T?^!PU|oQneP_tfnk*I%iaA($@23`+pKJT_ zQKtkqXS?ideZj*K63%sVD@4!@)CBQAXIR?8*I@@;F3s=}{d&mo?C2j?R`&?d*GtxA z8LAUw^3P^iAk^sBuRpyQBjToGM=S91a8;scg>Md}o*Jz{K%YroXJPRp8|(^biM;xF z5?1_-Ho~gdhGpyhtZ;X3lxx>B9v*yhK^tom**lFA#QNi$px{*a*q=mrtQt6C7SJm~ zn!D42`tJ&%yPX{5)qqkzUm5DUQ$dyR1ikSt4=PM2}< zAW1M~E0DfX|r^4GYl@Ck_ynZdgFOy*6Ui;|!p)r?S(|B-2O`;PFeC zRJt51sh@9S5@tzI9u_?F;c-1H;Z!?A$6W@h)No1;R)*e43v8szE-}A7Ma=Jet)P_f z>n0eXh%U^*%Fw$aGkzF#Mgv%Fv8&g*fVHpOp=pO^6MN zObg3Vi;azGt-3Z(0^eL!ee!OI$WnTS24~z28zT=*j&*E20z|kpxaT%8EyflY3wOnEA}JmFntxx@3B07m+9|tas0z5n-P0y?gad86MHKOTV6;@a|!r zlm;v6G7=PNmxwV|rgIr1VTh5a$S@;uyTmro#~l%FBnj))FS^g5QenN_5v8K-zw6$` z?d~!tWoLIpn61s@?x)lwC8?CX!s$~ZSyMK~NM1KVsF7lvkOKQ&6_uG^27dzLsRB4GK5a3 zGhcnk7|$(6bNg(aKHE;6<;|+M0_-!dQ7}ebTeIs1;k@l<;O3UkBwu?{TZPM^=N~C~ zcI(PWb#1LW(Mp|J!Le@?t~Xy3U%tRb;AZQ3Bb1z|v?EfT*{bh8P<3q;9@iiExzukl?zwGa)wS7t-~e@&Gh37X1384c*3*^3-?1zCUqQZ} z0sU4;y5Z_`vqMVYTFidPo1+uJF7{^9bJ+7oTiEa5@=kr=R-PV^=e10-x7PyLuOII^ zO6j%AcZXfFV&qGuN4}{;_Iyq;PJJ$-Hynfhuq=?*9lZSJJpo2WkC7>~?=T~Cw81SzSgp-xA2)EuD`>*=jM&(4W2Rq}20bZOJ_k%EAmjzk3JZM_<4_;$45p zJGdt95M5t8!aR2FPzdDj&4wOvB56E%QM3`cr$x>2O3rF<9(0Mv3qhA-j}H%T3^$1G z-6lZ(aQD9ItVf>5xK_~;r2p$rz_pt>u&l)Ie~|r41nAZOeoA^A`@J|HIq0!+r`ah6 z_H(`GV5MKa%TBcKJ`DZlg1o>*hfC0-d*`d`C1aGln2{VbGOyp{f`?evGSR3PNrZb2j2s>#r=l^JpT^vZ#-Ch99v_Wb92(7#HngfL zR*qEW@MRm}X8oxFT%7$QLg7|X{shzY$~(BZq#MlCZ&<^WoD5qQuFkyLQcRZz=7AnL zHXZbc*pW1?PMr}YB5+L<#7kgWgqN32Lp82I`Q-02;cZ5uUE>65Mu*A?#Qm%)8NJ3rL|`q_%=G%qhM z7VmnhTErb({? zjJzHrUufS5Z+b6lz8xaaiTaA0mh7JN9=}qp|YSkxS*5 zv>3%4-WNGm%sv3G=mk?^5%Dy8+A4YewFA7U%R^XpYgS)C?W1v2|YVyWID($n=Q7PobBFxU{L z37n5eN;z%qdHTCo3n|!!#X!F`GcCMs-7H7=$5piYl5GL5-D~ztv1e}S4*Rt_4EE|N zlY?z@FGYixMahlb%#L6a2?kNlEEoa1Sh;0j&ynW~lORa7wFi$V{n~FuAn)LOVB5U` zMlp|3Tm@$dKaC|7jip>hX&;Sc;?Su5`hccYVt;4lelax%a--fZh;l(#P0*`-(t*qT z^N2<-&XZ&U&KnFN^MWEnlpdab6y?~c!tv$9wvc=F1S7}lNlJEC@=~rNx}H)g^WozR zTEweHBge8VGZBO`I(t0*VqrrH&Rb8AW98{Gu!nUB2EDS$PjItizqqynxFruhK(LKA zAumhTMK0BU>;=7gmsqlY+vh>bPS!aZ!N#Vfg1mJqV~oOOn$;xl^_$-O6qYp$&joq$ z<0a|;Ml_lYCt<#3`orYs-*?eCTi(46Fv@z2a*D?Cei|z%8Y{YtNTEmEOMzf`77v><0wklHWN61H^NeGmS>(I7nvoqcwkT4-Wb<>ISG1v z_(TR;n{dXpMAlpb^RY3gVuCI@cbj5rG#C$)w=FFcuOpI+@D@P@-FYR4W0 z&NuXeemzCGVag8c=2LvmdprSM`tc5=f5Fc;cT}HZWq^QW4_|_Gm<~h4DvpF?OM#s z{5%A>_P|Uh8r&j>|g8rjE;txEPZElnNLq?w3P>&hFNFMCd(_sodsvO493 zKlKJpakds*rXo@<8RYc|87S(^lN7x?~7au+%e>(na9v)0LQ*DC%;cY{J?z^vtrGm%PO)Y(s^ z^3m)u>RM|SMIBF6PvCmRM(_%&)`xOt^I6buw!cK3MXOGf8>4Ommw}9e=VD^==99xH zWm<)3=(jqT0UKtTQNXQTCm^p^H~_n}v_W8l_w?RXUW+Ds=S`!WxvmA|`Td2^Co(mG z9^R@u*wGtzp%PeY2X#DO3qxKeW+<8F^NuI6`aWeRmc6)DZDma7uG`bXx ztz1THAB{i9q0v0|0%q`Oi#jX!v42ZbC$XYE%+PP->ZowpcQkOl)~}G)GdB!TxcO!` z+$J83f_dC!bW%8*)C=-vWa)VF;?@DUL0%h=jz`;^2OW>-v=4Og&8;Et!$%B(8?>`i z(DCS3t|KXEV{69KZ|$r@9nZ)N2rBNX3VXCkZ7A42e?{>4?9Z_jxSoC=N}prD$j{O7 zu;)ub3mbck{4@I#Iv###1vRcIGeN(cO&yQfBsT?T#|lJW|KV8Hda?f@2>G5riN37K z=6}(40ahEd^C29LnGgs0>g=(rBFJw zua!_LtxKhV^P-Q#6fI`2t|%S4_BU3QC5i#(XQC;Ww)q#UoBccvINMU^gj#LdP%5ov z{|&&cIj2#YtYz1rM>H?1EB*ZX$9-6x(;mmewPUT1D%{$&3DKah&DTZAYopy){(ra} z7zSJfpMxI#eqGYPeDEqIFT-k+d%d_kSqr$fEhFS*%BTpXU+f;d9{RPr7}sQUcP(e9 zg&(wrx8vE#HdgvAu07hBhOm?GJdND0efbPMjz3wCJv)@V7??jEuD^MEJirL?7@eFP zTh`t*>>Og~c8=}hXEju@8s;*>eXK^rVO2(kL#u4I1^4Xc9Sgz6kItak6fqGz*4!Tm zlK0{3$a== z2$h!qZy(LK_RaY62?c^mTQU;%YbENDp3qakt(38p^IJE;yyeMylw7=ivlhRl+}@!G z^y{0a;@YaY2IE_KaJfLjZrS6hmXddacP{H zev)#R$sb3mvv`~vZe}Y>0%u)kbyc`lZ9dHTFD^be1HIPkUm&mL&JTJ0mnzVst=KhL z{k^p*5$Tur;AXzNG3oy}4|Ivh9OQ=J3saQ595b*N&IvmLH^rxk>RSKnEj3tO=x$cu z*r;c`mG|NM zR{>`U%0j;-8)6x;&4tPSU6(Ll{$vC#ulDJITaA}OUbd`8t;WE_B;TO{$@g;uH#2Rq zX^i$5y%deT{WSJbG#W0$?W3`892)hIBq&*y2ttP;_P>W4tU}F!%ivOQKg*O1v{LZMDLTHpjZ1yrA)t=9XLCCnxZJGE9PsKiluDRdp^@{~Mvifo>V(B(*t6NRZf+el)PT+EQD)Rc6)$#PO zJu}FQBV3So^s@*Cd6vROEPV+At8zjiV&mItnxz*vW5NiZ{0;Iv-5P3iI{X3Lx{?|4 z*7MB3HNK4GgYuGmL|*E9chXZCJz@t~wK|=pAb&Lp^6Y$n*e?=BfjzBEVddEkA5Q4v~Z20vDH#z&&Q-dX!_2k3x=>*J)a8-6`Zivu!wX zEdSmJZZOZiMUE1WpHNU0C_&31n;SV+OYsv*pH-qVXwgTHfV_UV5#`M5RiQ`f?J37T z`I}1L*Uyw=o9%!-X3D*kllzrLE_LJ`+}b~ja&NQZ(9aLh!z;GD9MP5J9Ola&4n~fZ zeXnA^a_%C?TP-V6jxC-6^49*2kQe=;37>V&rg5~#7^7$$>!)#?qH(;-nBb#vVjLRv z+BebJSZR~?P;{}mm&pxYoK5+wyTWDBe?XTb@8IU#GH|Ed*BO;tn^YU}*6M9AUj!6@ zel7K4;Ox=}=+}1Ay}XvQ9_83En`!lr2!gymzXE8~HhdsAxI3XkvKCYWjqLPF;CjMI zaF=oa8Yk`*UnLV@ilI)y}cg{H#t_tj=&5GkvViio>cok{Tdw#7GLx+H`X$CsG5% zD=np9+dB=j|1Ylf=|cVOW@>Pf4nH8nt@=+6|KbNfIK ztKA9uwb;bSneuxK*ydjQt!Bp|Zw{$Nta{}gT)r9u+$^2h8~oG&nS0uj{TZnNGA~mD z#L9oee6zQV1?OBpjq? z-PUwFNkQI*+JEiPdIS$E(geY2<%%z_U8xNkMV*S6F7L4%!6`=k1R7Ov4QMN zFcZPakM||Fy(o>~(So*912kWQ{o21BsnK{l9Kp#JB%)lMWIW{A%Ka3~iyDJ2`N0GG z9bATwC3^E(kk`X&(!2+bAUnPOba3|XG2krY4#>+L=cpt;yA)4eZls{)b9>l(1&ch! zVnyRGej1l38h>>erjN#@acPXLM4d{=wQhXVeMCd_Up|dfUz5y;?rzH98m4Wj|w4=wui$GpWpBgl>fPRo?^}NqQyXT-B+ai+esXhYs z^OP+qN&_cR)`)rx{bFYx;Ow7kM2puyEO0+^Fndq$t!Q5R`QH;MH}`2APhS6}T#T9r zo0bx|e6mLX4Gmj$P$vb+ux3jmuc;V|85|R;>$> zh8jg)*^Yfzb7TzdM(^9kL>r96jvJ8hC}Z z-3od3$LAJGj|ir|lr(dO^Y^Hwa29z1k3f0F2{G#0kr!{zP>^`_ z>rd{qR-c;#X}adTmdHWo_-Q{YoF}P`=`ydpR^SNeHFJ7{%^ZgutHlaz6w32=zN^BCI|jXV4_ z?o>4Hav8gQH13H*qf8!*TqNdY30CHd{`8WSK7JW+YvHA~3fD{CgnJx$vq5^`vei)F zX4%>30JOKW+9`SVVIkpn!hkz^Shsd~W+ZQa1g^hqkNr>iirRM5E3YqT9jf$*Qcut~ z@{SuI&+3dr6v+gih^~;Fz|Dzc;eK8&7oLTgo&G`2G+!)4HjrhbAGSgFV{lgD9;mQwPYKDNX^Gx6UFGw|Hn`z z&x)}md7D7qu~UYu>8j-QG70b$#H_ZD?8zRCMp}O4uwVXGpO%CC z8PP5i7sqnoGiu{*ll6H3{d(|Ldb-o(0?}Kw5|)out@$-&kH&gIo|jD5OO4BqeS#mv z$|baH|J{#eB^uYeq~zt{a*((7Mq~M~uIc{>Fr>#gtY|#qr}3zw@tDgv?xXQU92&Js zp|m=0{?=X%E4z1~cih6J#>Yecf!p+WW%5>YK3@F^b7zoy5N;QvS76$$f2ku{Jv zvo6)uxcp`s)LyOKsCc;f>e5{`T{GLS6kYS%A$nQqqQF_g3BcKul2J;Jb}ot@a!rmU z`EE54bykHUppkoV?M^UozO56MkD2xnjr%E2sM5pBCteRX7i$aLk+%Xv=mm+DBS}AN zigD%8Na(jZO^=70%{2JuzvGIVJ&5)fMbud=8mTESl@+GjxQl3Fh5e?Hd#Vqfo#A^=L@G7*pytpVk z6SL6@CBtIWy?WS;2gohvpDW|(=V_?KiSGS-D7{)BJv$Sft5Zp-eu_%Zv;|<>thENX zm@B(0dAWEe#u2;eiLy+06FE+9DumCi1+f&IViCN}CcUEEDCl;4~!Z^aJ)-rM_bxtUzw$M#m2HrQW#2v8^U?3$f;_8si2C6UcFSN~^ccUZ;JoCg@v@@vip#j_qw!iC z8nyMWsnTaI-4JeSK^vrIn#AaMMAf;q6|TSTiri?4b&wZ*zSU8<^&>NIM_x}@hAUil z>5d?0)px|GYe$}?P7HZxT=8x*<=%ONFkL+|9G!tEoCUdE7F~wr;K*wWwjc*uIXgj~ zjTw!OK@7@|2AE&_Ii5Y@r#9Wx_-26^a_5ZI&~N@63);1Tg7B07BR?D+kDO_~Y6`j> zdrU9Re@O>i^Nfp8dacAMK{Lxin;2Hpt*}QMum!a7Kzh2vV(E5+JxWWZwF-4ea#kkD zJGiLd)TZ&e$GD+ryy>U$mZI^t%edpC@opR%<=YO(Au_p1IrcWW!K$_bxY!(mlEp{U zQxiwt!R7r2Gt)B_^%TF!tt5q&0eMRnK`&UpDEvJsh zJW!nGJ9jyCJW@mUapuc|DpE9-yhj~Rt61{GhewE3Gi^fZcm{kx$75A|iOzsdbc50Yk#*+P6>Y?M|gF3=~v+`V8 zZlCTW+RZ<+!5)^p6m>8Q7KEvB`I0`MQATF~uA5n+6>be{La#&BhybpwF4hb3v4_wh zS=BU>Pd*aM#G3T?W!Uf4uX}O!v8es(%a|s$?$kMtNjXe@q+Xkkk$HYcO}pM)E1>Af-2=-`RT>h_cEbRA= zt7WbnrR=dbT?H-rlqICci;L<{Pbqox>LuW!+IAW@Dje^RIQBUDwNk5(1sH#LjMx5- zr=0O7#CU6WJn#IhzE`Y%a2bF4Sp67>RqIX%i(D=CEFnC)KIHU{CW1#6J&t7L*u#D~jCsi6dub!LWC2{W@#)a< z@ISZHd{37~2gCPHM2;1m^B@_qKsWTVqlIZhw`~o`>&1SByr>s741UOBCkx|~$M{PH z=ih!BKPwvlaT#BHG=7akqt^dlRC?`Ivqo@(hu-_p-(^C_V})$41N;g>DOeI~t>|9X;&&ez=W&dj>t?)NJHf{?-6)<)?jatLHc5 zb}{o4Xptj-hddv?jyj&lB}mVW8R!_SV}sD~n4Rcxjx#TQ_Z!_k&JmQ`Xa0(g$I5>e zLB+z8(YO_pk^bMxqvVRJwP=}m?O|J{P$_OX3pC10b1^SH;tA;CvL9&St~F#&pFKod zChr#w6Fj1EgWpmKl^Y>1zodk`-Xa&4qpX_PrtzD{_^xRD*H7aQn?^TFz?jROz&<); zx)a(Larl%UsP!@19Hyk2lip2~xvBNh{~dwa#e%5C(Gqx&e0+GH4Db`*NSi&@BL+QE z+XO#}>ETGin!(Zfn03A(>6uSyvqvl_NSi$~rcgL}<;D5@p`6)ayYURw-u<-MV?R7Z zI9b2b-k*q|)`w+Y2Y>Uy9<)CCUTS^xC95f%{-%ee&U~4*9!Za-y@l4tH{Xx6F>h5K zht@|cL9LHj{29VlgylgpH*vwK9*X_2J_wEO~`Xm=`OS3dz>0b`arb zJ<9}nxtLlXvHl#TiyC>}1h^B~BzfG4?W4`AJBgyq`KF~isk*YcO=h26?&S8-$88Gr zk^ftldi^=*53Gc}+EtWxE-q(T4KzgKU9=eBqg_YSA z1Wpb#Ax<*4hCFLHo%*4Mzfo*loIe?Incuq9*<9NZJ1%VbZs>9B;df2=#-~45uTUkg z^+3 zwZ@xd#1@L~)nktCN->`&ZVFNWuUj2@| zxu(upRUY*ct4M#70BVww7}yrF((R0go1ulMr=7Z#*8Sokh;wl#fb2OwityQ;kXOyQ z9QmKMJw*6V!zeyIlW7~ti?gpufLrn!ttl_gvrO%-)(F2a8cmX{+nVe-mY?(-`GA*Q zHD}zAp%|Ct-A`+0?M#d-Rz~%Jp4ki0B=J)x;@QI=x$tuDf9;f6>zod7r?*M+xHH&C zr&TK7xr{rbeI4S?WFLzRb7!_se!{b;E1U4F_SxmmrbN77p?7CjANdK_7tKLEw?0LO zDFfx$E>uY0zo$C?>o+WJ*)|(+vwl_J@?*PDC2#d!1zg{I5(R`eHX+Y0(Mv+2PbB0; zlbw(k$vRM?+jkd*$oz5`L8U#Smu6U+bkM_A4eqA&^9|t$eyw`97`S-G~1Kc@ml05F5_E80ME=8FyQRh}yHn%_7XO}yVee`jwsgL4t8=V}%BoAf` zRlXB<=|;}V_`MUZGxeunYcvqP(Y^A*!JpF&t(kRI7bUNS-sr5(yy0NT^A)SWk@>%0 zIx1X77lH42&j)CNWT_<-JP+vIXKi9OaHQP|r5nxzxrnP4JrKlf&=UBVKR8E4@8W#o zuJ%phHsU0F=Gdv_%S!54swu_=&t#qxAEBcZu60cggRJFiWW$DPkUDsJ;D%6!}wP**m$1?{uTUC2KAxGk(c@^foeIZAUM`;F$- z_&#iLaC!CwaF#ES;>{~BGR^=lK2Z^~W_*E9SwxDL?!G;*1n^e?6R5O%?C2<^F9 zzXifqtjA?EM%ug_G}jYtAum2h0vA;i0+-*HfD@6q8*x~HzRhf1qdgh%r51c@7QBnK zUGrI2K#xJ#5DdGdVh#zy-f4n9fZ%d^%GzZ&wLNp`mIVhn|sRJaa+VD z$>T0+9~HO76lFeci>oV}+Y+<`uTLE=kZR& zCx4b6I6oN#ADV}UBUkHn3&RJjWQM` z3;3L$E)3s0_G?X&5}u_YPI;l$7U*Z0Mj)=NgGDhuyV8LgO~;-rSBaY=8(^nZ zX&3ae=Jahq*_!P4`Deyj#53JMYyv1$WOR-bsUnvcC>n!N+2?X-hh?clEDCN zS&2}Eo3&oiqF)gVd694&aQ)604yAyF(YuxD{qB9fV@~Rj<$|wpFx5X zt1p3AmYTkSMOTPu+$jj>mDae1_3rDiV^(cX! zxdp=b%JQ(^nJ-^Qn=96o*|eY*2SZ+V2nFF*&J9HPqY9XpS-c`}xxEPF<>f4p*AIu% z+ikZfnDyH$AN?oCM$V$er+kc$zUp`I4RQq~lyK zJ8|ok*M7VJuH~x&+vV8exE57ccUJP+jq{|Z@<|HH4B;rk<|%rqh;8Z>2DuEspa@&B z4=EUS-K4vWTumyZnXy+DfutTfud)nKS4h8;%xR~%vqc81@Y#{ zi^-d)2p?MvpGor}72$rVfm;(-0T(w1KtCT=2RN^Mp5!B&QBm6W0prRnX=$!kJCgkJ zmQXwB9Rpv3+{2QIE1KoRD1W?)|2!Nc322urX1)+`Nwr#=1h{xz zjOIJ|3ayRe?P-1|=g@qgKf*ODKwq9?&FwF>xf|FddE5={qvE!aqRhu_V|8V7+r&P* z+)eGHkJ~`?k)K<3I08Ok30rkh>s*|ggSk4mo;4rh(Tt$xS>$GrcW^$oIC8H(tvkij zzHt=ipKnq_cA!#xyjNMu)#WY#XY(3TP98_Er<)^@HNG z$8p4!wl5jv9b6otXQXoGeAvln&ZhX>)E1mL_FGrW;O}Ie0p$Cp=b&HLic_&4I)>u2 ze-He<_92Au;Vz29!3*iS*dF4h*;(ijrA@NuJpDk13@ix_wJwV>Uv27gq5RCdwSv53 zzgd>P#;d35^CH0A%qGd>Zf+m_MYx>1g?%03ZfPIwA{=C&{DiyIl}&gn`|NVJRw7Oj z{#kwGCtP&-g_8fBo?*%mGiiMgBG3Orc;T;D2y!yL^2|SS;O6YMp-Ns%S_gT4_beu& z?OQ?%$0|(37V8apE5Q!P^ECCa5LiL_u`ThH?u7Xo`p%a#F1zNv6Q0+Og6(9_7>w5{ z9E6J;pC}=F?c`6B5&q%<5g5UdK=m${K$u>y4C(RO!&|-rF|0Rzze5|@7VdNLe*gm`59qglz+m7m^INXly zj^fDYw1@9m5G5A=zu*q=W#=QhI`gE*;WG!6r-dW`1;XR>; z-KqhOtQ-p{$aCf;4s%p*5B(tp$j7^*fSU&gQ}U|Y0(xZTtI%UsZwouv(#2G8@)icS z*6}XTD+^Dd!m@lFa2fVL+Nx~ulDOYin&#!T-)cji+Zgx^03aEI6=dEA}sqvE!+qRhvwuC8ouyVz%!JJde!FEvh%+A@-e{&5{1(5qx50sz$~34Skbad)ya# z99*0ZfseHs^zA=AV-9Lm@=n9JqAC?BE8}3YXA%7>hFP;cII)r)AYZ?oOq@)i-x0Gq z)#rK3sD%zMP)*4>miv!6%2S(9n2-~KT_;9_ELS_?HAQxV=8N%E~{fI~ig6BS`E zF1Bu?ZT6?Fz_q?HEB;P?-%Wn2nu6w=@H^RCn0Dqwb$U08|DYzrdfT7uoJx1X+PcE@ zcj@hS#oXaGNgj8EeN^0bRh0R-jZ{}Qx83Zs%iY~R`nZi!ANjeJ)pXb(&jlb3<&7qk zlf5{b)E#llQqqf1diGB!KK$~x&MGeC`C^b41%^{D^UCXGl2R_3)hHe=YvqExwsSjh zzBDJ|)2uuOYeWp|3g4PX#!_)^N8cBA^qVpN(mEe+Q3I4_G32fHCkby^8G5vi-d_Y9 z&<~%>V_6VC`k6VHuW<15v5@DHtuZfgu_T?F&Ih-;tbv?vy`K+U)^15f`11r@n@{h9 z6H$eB&a5TgU6tfb@mNTt&Q11s=c@G{8KXK+b)+DP$znNhc2(lZj9yUoH zcTfB16yXZ?c679T9pdg~AMGOC+dlaT@1w44!VUZEa=Vp?Q-u4fkNkw2nfr}X#POU- z1Jzk%{({9V%cjA8IX``)p2Fpyzf+?Hh+f7n(tkEniC_Pa#IKcGk*EfftfQTNXhyPV`vXa;Ke*&?*{6lhB@{4r$ z@p=+%eLl3`98+?$hLir`*AQ&1P$Y0(CkP2qyFQhIKMCC-JN8?b-(f*$Vjc)Lz4psR z2Z`{lgDENG%P~ROBNF9Aq86)aK))!^eXPPQz8fzjiga|BDVxz(v$bCiV0`OJN7yf4 zK7)U(YqdtHzcY{8TOw|cO_Kd%z4lQBbALsdFPI0YE1TPa_SxkgWFLLp4ptxexs{)` zQ^ETYdyf_j|zW+`y#qz5s!7XRBmFdVkxH$eNCA1Xn!G#rg90{B!zCdfCcK{Np;CHAf zZoCHljy?LZF8Di{Xd99XPj(tX#wz?t+sOyolb(%d5wt9OISS^toxveb_zSHKFD_k& zsgv(S+gx(zi($$S`i1e(!-f`t-^|kez^!?cni|b3FG?Sef&8o2kay;5*4sNu;p|K! zo7*8aNgnr5`>41bW}p4s4p&z;wmn z9#=Sjc&CWMtwk-#r>kz`B`wP<@8I^&jGT!(t{zoM==ZXU z($CpfDyr8r)5}lEw^xgY^Q(B?Z_eG9SK(qpR@fs~C&ueV;(;rd!p#g7$WGB2+&X^H zxBP~+#cEH&^H#?XyzqM9dMy1;iO`dB=pRBmfO4`)`u~HAUys?l95FUY9``8wsJI=i zDD%bb7MxEkBerdM9B32dk2%`&<1D!g3Z13AvYdA?YBRcj&nL|=X$#h*7F4n1ss z0>q7jYc;OXdxGyfQL)=bji5O6j9%?}kej%UJ%)H@)A9q?LR+9|7SrgZavAlOd_STW z;@0X^3p-7EwfT6i&)Y4e_|MdUine4p(2kgPxfey$=0WvHldNyG-OY z{W9ed9=#kmYvH1=I;@~aiCXPdh-Yd3zVW1+4)y;{nh!q_ToU`8b z6)sN9rvzL26^e`xFYev?xIy1O6=haq(Q4J|ko>kKkk_-Ns;l&{g@0k9R^l8el6tdU zNYqxR0+hJC`n5AFP=t9D6-hbd2jqE+Bp{eCe?$rP+e{Q;>%#`R>&nr967`8cC#vzR z)bp`Gg-*{y`Dyxb9DX(kdPFGMX=Q1FqH1OMfd$G#b|O)m?~cKKmWQ@RELjFPTbK~? z=DcX3#^E)mV_fsE;xymtYiS37(>A;4a0g9|wf+c-Fnc--_B;L;eG?J>`%}B4o@tZh zanG`kDwt<0%6!2*M_t+6&b7}j_dNUP<95FK$j>d$O}{xUQvTXf*~m_YQ>W_1t$)+E zP<}5R2O<|NIH9g~lT>SJAf*im)Rustkhd*6eNYt%GaryhS*q9)d>P`vGnxr#o1& zuqJ}Qy5RxN7G;MX<|+j)wDz>cY0kX^TyE)3ce>%pQB>J*de6X$rWfF>i;Tv}y8{W< zJgW^;^I`qgBAEF%+D>!iMepyJhj{x64VIQ6nv&XwI!H2X#bfA^UpW=)rhk&Y;MTCm z%=($;d-*7Wme;3m%CJFSq1S#68>7b8BD2z3@XCuU`$j38vut(|USN~taWAxwirYns zG9R~#)s@ZdFZS8xUSc18-2SRQ@^h;P-zlqfS~J(7$nc4w$5qjjbJLemxG3RmgyuDq z3fEUnK@m1@OoGp}(sz$4+{(SWypq>1(AK%0cg<;qvn2IUgtdU*sE9qPd063Ep3f-4 zd}?dpa>yMjk{NrW=&^_N@|#vXBZ|B;uKrUr#oP3{czMsU)4|!i0`yW}w`5hhJ&vrH zljha+HeQzDB`%=|JNjks??}{Ex^pPP4lXv;#9G$>V~nqTjJ(+36)C;l=sHmWXrFA<8borcIK^z0^J`ZkH*_eB3TqS2njR z?6b?g(mwjQU8O$qb1OnNV4Z0h#-e!2=m09VUGD%l#h9O!P1+k5aM5}waJI5Oim;Wh z3g%>b<*neP!AhR@UxF8v99*7076d)ZlTcAQ91EXV*&m|_o9!l25!>7mI4e;vRLNV> z0ce2qjM20UkT@xdt!0b=hdjamfb*g!fU^v*U?+P@MNi*z6l;Oy?vLWhBXSQ@dgZ4`A*~+nJO*Tf-|o4 z;_TZ8=rO(XHM6%tJhRgDd)Pkt;=8EX?r{4_fP1w~lE=NqKKhGrIrm!oI>f!sKH5ci zy?yc%zCm5tgm1LZF83xS;uPV{>LWkl`T_dUDP8{ugvh`lSgiVk0+@)Fa1JHv0eOfJ zFV6CJZKfQ@4yC|imA70-M2 zuL7iZ#o-v`H(k29>o1)AY%-hwK&Fv2R>~in4 zk3Md9sgL4to5K6^9)-8#j#tjE3E${P3sK{>DnB(?2cHu@B@zvUHf~%)Ws9|BCK@uc z^hz`k`qK_bNaDyT$n*3$A#Zv0$bp@y!TS9nH9)TA;80$rFFELErci_OWH>cg2fZ%` zC0j-f*{-kPguVWRHN~@g|Eq(`<)V~5`ta+t7P9t%e%5&^C7&}5VUJd{B^nuh(`;~V zZMz8Ee32Cm6MIw}@;uQhG#9#ezSf7d6zug+6X)ySW4>%#H=)MmE&j&1{PIUMe9Yth z?`ERYkp5`zThMu%KQ8u@`ZIV3hJ@!#?yH`=><946Avbo)FpIz=) z`{?8LfcnVKE&C<5EPN8WkBVXQ&$tuT{(e$g;Ua{#1m&8#D4u+W3q{z$&D^CyU<`c<7$tNAYqHLD^LjZT4T#AJq|9b)6X1eR}#~j|Jsfo zBQPdEA}z4q3)$Pi9j~r!r?~Oz*IQH0 z*QWjopIcwYBPVNtso;C-!+H2h`*Z_1zq<|gSew&QZv7fbk0$c;ry6mEURKk+IDfJZ zHAQsz8#RI->wv%Gx#

ExL!C%J#p#zrmEYvn-B`=!y@RL`tmq`FX>04lKMpQdE~BQp%P726&K}N%yn~ChU+f}$ R+$PE6K4Bm2BHXXk{{hT!cc}mX literal 0 HcmV?d00001 diff --git a/tests/test_datasets/test_body3d_dataset.py b/tests/test_datasets/test_body3d_dataset.py index a9cd94ee4d..610a5b951a 100644 --- a/tests/test_datasets/test_body3d_dataset.py +++ b/tests/test_datasets/test_body3d_dataset.py @@ -1,4 +1,5 @@ # Copyright (c) OpenMMLab. All rights reserved. +import copy import tempfile import numpy as np @@ -327,7 +328,6 @@ def test_body3dmview_direct_panoptic_dataset(): dataset_info=dataset_info, test_mode=False) - import copy gt_num = test_dataset.db_size // test_dataset.num_cameras results = [] for i in range(gt_num): @@ -345,3 +345,181 @@ def test_body3dmview_direct_panoptic_dataset(): results.append(dict(pose_3d=gt_pose, sample_id=[i])) _ = test_dataset.evaluate(results, metric=['mAP', 'mpjpe']) + + +def test_body3dmview_direct_campus_dataset(): + # Test Mview-Campus dataset + dataset = 'Body3DMviewDirectCampusDataset' + dataset_class = DATASETS.get(dataset) + dataset_info = Config.fromfile( + 'configs/_base_/datasets/campus.py').dataset_info + space_size = [12000.0, 12000.0, 12000.0] + space_center = [3000.0, 4500.0, 1000.0] + cube_size = [80, 80, 20] + + data_root = 'tests/data/campus' + train_data_cfg = dict( + space_size=space_size, + space_center=space_center, + cube_size=cube_size, + image_size=[800, 640], + heatmap_size=[[200, 160]], + num_joints=17, + cam_list=[0, 1, 2], + num_cameras=3, + frame_range=list(range(0, 3)), + width=360, + height=288, + root_id=[11, 12], + max_nposes=3, + min_nposes=1, + num_train_samples=10, + maximum_person=10, + cam_file=f'{data_root}/calibration_campus.json', + train_pose_db_file=f'{data_root}/panoptic_training_pose.pkl', + test_pose_db_file=f'{data_root}/pred_campus_maskrcnn_hrnet_coco.pkl', + gt_pose_db_file=f'{data_root}/actorsGT.mat', + ) + + test_data_cfg = dict( + space_size=space_size, + space_center=space_center, + cube_size=cube_size, + image_size=[800, 640], + heatmap_size=[[200, 160]], + num_joints=17, + cam_list=[0, 1, 2], + num_cameras=3, + frame_range=list(range(0, 3)), + width=360, + height=288, + maximum_person=10, + cam_file=f'{data_root}/calibration_campus.json', + train_pose_db_file=f'{data_root}/panoptic_training_pose.pkl', + test_pose_db_file=f'{data_root}/pred_campus_maskrcnn_hrnet_coco.pkl', + gt_pose_db_file=f'{data_root}/actorsGT.mat', + ) + + train_dataset = dataset_class( + ann_file=None, + img_prefix=data_root, + data_cfg=train_data_cfg, + pipeline=[], + dataset_info=dataset_info, + test_mode=False) + + test_dataset = dataset_class( + ann_file=None, + img_prefix=data_root, + data_cfg=test_data_cfg, + pipeline=[], + dataset_info=dataset_info, + test_mode=True) + + assert len(train_dataset) == train_data_cfg['num_train_samples'] + + # test the length of dataset + gt_num = len(test_dataset) + assert gt_num == len(test_data_cfg['frame_range']) + results = [] + for i in range(gt_num): + # test the __getitem__ method of dataset + _ = copy.deepcopy(test_dataset[i]) + # due to the empty pipeline, each sample is a dict of multi-view infos + pose_3d = np.ones( + (1, test_data_cfg['maximum_person'], test_dataset.num_joints, 5)) + + # due to the complex process, we do not use the gt + # coco pose converting from shelf poses + results.append(dict(pose_3d=pose_3d, sample_id=[i])) + + _ = test_dataset.evaluate(results, metric='pcp') + + +def test_body3dmview_direct_shelf_dataset(): + # Test Mview-Shelf dataset + dataset = 'Body3DMviewDirectShelfDataset' + dataset_class = DATASETS.get(dataset) + dataset_info = Config.fromfile( + 'configs/_base_/datasets/shelf.py').dataset_info + space_size = [8000, 8000, 2000] + space_center = [450, -320, 800] + cube_size = [48, 48, 12] + + data_root = 'tests/data/shelf' + train_data_cfg = dict( + space_size=space_size, + space_center=space_center, + cube_size=cube_size, + image_size=[800, 608], + heatmap_size=[[200, 152]], + num_joints=17, + cam_list=[0, 1, 2, 3, 4], + num_cameras=5, + frame_range=list(range(0, 3)), + width=1032, + height=776, + root_id=[11, 12], + max_nposes=3, + min_nposes=1, + num_train_samples=10, + maximum_person=10, + cam_file=f'{data_root}/calibration_shelf.json', + train_pose_db_file=f'{data_root}/panoptic_training_pose.pkl', + test_pose_db_file=f'{data_root}/pred_shelf_maskrcnn_hrnet_coco.pkl', + gt_pose_db_file=f'{data_root}/actorsGT.mat', + ) + + test_data_cfg = dict( + space_size=space_size, + space_center=space_center, + cube_size=cube_size, + image_size=[800, 608], + heatmap_size=[[200, 152]], + num_joints=17, + cam_list=[0, 1, 2, 3, 4], + num_cameras=5, + frame_range=list(range(0, 3)), + width=1032, + height=776, + maximum_person=10, + cam_file=f'{data_root}/calibration_shelf.json', + train_pose_db_file=f'{data_root}/panoptic_training_pose.pkl', + test_pose_db_file=f'{data_root}/pred_shelf_maskrcnn_hrnet_coco.pkl', + gt_pose_db_file=f'{data_root}/actorsGT.mat', + ) + + train_dataset = dataset_class( + ann_file=None, + img_prefix=data_root, + data_cfg=train_data_cfg, + pipeline=[], + dataset_info=dataset_info, + test_mode=False) + + test_dataset = dataset_class( + ann_file=None, + img_prefix=data_root, + data_cfg=test_data_cfg, + pipeline=[], + dataset_info=dataset_info, + test_mode=True) + + assert len(train_dataset) == train_data_cfg['num_train_samples'] + + # test the length of dataset + gt_num = len(test_dataset) + assert gt_num == len(test_data_cfg['frame_range']) + results = [] + for i in range(gt_num): + # test the __getitem__ method of dataset + _ = copy.deepcopy(test_dataset[i]) + # due to the empty pipeline, each sample is a dict of multi-view infos + pose_3d = np.ones( + (1, test_data_cfg['maximum_person'], test_dataset.num_joints, 5)) + + # due to the complex process, we do not use the gt + # coco pose converting from shelf poses + results.append(dict(pose_3d=pose_3d, sample_id=[i])) + + _ = test_dataset.evaluate(results, metric='pcp') diff --git a/tests/test_pipelines/test_pose3d_transform.py b/tests/test_pipelines/test_pose3d_transform.py index b6a52d9d05..f111f89e1a 100644 --- a/tests/test_pipelines/test_pose3d_transform.py +++ b/tests/test_pipelines/test_pose3d_transform.py @@ -334,3 +334,114 @@ def test_3d_heatmap_generation(): results_3d = pipeline(results) assert results_3d['target'].shape == (3, 64, 64, 64) assert results_3d['target_weight'].shape == (3, 1) + + +def test_voxel3D_heatmap_generation(): + heatmap_size = [200, 160] + cube_size = [8, 8, 2] + ann_info = dict( + image_size=np.array([800, 640]), + heatmap_size=np.array([heatmap_size]), + num_joints=17, + num_scales=1, + space_size=[12000.0, 12000.0, 2000.0], + space_center=[3000.0, 4500.0, 1000.0], + cube_size=cube_size) + + results = dict( + joints_3d=np.ones([2, 17, 3]), + joints_3d_visible=np.ones([2, 17, 3]), + ann_info=ann_info) + + # test single joint index + joint_indices = [[11, 12]] + pipeline = Compose([ + dict( + type='GenerateVoxel3DHeatmapTarget', + sigma=200.0, + joint_indices=joint_indices, + ), + ]) + results_ = pipeline(results) + assert results_['targets_3d'].shape == (8, 8, 2) + + # test multiple joint indices + joint_indices = [0, 8, 6] + pipeline = Compose([ + dict( + type='GenerateVoxel3DHeatmapTarget', + sigma=200.0, + joint_indices=joint_indices, + ), + ]) + results_ = pipeline(results) + assert results_['targets_3d'].shape == (3, 8, 8, 2) + + +def test_input_heatmap_generation(): + heatmap_size = [200, 160] + ann_info = dict( + image_size=np.array([800, 640]), + heatmap_size=np.array([heatmap_size]), + num_joints=17, + num_scales=1, + ) + + results = dict( + joints=np.zeros([2, 17, 3]), + joints_visible=np.ones([2, 17, 3]), + ann_info=ann_info) + + pipeline = Compose([ + dict( + type='GenerateInputHeatmaps', + item='joints', + visible_item='joints_visible', + obscured=0.0, + from_pred=False, + sigma=3, + scale=1.0, + base_size=96, + target_type='gaussian', + heatmap_cfg=dict( + base_scale=0.9, + offset=0.03, + threshold=0.6, + extra=[ + dict(joint_ids=[7, 8], scale_factor=0.5, threshold=0.1), + dict( + joint_ids=[9, 10], + scale_factor=0.2, + threshold=0.1, + ), + dict( + joint_ids=[ + 0, 1, 2, 3, 4, 5, 6, 11, 12, 13, 14, 15, 16 + ], + scale_factor=0.5, + threshold=0.05) + ])) + ]) + results_ = pipeline(results) + assert results_['input_heatmaps'][0].shape == (17, heatmap_size[1], + heatmap_size[0]) + + +def test_affine_joints(): + ann_info = dict(image_size=np.array([800, 640])) + + results = dict( + center=np.array([180, 144]), + scale=np.array([360, 288], dtype=np.float32), + rotation=0.0, + joints=np.ones((3, 17, 2)), + joints_visible=np.ones((3, 17, 2)), + ann_info=ann_info) + + pipeline = Compose([ + dict( + type='AffineJoints', item='joints', visible_item='joints_visible') + ]) + results_ = pipeline(results) + assert results_['joints'].shape == (3, 17, 2) + assert results_['joints_visible'].shape == (3, 17, 2) From 0486755c81109cc5e63ed66b748dcc1e9dcba8f5 Mon Sep 17 00:00:00 2001 From: liqikai Date: Fri, 11 Mar 2022 16:31:14 +0800 Subject: [PATCH 04/19] unittest for dataset --- tests/test_datasets/test_body3d_dataset.py | 87 +++++++++++++++++++++- 1 file changed, 83 insertions(+), 4 deletions(-) diff --git a/tests/test_datasets/test_body3d_dataset.py b/tests/test_datasets/test_body3d_dataset.py index 610a5b951a..1b41c20e21 100644 --- a/tests/test_datasets/test_body3d_dataset.py +++ b/tests/test_datasets/test_body3d_dataset.py @@ -3,6 +3,7 @@ import tempfile import numpy as np +import pytest from mmcv import Config from mmpose.datasets import DATASETS @@ -355,7 +356,7 @@ def test_body3dmview_direct_campus_dataset(): 'configs/_base_/datasets/campus.py').dataset_info space_size = [12000.0, 12000.0, 12000.0] space_center = [3000.0, 4500.0, 1000.0] - cube_size = [80, 80, 20] + cube_size = [8, 8, 2] data_root = 'tests/data/campus' train_data_cfg = dict( @@ -400,12 +401,25 @@ def test_body3dmview_direct_campus_dataset(): gt_pose_db_file=f'{data_root}/actorsGT.mat', ) + # test when dataset_info is None train_dataset = dataset_class( ann_file=None, img_prefix=data_root, data_cfg=train_data_cfg, pipeline=[], - dataset_info=dataset_info, + dataset_info=None, + test_mode=False) + + # test different `image_size` + train_data_cfg_copy = copy.deepcopy(train_data_cfg) + train_data_cfg_copy['image_size'] = [800, 600] + train_data_cfg_copy['heatmap_size'] = [[200, 150]] + _ = dataset_class( + ann_file=None, + img_prefix=data_root, + data_cfg=train_data_cfg_copy, + pipeline=[], + dataset_info=None, test_mode=False) test_dataset = dataset_class( @@ -416,7 +430,33 @@ def test_body3dmview_direct_campus_dataset(): dataset_info=dataset_info, test_mode=True) + # test the length of dataset assert len(train_dataset) == train_data_cfg['num_train_samples'] + gt_num = len(train_dataset) + results = [] + for i in range(gt_num): + # test the __getitem__ method of dataset + _ = copy.deepcopy(train_dataset[i]) + # due to the empty pipeline, each sample is a dict of multi-view infos + pose_3d = np.ones( + (1, train_data_cfg['maximum_person'], train_dataset.num_joints, 5)) + + # due to the complex process, we do not use the gt + # coco pose converting from shelf poses + results.append(dict(pose_3d=pose_3d, sample_id=[i])) + + _ = train_dataset.evaluate(results, metric='pcp') + + # test unsupported metric + with pytest.raises(ValueError): + _ = train_dataset.evaluate(results, metric='mpjpe') + + # test res_folder in `evaluate` + with tempfile.TemporaryDirectory() as tmpdir: + _ = train_dataset.evaluate(results, res_folder=tmpdir, metric='pcp') + + # test recall_threshold + _ = train_dataset.evaluate(results, recall_threshold=50000, metric='pcp') # test the length of dataset gt_num = len(test_dataset) @@ -444,7 +484,7 @@ def test_body3dmview_direct_shelf_dataset(): 'configs/_base_/datasets/shelf.py').dataset_info space_size = [8000, 8000, 2000] space_center = [450, -320, 800] - cube_size = [48, 48, 12] + cube_size = [8, 8, 2] data_root = 'tests/data/shelf' train_data_cfg = dict( @@ -489,12 +529,25 @@ def test_body3dmview_direct_shelf_dataset(): gt_pose_db_file=f'{data_root}/actorsGT.mat', ) + # test when dataset_info is None train_dataset = dataset_class( ann_file=None, img_prefix=data_root, data_cfg=train_data_cfg, pipeline=[], - dataset_info=dataset_info, + dataset_info=None, + test_mode=False) + + # test different `image_size` + train_data_cfg_copy = copy.deepcopy(train_data_cfg) + train_data_cfg_copy['image_size'] = [800, 400] + train_data_cfg_copy['heatmap_size'] = [[200, 100]] + _ = dataset_class( + ann_file=None, + img_prefix=data_root, + data_cfg=train_data_cfg_copy, + pipeline=[], + dataset_info=None, test_mode=False) test_dataset = dataset_class( @@ -505,7 +558,33 @@ def test_body3dmview_direct_shelf_dataset(): dataset_info=dataset_info, test_mode=True) + # test the length of dataset assert len(train_dataset) == train_data_cfg['num_train_samples'] + gt_num = len(train_dataset) + results = [] + for i in range(gt_num): + # test the __getitem__ method of dataset + _ = copy.deepcopy(train_dataset[i]) + # due to the empty pipeline, each sample is a dict of multi-view infos + pose_3d = np.ones( + (1, train_data_cfg['maximum_person'], train_dataset.num_joints, 5)) + + # due to the complex process, we do not use the gt + # coco pose converting from shelf poses + results.append(dict(pose_3d=pose_3d, sample_id=[i])) + + _ = train_dataset.evaluate(results, metric='pcp') + + # test unsupported metric + with pytest.raises(ValueError): + _ = train_dataset.evaluate(results, metric='mpjpe') + + # test res_folder in `evaluate` + with tempfile.TemporaryDirectory() as tmpdir: + _ = train_dataset.evaluate(results, res_folder=tmpdir, metric='pcp') + + # test recall_threshold + _ = train_dataset.evaluate(results, recall_threshold=50000, metric='pcp') # test the length of dataset gt_num = len(test_dataset) From 5efef39719d8db13aa1a6e5d8a96061d1320a21e Mon Sep 17 00:00:00 2001 From: liqikai Date: Fri, 11 Mar 2022 18:26:23 +0800 Subject: [PATCH 05/19] add more unittests for pose3d_transform --- tests/test_pipelines/test_pose3d_transform.py | 99 +++++++++++++------ 1 file changed, 68 insertions(+), 31 deletions(-) diff --git a/tests/test_pipelines/test_pose3d_transform.py b/tests/test_pipelines/test_pose3d_transform.py index f111f89e1a..f7ac7ae776 100644 --- a/tests/test_pipelines/test_pose3d_transform.py +++ b/tests/test_pipelines/test_pose3d_transform.py @@ -392,37 +392,67 @@ def test_input_heatmap_generation(): joints_visible=np.ones([2, 17, 3]), ann_info=ann_info) - pipeline = Compose([ - dict( - type='GenerateInputHeatmaps', - item='joints', - visible_item='joints_visible', - obscured=0.0, - from_pred=False, - sigma=3, - scale=1.0, - base_size=96, - target_type='gaussian', - heatmap_cfg=dict( - base_scale=0.9, - offset=0.03, - threshold=0.6, - extra=[ - dict(joint_ids=[7, 8], scale_factor=0.5, threshold=0.1), - dict( - joint_ids=[9, 10], - scale_factor=0.2, - threshold=0.1, - ), - dict( - joint_ids=[ - 0, 1, 2, 3, 4, 5, 6, 11, 12, 13, 14, 15, 16 - ], - scale_factor=0.5, - threshold=0.05) - ])) - ]) - results_ = pipeline(results) + pipeline = dict( + type='GenerateInputHeatmaps', + item='joints', + visible_item='joints_visible', + obscured=0.0, + from_pred=False, + sigma=3, + scale=1.0, + base_size=96, + target_type='gaussian', + heatmap_cfg=dict( + base_scale=0.9, + offset=0.03, + threshold=0.6, + extra=[ + dict(joint_ids=[7, 8], scale_factor=0.5, threshold=0.1), + dict( + joint_ids=[9, 10], + scale_factor=0.2, + threshold=0.1, + ), + dict( + joint_ids=[0, 1, 2, 3, 4, 5, 6, 11, 12, 13, 14, 15, 16], + scale_factor=0.5, + threshold=0.05) + ])) + + pipelines = Compose([pipeline]) + results_ = pipelines(results) + assert results_['input_heatmaps'][0].shape == (17, heatmap_size[1], + heatmap_size[0]) + + # test `obscured` + pipeline_copy = copy.deepcopy(pipeline) + pipeline_copy['obscured'] = 0.5 + pipelines = Compose([pipeline]) + results_ = pipelines(results) + assert results_['input_heatmaps'][0].shape == (17, heatmap_size[1], + heatmap_size[0]) + + # test `heatmap_cfg` + pipeline_copy = copy.deepcopy(pipeline) + pipeline_copy['heatmap_cfg'] = None + pipelines = Compose([pipeline]) + results_ = pipelines(results) + assert results_['input_heatmaps'][0].shape == (17, heatmap_size[1], + heatmap_size[0]) + + # test `from_pred` + pipeline_copy = copy.deepcopy(pipeline) + pipeline_copy['from_pred'] = True + pipelines = Compose([pipeline]) + results_ = pipelines(results) + assert results_['input_heatmaps'][0].shape == (17, heatmap_size[1], + heatmap_size[0]) + # test `from_pred` & `scale` + pipeline_copy = copy.deepcopy(pipeline) + pipeline_copy['from_pred'] = True + pipeline_copy['scale'] = None + pipelines = Compose([pipeline]) + results_ = pipelines(results) assert results_['input_heatmaps'][0].shape == (17, heatmap_size[1], heatmap_size[0]) @@ -445,3 +475,10 @@ def test_affine_joints(): results_ = pipeline(results) assert results_['joints'].shape == (3, 17, 2) assert results_['joints_visible'].shape == (3, 17, 2) + + # test `joints_visible` is zero + results_copy = copy.deepcopy(results) + results_copy['joints_visible'] = np.zeros((3, 17, 2)) + results_ = pipeline(results) + assert results_['joints'].shape == (3, 17, 2) + assert results_['joints_visible'].shape == (3, 17, 2) From efdc6b0a8e19b87bb681808ae5b450c56fa73b44 Mon Sep 17 00:00:00 2001 From: liqikai Date: Wed, 23 Mar 2022 19:53:02 +0800 Subject: [PATCH 06/19] move several general functions to base class --- .../kpt_3d_mview_rgb_img_direct_dataset.py | 115 +++++++++++++++++- .../body3d_mview_direct_campus_dataset.py | 113 +---------------- .../body3d_mview_direct_panoptic_dataset.py | 29 ----- .../body3d_mview_direct_shelf_dataset.py | 112 +---------------- 4 files changed, 115 insertions(+), 254 deletions(-) diff --git a/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py b/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py index ca86f9e092..0f1a7c30c2 100644 --- a/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py +++ b/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py @@ -1,9 +1,13 @@ # Copyright (c) OpenMMLab. All rights reserved. import copy +import json +import os.path as osp +import pickle from abc import ABCMeta, abstractmethod -import json_tricks as json +import json_tricks as json_ import numpy as np +import scipy.io as scio from torch.utils.data import Dataset from mmpose.datasets import DatasetInfo @@ -124,7 +128,7 @@ def _write_keypoint_results(keypoints, res_file): """Write results into a json file.""" with open(res_file, 'w') as f: - json.dump(keypoints, f, sort_keys=True, indent=4) + json_.dump(keypoints, f, sort_keys=True, indent=4) def __len__(self): """Get the size of the dataset.""" @@ -140,3 +144,110 @@ def __getitem__(self, idx): results[c] = result return self.pipeline(results) + + @staticmethod + def _sort_and_unique_outputs(outputs, key='sample_id'): + """sort outputs and remove the repeated ones.""" + outputs = sorted(outputs, key=lambda x: x[key]) + num_outputs = len(outputs) + for i in range(num_outputs - 1, 0, -1): + if outputs[i][key] == outputs[i - 1][key]: + del outputs[i] + + return outputs + + def _get_scale(self, raw_image_size): + heatmap_size = self.ann_info['heatmap_size'] + image_size = self.ann_info['image_size'] + assert heatmap_size[0][0] / heatmap_size[0][1] \ + == image_size[0] / image_size[1] + w, h = raw_image_size + w_resized, h_resized = image_size + if w / w_resized < h / h_resized: + w_pad = h / h_resized * w_resized + h_pad = h + else: + w_pad = w + h_pad = w / w_resized * h_resized + + scale = np.array([w_pad, h_pad], dtype=np.float32) + + return scale + + @staticmethod + def rotate_points(points, center, rot_rad): + """ + :param points: N*2 + :param center: 2 + :param rot_rad: scalar + :return: N*2 + """ + rot_rad = rot_rad * np.pi / 180.0 + rotate_mat = np.array([[np.cos(rot_rad), -np.sin(rot_rad)], + [np.sin(rot_rad), + np.cos(rot_rad)]]) + center = center.reshape(2, 1) + points = points.T + points = rotate_mat.dot(points - center) + center + + return points.T + + @staticmethod + def calc_bbox(pose, pose_vis): + index = pose_vis[:, 0] > 0 + bbox = [ + np.min(pose[index, 0]), + np.min(pose[index, 1]), + np.max(pose[index, 0]), + np.max(pose[index, 1]) + ] + + return np.array(bbox) + + def _get_cam(self, calib): + """Get camera parameters. + + Returns: Camera parameters. + """ + cameras = {} + for id, cam in calib.items(): + sel_cam = {} + # note the transpose operation different from from VoxelPose + sel_cam['R'] = np.array(cam['R'], dtype=np.float32).T + sel_cam['T'] = np.array(cam['T'], dtype=np.float32) + + sel_cam['k'] = np.array(cam['k'], dtype=np.float32) + sel_cam['p'] = np.array(cam['p'], dtype=np.float32) + + sel_cam['f'] = [[cam['fx']], [cam['fy']]] + sel_cam['c'] = [[cam['cx']], [cam['cy']]] + + cameras[id] = sel_cam + + return cameras + + def _load_files(self): + """load related db files.""" + assert osp.exists(self.cam_file), f'camera calibration file ' \ + f"{self.cam_file} doesn't exist, please check again" + with open(self.cam_file) as cfile: + calib = json.load(cfile) + self.cameras = self._get_cam(calib) + + assert osp.exists(self.train_pose_db_file), f'train_pose_db_file ' \ + f"{self.train_pose_db_file} doesn't exist, please check again" + with open(self.train_pose_db_file, 'rb') as pfile: + self.train_pose_db = pickle.load(pfile) + + assert osp.exists(self.test_pose_db_file), f'test_pose_db_file ' \ + f"{self.test_pose_db_file} doesn't exist, please check again" + with open(self.test_pose_db_file, 'rb') as pfile: + self.test_pose_db = pickle.load(pfile) + + assert osp.exists(self.gt_pose_db_file), f'gt_pose_db_file ' \ + f"{self.gt_pose_db_file} doesn't exist, please check again" + gt = scio.loadmat(self.gt_pose_db_file) + self.gt_pose_db = np.array(np.array( + gt['actor3D'].tolist()).tolist()).squeeze() + + self.num_persons = len(self.gt_pose_db) diff --git a/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py b/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py index 73d0b0cea6..e8fc96cd15 100644 --- a/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py +++ b/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py @@ -1,9 +1,7 @@ # Copyright (c) OpenMMLab. All rights reserved. import copy # import glob -import json import os.path as osp -import pickle import random import tempfile import warnings @@ -11,7 +9,6 @@ import mmcv import numpy as np -import scipy.io as scio from mmcv import Config from mmpose.core.camera import SimpleCamera @@ -108,7 +105,6 @@ def load_config(self, data_cfg): self.need_camera_param = True self.frame_range = data_cfg['frame_range'] - self.width = data_cfg.get('width', 360) self.height = data_cfg.get('height', 288) self.center = np.array((self.width / 2, self.height / 2), @@ -137,73 +133,7 @@ def load_config(self, data_cfg): self.gt_pose_db_file = data_cfg.get( 'gt_pose_db_file', osp.join(self.img_prefix, 'actorsGT.mat')) - self._get_db() - - def _get_scale(self, raw_image_size): - heatmap_size = self.ann_info['heatmap_size'] - image_size = self.ann_info['image_size'] - assert heatmap_size[0][0] / heatmap_size[0][1] \ - == image_size[0] / image_size[1] - w, h = raw_image_size - w_resized, h_resized = image_size - if w / w_resized < h / h_resized: - w_pad = h / h_resized * w_resized - h_pad = h - else: - w_pad = w - h_pad = w / w_resized * h_resized - - scale = np.array([w_pad, h_pad], dtype=np.float32) - - return scale - - def _get_db(self): - """load related db files.""" - assert osp.exists(self.cam_file), f'camera calibration file ' \ - f"{self.cam_file} doesn't exist, please check again" - with open(self.cam_file) as cfile: - calib = json.load(cfile) - self.cameras = self._get_cam(calib) - - assert osp.exists(self.train_pose_db_file), f'train_pose_db_file ' \ - f"{self.train_pose_db_file} doesn't exist, please check again" - with open(self.train_pose_db_file, 'rb') as pfile: - self.train_pose_db = pickle.load(pfile) - - assert osp.exists(self.test_pose_db_file), f'test_pose_db_file ' \ - f"{self.test_pose_db_file} doesn't exist, please check again" - with open(self.test_pose_db_file, 'rb') as pfile: - self.test_pose_db = pickle.load(pfile) - - assert osp.exists(self.gt_pose_db_file), f'gt_pose_db_file ' \ - f"{self.gt_pose_db_file} doesn't exist, please check again" - gt = scio.loadmat(self.gt_pose_db_file) - self.gt_pose_db = np.array(np.array( - gt['actor3D'].tolist()).tolist()).squeeze() - - self.num_persons = len(self.gt_pose_db) - - def _get_cam(self, calib): - """Get camera parameters. - - Returns: Camera parameters. - """ - cameras = {} - for id, cam in calib.items(): - sel_cam = {} - # note the transpose operation different from from VoxelPose - sel_cam['R'] = np.array(cam['R'], dtype=np.float32).T - sel_cam['T'] = np.array(cam['T'], dtype=np.float32) - - sel_cam['k'] = np.array(cam['k'], dtype=np.float32) - sel_cam['p'] = np.array(cam['p'], dtype=np.float32) - - sel_cam['f'] = [[cam['fx']], [cam['fy']]] - sel_cam['c'] = [[cam['cx']], [cam['cy']]] - - cameras[id] = sel_cam - - return cameras + self._load_files() def __getitem__(self, idx): """Get the sample given index.""" @@ -415,36 +345,6 @@ def isvalid(self, new_center, bbox, bbox_list): return vis >= 2 and np.max(iou_list) < 0.01 - @staticmethod - def calc_bbox(pose, pose_vis): - index = pose_vis[:, 0] > 0 - bbox = [ - np.min(pose[index, 0]), - np.min(pose[index, 1]), - np.max(pose[index, 0]), - np.max(pose[index, 1]) - ] - - return np.array(bbox) - - @staticmethod - def rotate_points(points, center, rot_rad): - """ - :param points: N*2 - :param center: 2 - :param rot_rad: scalar - :return: N*2 - """ - rot_rad = rot_rad * np.pi / 180.0 - rotate_mat = np.array([[np.cos(rot_rad), -np.sin(rot_rad)], - [np.sin(rot_rad), - np.cos(rot_rad)]]) - center = center.reshape(2, 1) - points = points.T - points = rotate_mat.dot(points - center) + center - - return points.T - def evaluate(self, results, res_folder=None, @@ -614,14 +514,3 @@ def coco2campus3D(coco_pose): campus_pose[13] += head_top return campus_pose - - @staticmethod - def _sort_and_unique_outputs(outputs, key='sample_id'): - """sort outputs and remove the repeated ones.""" - outputs = sorted(outputs, key=lambda x: x[key]) - num_outputs = len(outputs) - for i in range(num_outputs - 1, 0, -1): - if outputs[i][key] == outputs[i - 1][key]: - del outputs[i] - - return outputs diff --git a/mmpose/datasets/datasets/body3d/body3d_mview_direct_panoptic_dataset.py b/mmpose/datasets/datasets/body3d/body3d_mview_direct_panoptic_dataset.py index b5bf92d182..2c6df4b787 100644 --- a/mmpose/datasets/datasets/body3d/body3d_mview_direct_panoptic_dataset.py +++ b/mmpose/datasets/datasets/body3d/body3d_mview_direct_panoptic_dataset.py @@ -135,24 +135,6 @@ def load_config(self, data_cfg): self.root_id = data_cfg.get('root_id', 0) self.max_persons = data_cfg.get('max_num', 10) - def _get_scale(self, raw_image_size): - heatmap_size = self.ann_info['heatmap_size'] - image_size = self.ann_info['image_size'] - assert heatmap_size[0][0] / heatmap_size[0][1] \ - == image_size[0] / image_size[1] - w, h = raw_image_size - w_resized, h_resized = image_size - if w / w_resized < h / h_resized: - w_pad = h / h_resized * w_resized - h_pad = h - else: - w_pad = w - h_pad = w / w_resized * h_resized - - scale = np.array([w_pad, h_pad], dtype=np.float32) - - return scale - def _get_cam(self, seq): """Get camera parameters. @@ -480,14 +462,3 @@ def __getitem__(self, idx): results[c] = result return self.pipeline(results) - - @staticmethod - def _sort_and_unique_outputs(outputs, key='sample_id'): - """sort outputs and remove the repeated ones.""" - outputs = sorted(outputs, key=lambda x: x[key]) - num_outputs = len(outputs) - for i in range(num_outputs - 1, 0, -1): - if outputs[i][key] == outputs[i - 1][key]: - del outputs[i] - - return outputs diff --git a/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py b/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py index 983e477cfb..6ce0652e94 100644 --- a/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py +++ b/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py @@ -1,9 +1,7 @@ # Copyright (c) OpenMMLab. All rights reserved. import copy # import glob -import json import os.path as osp -import pickle import random import tempfile import warnings @@ -11,7 +9,6 @@ import mmcv import numpy as np -import scipy.io as scio from mmcv import Config from mmpose.core.camera import SimpleCamera @@ -136,73 +133,7 @@ def load_config(self, data_cfg): self.gt_pose_db_file = data_cfg.get( 'gt_pose_db_file', osp.join(self.img_prefix, 'actorsGT.mat')) - self._get_db() - - def _get_scale(self, raw_image_size): - heatmap_size = self.ann_info['heatmap_size'] - image_size = self.ann_info['image_size'] - assert heatmap_size[0][0] / heatmap_size[0][1] \ - == image_size[0] / image_size[1] - w, h = raw_image_size - w_resized, h_resized = image_size - if w / w_resized < h / h_resized: - w_pad = h / h_resized * w_resized - h_pad = h - else: - w_pad = w - h_pad = w / w_resized * h_resized - - scale = np.array([w_pad, h_pad], dtype=np.float32) - - return scale - - def _get_db(self): - """load related files.""" - assert osp.exists(self.cam_file), f'camera calibration file ' \ - f"{self.cam_file} doesn't exist, please check again" - with open(self.cam_file) as cfile: - calib = json.load(cfile) - self.cameras = self._get_cam(calib) - - assert osp.exists(self.train_pose_db_file), f'train_pose_db_file ' \ - f"{self.train_pose_db_file} doesn't exist, please check again" - with open(self.train_pose_db_file, 'rb') as pfile: - self.train_pose_db = pickle.load(pfile) - - assert osp.exists(self.test_pose_db_file), f'test_pose_db_file ' \ - f"{self.test_pose_db_file} doesn't exist, please check again" - with open(self.test_pose_db_file, 'rb') as pfile: - self.test_pose_db = pickle.load(pfile) - - assert osp.exists(self.gt_pose_db_file), f'gt_pose_db_file ' \ - f"{self.gt_pose_db_file} doesn't exist, please check again" - gt = scio.loadmat(self.gt_pose_db_file) - self.gt_pose_db = np.array(np.array( - gt['actor3D'].tolist()).tolist()).squeeze() - - self.num_persons = len(self.gt_pose_db) - - def _get_cam(self, calib): - """Get camera parameters. - - Returns: Camera parameters. - """ - cameras = {} - for id, cam in calib.items(): - sel_cam = {} - # note the transpose operation different from from VoxelPose - sel_cam['R'] = np.array(cam['R'], dtype=np.float32).T - sel_cam['T'] = np.array(cam['T'], dtype=np.float32) - - sel_cam['k'] = np.array(cam['k'], dtype=np.float32) - sel_cam['p'] = np.array(cam['p'], dtype=np.float32) - - sel_cam['f'] = [[cam['fx']], [cam['fy']]] - sel_cam['c'] = [[cam['cx']], [cam['cy']]] - - cameras[id] = sel_cam - - return cameras + self._load_files() def __getitem__(self, idx): """Get the sample given index.""" @@ -385,24 +316,6 @@ def get_new_center(center_list): return new_center - @staticmethod - def rotate_points(points, center, rot_rad): - """ - :param points: N*2 - :param center: 2 - :param rot_rad: scalar - :return: N*2 - """ - rot_rad = rot_rad * np.pi / 180.0 - rotate_mat = np.array([[np.cos(rot_rad), -np.sin(rot_rad)], - [np.sin(rot_rad), - np.cos(rot_rad)]]) - center = center.reshape(2, 1) - points = points.T - points = rotate_mat.dot(points - center) + center - - return points.T - @staticmethod def isvalid(bbox, bbox_list): if len(bbox_list) == 0: @@ -422,18 +335,6 @@ def isvalid(bbox, bbox_list): return np.max(iou_list) < 0.01 - @staticmethod - def calc_bbox(pose, pose_vis): - index = pose_vis[:, 0] > 0 - bbox = [ - np.min(pose[index, 0]), - np.min(pose[index, 1]), - np.max(pose[index, 0]), - np.max(pose[index, 1]) - ] - - return np.array(bbox) - def evaluate(self, results, res_folder=None, @@ -614,14 +515,3 @@ def coco2shelf3D(coco_pose, alpha=0.75): shelf_pose[12] = shelf_pose[12] * alpha + head_bottom * (1 - alpha) return shelf_pose - - @staticmethod - def _sort_and_unique_outputs(outputs, key='sample_id'): - """sort outputs and remove the repeated ones.""" - outputs = sorted(outputs, key=lambda x: x[key]) - num_outputs = len(outputs) - for i in range(num_outputs - 1, 0, -1): - if outputs[i][key] == outputs[i - 1][key]: - del outputs[i] - - return outputs From 3dadf6abfce8065da4d3bce15f159f112c9c6cac Mon Sep 17 00:00:00 2001 From: liqikai Date: Wed, 23 Mar 2022 20:00:46 +0800 Subject: [PATCH 07/19] add related docs for campus and shelf dataset --- README.md | 3 +- .../voxelpose/campus/voxelpose_campus.md | 41 +++++++++++ .../voxelpose/campus/voxelpose_campus.yml | 30 ++++++++ .../voxelpose/shelf/voxelpose_shelf.md | 41 +++++++++++ .../voxelpose/shelf/voxelpose_shelf.yml | 30 ++++++++ docs/en/tasks/3d_body_keypoint.md | 70 +++++++++++++++++++ model-index.yml | 2 + 7 files changed, 216 insertions(+), 1 deletion(-) create mode 100644 configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_campus.md create mode 100644 configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_campus.yml create mode 100644 configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.md create mode 100644 configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.yml diff --git a/README.md b/README.md index a3acf8ebfe..5562f7f2cf 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,8 @@ Supported [datasets](https://mmpose.readthedocs.io/en/latest/datasets.html): * [x] [MPII](https://mmpose.readthedocs.io/en/latest/papers/datasets.html#mpii-cvpr-2014) \[[homepage](http://human-pose.mpi-inf.mpg.de/)\] (CVPR'2014) * [x] [Human3.6M](https://mmpose.readthedocs.io/en/latest/papers/datasets.html#human3-6m-tpami-2014) \[[homepage](http://vision.imar.ro/human3.6m/description.php)\] (TPAMI'2014) * [x] [COCO](https://mmpose.readthedocs.io/en/latest/papers/datasets.html#coco-eccv-2014) \[[homepage](http://cocodataset.org/)\] (ECCV'2014) -* [x] [CMU Panoptic](https://mmpose.readthedocs.io/en/latest/papers/datasets.html#cmu-panoptic-iccv-2015) (ICCV'2015) +* [x] [CMU Panoptic](https://mmpose.readthedocs.io/en/latest/papers/datasets.html#cmu-panoptic-iccv-2015) \[[homepage](http://domedb.perception.cs.cmu.edu/)\] (ICCV'2015) +* [x] [Campus/Shelf](https://mmpose.readthedocs.io/en/latest/papers/datasets.html#campus-shelf-cvpr2014) \[[homepage](http://campar.in.tum.de/Chair/MultiHumanPose)\] (CVPR'2014) * [x] [DeepFashion](https://mmpose.readthedocs.io/en/latest/papers/datasets.html#deepfashion-cvpr-2016) \[[homepage](http://mmlab.ie.cuhk.edu.hk/projects/DeepFashion/LandmarkDetection.html)\] (CVPR'2016) * [x] [300W](https://mmpose.readthedocs.io/en/latest/papers/datasets.html#300w-imavis-2016) \[[homepage](https://ibug.doc.ic.ac.uk/resources/300-W/)\] (IMAVIS'2016) * [x] [RHD](https://mmpose.readthedocs.io/en/latest/papers/datasets.html#rhd-iccv-2017) \[[homepage](https://lmb.informatik.uni-freiburg.de/resources/datasets/RenderedHandposeDataset.en.html)\] (ICCV'2017) diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_campus.md b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_campus.md new file mode 100644 index 0000000000..2a45fe5814 --- /dev/null +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_campus.md @@ -0,0 +1,41 @@ + + +

+VoxelPose (ECCV'2020) + +```bibtex +@inproceedings{tumultipose, + title={VoxelPose: Towards Multi-Camera 3D Human Pose Estimation in Wild Environment}, + author={Tu, Hanyue and Wang, Chunyu and Zeng, Wenjun}, + booktitle={ECCV}, + year={2020} +} +``` + +
+ + + +
+Campus (CVPR'2014) + +```bibtex +@inproceedings {belagian14multi, + title = {{3D} Pictorial Structures for Multiple Human Pose Estimation}, + author = {Belagiannis, Vasileios and Amin, Sikandar and Andriluka, Mykhaylo and Schiele, Bernt and Navab + Nassir and Ilic, Slobodan}, + booktitle = {IEEE Computer Society Conference on Computer Vision and Pattern Recognition (CVPR)}, + year = {2014}, + month = {June}, + organization={IEEE} +} +``` + +
+ +Results on Campus dataset. + +| Arch | Actor 1 | Actor 2 | Actor 3 | Average| ckpt | log | +| :--- | :---: | :---: | :---: | :---: | :---: | :---: | +| [prn32_cpn80_res50](/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3.py) | 97.76 | 93.92 | 98.48 | 96.72 | [ckpt](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3-3ecee30e_20220323.pth) | [log](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3.log.json) | +| [prn64_cpn80_res50](/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py) | 97.76 | 93.33 | 98.77 | 96.62| [ckpt](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3-d8decbf7_20220323.pth) | [log](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.log.json) | diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_campus.yml b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_campus.yml new file mode 100644 index 0000000000..d805f00aee --- /dev/null +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_campus.yml @@ -0,0 +1,30 @@ +Collections: +- Name: VoxelPose + Paper: + Title: 'VoxelPose: Towards Multi-Camera 3D Human Pose Estimation in Wild Environment' + URL: https://www.ecva.net/papers/eccv_2020/papers_ECCV/papers/123460188.pdf + README: https://github.com/open-mmlab/mmpose/blob/master/docs/en/papers/algorithms/voxelpose.md +Models: +- Config: configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3.py + In Collection: VoxelPose + Metadata: + Architecture: &id001 + - VoxelPose + Training Data: Campus + Name: voxelpose_voxelpose_prn32x32x32_cpn80x80x20_campus_cam3 + Results: + - Dataset: Campus + Metrics: {} + Task: Body 3D Keypoint + Weights: https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3-3ecee30e_20220323.pth +- Config: configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py + In Collection: VoxelPose + Metadata: + Architecture: *id001 + Training Data: Campus + Name: voxelpose_voxelpose_prn64x64x64_cpn80x80x20_campus_cam3 + Results: + - Dataset: Campus + Metrics: {} + Task: Body 3D Keypoint + Weights: https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3-d8decbf7_20220323.pth diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.md b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.md new file mode 100644 index 0000000000..08ad672ff4 --- /dev/null +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.md @@ -0,0 +1,41 @@ + + +
+VoxelPose (ECCV'2020) + +```bibtex +@inproceedings{tumultipose, + title={VoxelPose: Towards Multi-Camera 3D Human Pose Estimation in Wild Environment}, + author={Tu, Hanyue and Wang, Chunyu and Zeng, Wenjun}, + booktitle={ECCV}, + year={2020} +} +``` + +
+ + + +
+Shelf (CVPR'2014) + +```bibtex +@inproceedings {belagian14multi, + title = {{3D} Pictorial Structures for Multiple Human Pose Estimation}, + author = {Belagiannis, Vasileios and Amin, Sikandar and Andriluka, Mykhaylo and Schiele, Bernt and Navab + Nassir and Ilic, Slobo + booktitle = {IEEE Computer Society Conference on Computer Vision and Pattern Recognition (CVPR)}, + year = {2014}, + month = {June}, + organization={IEEE} +} +``` + +
+ +Results on Shelf dataset. + +| Arch | Actor 1 | Actor 2 | Actor 3 | Average| ckpt | log | +| :--- | :---: | :---: | :---: | :---: | :---: | :---: | +| [prn32_cpn48_res50](/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5.py) | 99.10 | 94.86 | 97.52 | 97.16| [ckpt](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5-24721ec7_20220323.pth) | [log](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5.log.json) | +| [prn64_cpn80_res50](/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py) | 99.00 | 94.59 | 97.64 | 97.08| [ckpt](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5-f406fefe_20220323.pth) | [log](https://download.openmmlab.com/mmpose/body3d/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.log.json) | diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.yml b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.yml new file mode 100644 index 0000000000..d25750a849 --- /dev/null +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.yml @@ -0,0 +1,30 @@ +Collections: +- Name: VoxelPose + Paper: + Title: 'VoxelPose: Towards Multi-Camera 3D Human Pose Estimation in Wild Environment' + URL: https://www.ecva.net/papers/eccv_2020/papers_ECCV/papers/123460188.pdf + README: https://github.com/open-mmlab/mmpose/blob/master/docs/en/papers/algorithms/voxelpose.md +Models: +- Config: configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5.py + In Collection: VoxelPose + Metadata: + Architecture: &id001 + - VoxelPose + Training Data: Shelf + Name: voxelpose_voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5 + Results: + - Dataset: Shelf + Metrics: {} + Task: Body 3D Keypoint + Weights: https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5-24721ec7_20220323.pth +- Config: configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py + In Collection: VoxelPose + Metadata: + Architecture: *id001 + Training Data: Shelf + Name: voxelpose_voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5 + Results: + - Dataset: Shelf + Metrics: {} + Task: Body 3D Keypoint + Weights: https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5-f406fefe_20220323.pth diff --git a/docs/en/tasks/3d_body_keypoint.md b/docs/en/tasks/3d_body_keypoint.md index c5ca2a1dba..39da9c934d 100644 --- a/docs/en/tasks/3d_body_keypoint.md +++ b/docs/en/tasks/3d_body_keypoint.md @@ -7,6 +7,7 @@ MMPose supported datasets: - [Human3.6M](#human36m) \[ [Homepage](http://vision.imar.ro/human3.6m/description.php) \] - [CMU Panoptic](#cmu-panoptic) \[ [Homepage](http://domedb.perception.cs.cmu.edu/) \] +- [Campus/Shelf](#campus/shelf) \[ [Homepage](http://campar.in.tum.de/Chair/MultiHumanPose) \] ## Human3.6M @@ -118,3 +119,72 @@ mmpose ├── 160226_haggling1 ├── ... ``` + +## Campus/Shelf + +
+Campus/Shelf (CVPR'2014) + +```bibtex +@inproceedings {belagian14multi, + title = {{3D} Pictorial Structures for Multiple Human Pose Estimation}, + author = {Belagiannis, Vasileios and Amin, Sikandar and Andriluka, Mykhaylo and Schiele, Bernt and Navab + Nassir and Ilic, Slobo + booktitle = {IEEE Computer Society Conference on Computer Vision and Pattern Recognition (CVPR)}, + year = {2014}, + month = {June}, + organization={IEEE} +} +``` + +
+ +Please follow [voxelpose-pytorch](https://github.com/microsoft/voxelpose-pytorch) to prepare these two datasets. + +1. Please download the datasets from the [official website](http://campar.in.tum.de/Chair/MultiHumanPose) and extract them under `$MMPOSE/data/campus` and `$MMPOSE/data/shelf`, respectively. The original data include images as well as the ground truth pose file `actorsGT.mat`. + +2. We directly use the processed camera parameters from [voxelpose-pytorch](https://github.com/microsoft/voxelpose-pytorch). You can download them from this repository and place in under `$MMPOSE/data/campus/calibration_campus.json` and `$MMPOSE/data/shelf/calibration_shelf.json`, respectively. + +3. Like [Voxelpose](https://github.com/microsoft/voxelpose-pytorch), due to the limited and incomplete annotations of the two datasets, we don't train the model using this dataset. Instead, we directly use the 2D pose estimator trained on COCO, and use independent 3D human poses from the CMU Panoptic dataset to train our 3D model. It lies in `${MMPOSE}/data/campus/panoptic_training_pose.pkl` and `${MMPOSE}/data/shelf/panoptic_training_pose.pkl`, respectively. + +4. Like [Voxelpose](https://github.com/microsoft/voxelpose-pytorch), for testing, we first estimate 2D poses and generate 2D heatmaps for these two datasets. You can download the predicted poses from [voxelpose-pytorch](https://github.com/microsoft/voxelpose-pytorch) and place them in `$MMPOSE/data/campus/pred_campus_maskrcnn_hrnet_coco.pkl` and `$MMPOSE/data/shelf/pred_shelf_maskrcnn_hrnet_coco.pkl`, respectively. You can also use the models trained on COCO dataset (like HigherHRNet) to generate 2D heatmaps directly. + +The directory tree should be like this: + +```text +mmpose +├── mmpose +├── docs +├── tests +├── tools +├── configs +`── data + ├── campus + | ├── Camera0 + | | | ├── campus4-c0-00000.png + | | | ├── ... + | | | ├── campus4-c0-01999.png + | ... + | ├── Camera2 + | | | ├── campus4-c2-00000.png + | | | ├── ... + | | | ├── campus4-c2-01999.png + | ├── calibration_campus.json + | ├── pred_campus_maskrcnn_hrnet_coco.pkl + | ├── panoptic_training_pose.pkl + | ├── actorsGT.mat + ├── shelf + | ├── Camera0 + | | | ├── img_000000.png + | | | ├── ... + | | | ├── img_003199.png + | ... + | ├── Camera4 + | | | ├── img_000000.png + | | | ├── ... + | | | ├── img_003199.png + | ├── calibration_shelf.json + | ├── pred_shelf_maskrcnn_hrnet_coco.pkl + | ├── panoptic_training_pose.pkl + | ├── actorsGT.mat +``` diff --git a/model-index.yml b/model-index.yml index c5522f6fc1..9a250dd06e 100644 --- a/model-index.yml +++ b/model-index.yml @@ -77,7 +77,9 @@ Import: - configs/body/2d_kpt_sview_rgb_img/topdown_heatmap/posetrack18/hrnet_posetrack18.yml - configs/body/2d_kpt_sview_rgb_img/topdown_heatmap/posetrack18/resnet_posetrack18.yml - configs/body/2d_kpt_sview_rgb_vid/posewarper/posetrack18/hrnet_posetrack18_posewarper.yml +- configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_campus.yml - configs/body/3d_kpt_mview_rgb_img/voxelpose/panoptic/voxelpose_prn64x64x64_cpn80x80x20_panoptic_cam5.yml +- configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.yml - configs/body/3d_kpt_sview_rgb_img/pose_lift/h36m/simplebaseline3d_h36m.yml - configs/body/3d_kpt_sview_rgb_img/pose_lift/mpi_inf_3dhp/simplebaseline3d_mpi-inf-3dhp.yml - configs/body/3d_kpt_sview_rgb_vid/video_pose_lift/h36m/videopose3d_h36m.yml From ac0c41bba87e275d4a05e8d0b8592e2f428a1c54 Mon Sep 17 00:00:00 2001 From: liqikai Date: Wed, 23 Mar 2022 20:08:36 +0800 Subject: [PATCH 08/19] modify config for campus --- .../campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py index f44eab55d2..ade586945f 100644 --- a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py @@ -18,7 +18,7 @@ warmup='linear', warmup_iters=500, warmup_ratio=0.001, - step=[10, 20]) + step=[20, 25]) total_epochs = 30 log_config = dict( interval=50, hooks=[ From 80d572f3dae0f3c5f2e06bff67ea5909bae4f3ab Mon Sep 17 00:00:00 2001 From: liqikai Date: Wed, 23 Mar 2022 20:32:30 +0800 Subject: [PATCH 09/19] fix typo --- .../datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py b/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py index 0f1a7c30c2..1693246709 100644 --- a/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py +++ b/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py @@ -114,7 +114,6 @@ def _get_mapping_id_name(imgs): return id2name, name2id - @abstractmethod def _get_db(self): """Load dataset.""" raise NotImplementedError From c613afc59f00d6204cc396bfd6e592366179ff16 Mon Sep 17 00:00:00 2001 From: liqikai Date: Thu, 24 Mar 2022 16:26:14 +0800 Subject: [PATCH 10/19] fix some docs --- README.md | 1 - .../voxelpose/campus/voxelpose_campus.md | 2 +- .../voxelpose/shelf/voxelpose_shelf.md | 2 +- docs/en/papers/datasets/campus_and_shelf.md | 20 +++++++++++++++++++ docs/en/tasks/3d_body_keypoint.md | 6 +++--- 5 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 docs/en/papers/datasets/campus_and_shelf.md diff --git a/README.md b/README.md index 5562f7f2cf..523178acb3 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,6 @@ Supported [datasets](https://mmpose.readthedocs.io/en/latest/datasets.html): * [x] [Human3.6M](https://mmpose.readthedocs.io/en/latest/papers/datasets.html#human3-6m-tpami-2014) \[[homepage](http://vision.imar.ro/human3.6m/description.php)\] (TPAMI'2014) * [x] [COCO](https://mmpose.readthedocs.io/en/latest/papers/datasets.html#coco-eccv-2014) \[[homepage](http://cocodataset.org/)\] (ECCV'2014) * [x] [CMU Panoptic](https://mmpose.readthedocs.io/en/latest/papers/datasets.html#cmu-panoptic-iccv-2015) \[[homepage](http://domedb.perception.cs.cmu.edu/)\] (ICCV'2015) -* [x] [Campus/Shelf](https://mmpose.readthedocs.io/en/latest/papers/datasets.html#campus-shelf-cvpr2014) \[[homepage](http://campar.in.tum.de/Chair/MultiHumanPose)\] (CVPR'2014) * [x] [DeepFashion](https://mmpose.readthedocs.io/en/latest/papers/datasets.html#deepfashion-cvpr-2016) \[[homepage](http://mmlab.ie.cuhk.edu.hk/projects/DeepFashion/LandmarkDetection.html)\] (CVPR'2016) * [x] [300W](https://mmpose.readthedocs.io/en/latest/papers/datasets.html#300w-imavis-2016) \[[homepage](https://ibug.doc.ic.ac.uk/resources/300-W/)\] (IMAVIS'2016) * [x] [RHD](https://mmpose.readthedocs.io/en/latest/papers/datasets.html#rhd-iccv-2017) \[[homepage](https://lmb.informatik.uni-freiburg.de/resources/datasets/RenderedHandposeDataset.en.html)\] (ICCV'2017) diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_campus.md b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_campus.md index 2a45fe5814..f3cfd7d999 100644 --- a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_campus.md +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_campus.md @@ -17,7 +17,7 @@
-Campus (CVPR'2014) +Campus (CVPR'2014) ```bibtex @inproceedings {belagian14multi, diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.md b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.md index 08ad672ff4..a34af00471 100644 --- a/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.md +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.md @@ -17,7 +17,7 @@
-Shelf (CVPR'2014) +Shelf (CVPR'2014) ```bibtex @inproceedings {belagian14multi, diff --git a/docs/en/papers/datasets/campus_and_shelf.md b/docs/en/papers/datasets/campus_and_shelf.md new file mode 100644 index 0000000000..8748be137e --- /dev/null +++ b/docs/en/papers/datasets/campus_and_shelf.md @@ -0,0 +1,20 @@ +# 3D Pictorial Structures for Multiple Human Pose Estimation + + + +
+Campus and Shelf (CVPR'2014) + +```bibtex +@inproceedings {belagian14multi, + title = {{3D} Pictorial Structures for Multiple Human Pose Estimation}, + author = {Belagiannis, Vasileios and Amin, Sikandar and Andriluka, Mykhaylo and Schiele, Bernt and Navab + Nassir and Ilic, Slobodan}, + booktitle = {IEEE Computer Society Conference on Computer Vision and Pattern Recognition (CVPR)}, + year = {2014}, + month = {June}, + organization={IEEE} +} +``` + +
diff --git a/docs/en/tasks/3d_body_keypoint.md b/docs/en/tasks/3d_body_keypoint.md index 39da9c934d..bef778d5f6 100644 --- a/docs/en/tasks/3d_body_keypoint.md +++ b/docs/en/tasks/3d_body_keypoint.md @@ -7,7 +7,7 @@ MMPose supported datasets: - [Human3.6M](#human36m) \[ [Homepage](http://vision.imar.ro/human3.6m/description.php) \] - [CMU Panoptic](#cmu-panoptic) \[ [Homepage](http://domedb.perception.cs.cmu.edu/) \] -- [Campus/Shelf](#campus/shelf) \[ [Homepage](http://campar.in.tum.de/Chair/MultiHumanPose) \] +- [Campus/Shelf](#campus-and-shelf) \[ [Homepage](http://campar.in.tum.de/Chair/MultiHumanPose) \] ## Human3.6M @@ -120,10 +120,10 @@ mmpose ├── ... ``` -## Campus/Shelf +## Campus and Shelf
-Campus/Shelf (CVPR'2014) +Campus and Shelf (CVPR'2014) ```bibtex @inproceedings {belagian14multi, From 2cca6580678a51eaf9e0455ab73d15a60b087603 Mon Sep 17 00:00:00 2001 From: liqikai Date: Thu, 24 Mar 2022 16:30:43 +0800 Subject: [PATCH 11/19] remove _get_db method in base class --- .../datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py b/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py index 1693246709..5478e28a94 100644 --- a/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py +++ b/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py @@ -114,10 +114,6 @@ def _get_mapping_id_name(imgs): return id2name, name2id - def _get_db(self): - """Load dataset.""" - raise NotImplementedError - @abstractmethod def evaluate(self, results, *args, **kwargs): """Evaluate keypoint results.""" From cd18fd522ec2f4ac65e56d470b59e9739bdb66f1 Mon Sep 17 00:00:00 2001 From: liqikai Date: Wed, 30 Mar 2022 15:50:18 +0800 Subject: [PATCH 12/19] update log path --- .../3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_campus.md | 4 ++-- .../3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_campus.md b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_campus.md index f3cfd7d999..9fdfc877a7 100644 --- a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_campus.md +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_campus.md @@ -37,5 +37,5 @@ Results on Campus dataset. | Arch | Actor 1 | Actor 2 | Actor 3 | Average| ckpt | log | | :--- | :---: | :---: | :---: | :---: | :---: | :---: | -| [prn32_cpn80_res50](/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3.py) | 97.76 | 93.92 | 98.48 | 96.72 | [ckpt](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3-3ecee30e_20220323.pth) | [log](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3.log.json) | -| [prn64_cpn80_res50](/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py) | 97.76 | 93.33 | 98.77 | 96.62| [ckpt](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3-d8decbf7_20220323.pth) | [log](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.log.json) | +| [prn32_cpn80_res50](/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3.py) | 97.76 | 93.92 | 98.48 | 96.72 | [ckpt](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3-3ecee30e_20220323.pth) | [log](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3_20220323.log.json) | +| [prn64_cpn80_res50](/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py) | 97.76 | 93.33 | 98.77 | 96.62| [ckpt](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3-d8decbf7_20220323.pth) | [log](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3_20220323.log.json) | diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.md b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.md index a34af00471..8041eeba89 100644 --- a/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.md +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.md @@ -37,5 +37,5 @@ Results on Shelf dataset. | Arch | Actor 1 | Actor 2 | Actor 3 | Average| ckpt | log | | :--- | :---: | :---: | :---: | :---: | :---: | :---: | -| [prn32_cpn48_res50](/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5.py) | 99.10 | 94.86 | 97.52 | 97.16| [ckpt](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5-24721ec7_20220323.pth) | [log](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5.log.json) | -| [prn64_cpn80_res50](/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py) | 99.00 | 94.59 | 97.64 | 97.08| [ckpt](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5-f406fefe_20220323.pth) | [log](https://download.openmmlab.com/mmpose/body3d/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.log.json) | +| [prn32_cpn48_res50](/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5.py) | 99.10 | 94.86 | 97.52 | 97.16| [ckpt](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5-24721ec7_20220323.pth) | [log](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5_20220323.log.json) | +| [prn64_cpn80_res50](/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py) | 99.00 | 94.59 | 97.64 | 97.08| [ckpt](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5-f406fefe_20220323.pth) | [log](https://download.openmmlab.com/mmpose/body3d/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5_20220323.log.json) | From adbf55c559b88e0393a51bd306c76b4599155c87 Mon Sep 17 00:00:00 2001 From: liqikai Date: Wed, 30 Mar 2022 20:30:03 +0800 Subject: [PATCH 13/19] fix log path --- .../3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.md b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.md index 8041eeba89..20c8e7c364 100644 --- a/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.md +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_shelf.md @@ -38,4 +38,4 @@ Results on Shelf dataset. | Arch | Actor 1 | Actor 2 | Actor 3 | Average| ckpt | log | | :--- | :---: | :---: | :---: | :---: | :---: | :---: | | [prn32_cpn48_res50](/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5.py) | 99.10 | 94.86 | 97.52 | 97.16| [ckpt](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5-24721ec7_20220323.pth) | [log](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5_20220323.log.json) | -| [prn64_cpn80_res50](/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py) | 99.00 | 94.59 | 97.64 | 97.08| [ckpt](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5-f406fefe_20220323.pth) | [log](https://download.openmmlab.com/mmpose/body3d/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5_20220323.log.json) | +| [prn64_cpn80_res50](/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py) | 99.00 | 94.59 | 97.64 | 97.08| [ckpt](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5-f406fefe_20220323.pth) | [log](https://download.openmmlab.com/mmpose/body3d/voxelpose/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5_20220323.log.json) | From 6f27f1d49aa70cd14e57b5f317addb054b250726 Mon Sep 17 00:00:00 2001 From: liqikai Date: Wed, 30 Mar 2022 21:12:00 +0800 Subject: [PATCH 14/19] add more descriptions in configs --- ...ose_prn32x32x32_cpn80x80x20_campus_cam3.py | 26 +++++++++++++++++++ ...ose_prn64x64x64_cpn80x80x20_campus_cam3.py | 26 +++++++++++++++++++ ...pose_prn32x32x32_cpn48x48x12_shelf_cam5.py | 25 ++++++++++++++++++ ...pose_prn64x64x64_cpn80x80x20_shelf_cam5.py | 25 ++++++++++++++++++ 4 files changed, 102 insertions(+) diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3.py b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3.py index 83a4b241f5..44f9871f57 100644 --- a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3.py +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3.py @@ -25,14 +25,25 @@ dict(type='TextLoggerHook'), ]) +# size of the 3D space space_size = [12000.0, 12000.0, 2000.0] + +# coordinate of the center of the 3D space space_center = [3000.0, 4500.0, 1000.0] +# size of the input volume to the human_detector 3D CNN in voxelpose cube_size = [80, 80, 20] + +# size of the cuboid human proposal sub_space_size = [2000.0, 2000.0, 2000.0] +# size of the input volume to the pose_regressor 3D CNN in voxelpose sub_cube_size = [32, 32, 32] + +# input size of the network image_size = [800, 640] +# output heatmap size of the 2D model heatmap_size = [200, 160] +# number of joints num_joints = 17 data_root = 'data/campus' @@ -44,19 +55,34 @@ heatmap_size=[heatmap_size], num_joints=num_joints, cam_list=[0, 1, 2], + # number of cameras num_cameras=3, + # The range of frame indices. In Campus, there are 2000 frames in total. + # Frames [350 - 470] and [650 - 750] are for evaluation, the rest frames + # are for training. frame_range=list(range(0, 350)) + list(range(471, 650)) + list(range(751, 2000)), + # the width and height of the input image width=360, height=288, + # the indices of keypoints related to the center of a person root_id=[11, 12], + # maximum number of poses in a single training sample max_nposes=10, + # minimum number of poses in a single training sample min_nposes=1, + # number of training samples each epoch num_train_samples=3000, + # maximum number of persons in a single image maximum_person=10, + # the camera calibration file cam_file=f'{data_root}/calibration_campus.json', + # predicted 2D poses for each image in each view, usually generated by + # 2D pose estimator trained on COCO test_pose_db_file=f'{data_root}/pred_campus_maskrcnn_hrnet_coco.pkl', + # 3D human poses, use for generating input heatmap to train 3D model train_pose_db_file=f'{data_root}/panoptic_training_pose.pkl', + # 2D and 3D ground-truth poses gt_pose_db_file=f'{data_root}/actorsGT.mat', ) diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py index ade586945f..33b8c9cf9d 100644 --- a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py @@ -25,14 +25,25 @@ dict(type='TextLoggerHook'), ]) +# size of the 3D space space_size = [12000.0, 12000.0, 2000.0] + +# coordinate of the center of the 3D space space_center = [3000.0, 4500.0, 1000.0] +# size of the input volume to the human_detector 3D CNN in voxelpose cube_size = [80, 80, 20] + +# size of the cuboid human proposal sub_space_size = [2000.0, 2000.0, 2000.0] +# size of the input volume to the pose_regressor 3D CNN in voxelpose sub_cube_size = [64, 64, 64] + +# input size of the network image_size = [800, 640] +# output heatmap size of the 2D model heatmap_size = [200, 160] +# number of joints num_joints = 17 data_root = 'data/campus' @@ -44,19 +55,34 @@ heatmap_size=[heatmap_size], num_joints=num_joints, cam_list=[0, 1, 2], + # number of cameras num_cameras=3, + # The range of frame indices. In Campus, there are 2000 frames in total. + # Frames [350 - 470] and [650 - 750] are for evaluation, the rest frames + # are for training. frame_range=list(range(0, 350)) + list(range(471, 650)) + list(range(751, 2000)), + # the width and height of the input image width=360, height=288, + # the indices of keypoints related to the center of a person root_id=[11, 12], + # maximum number of poses in a single training sample max_nposes=10, + # minimum number of poses in a single training sample min_nposes=1, + # number of training samples each epoch num_train_samples=3000, + # maximum number of persons in a single image maximum_person=10, + # the camera calibration file cam_file=f'{data_root}/calibration_campus.json', + # predicted 2D poses for each image in each view, usually generated by + # 2D pose estimator trained on COCO test_pose_db_file=f'{data_root}/pred_campus_maskrcnn_hrnet_coco.pkl', + # 3D human poses, use for generating input heatmap to train 3D model train_pose_db_file=f'{data_root}/panoptic_training_pose.pkl', + # 2D and 3D ground-truth poses gt_pose_db_file=f'{data_root}/actorsGT.mat', ) diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5.py b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5.py index ea8c8f0672..920f47460e 100644 --- a/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5.py +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5.py @@ -25,14 +25,25 @@ dict(type='TextLoggerHook'), ]) +# size of the 3D space space_size = [8000, 8000, 2000] + +# coordinate of the center of the 3D space space_center = [450, -320, 800] +# size of the input volume to the human_detector 3D CNN in voxelpose cube_size = [48, 48, 12] + +# size of the cuboid human proposal sub_space_size = [2000, 2000, 2000] +# size of the input volume to the pose_regressor 3D CNN in voxelpose sub_cube_size = [32, 32, 32] + +# input size of the network image_size = [800, 608] +# output heatmap size of the 2D model heatmap_size = [200, 152] +# number of joints num_joints = 17 data_root = 'data/shelf' @@ -44,18 +55,32 @@ heatmap_size=[heatmap_size], num_joints=num_joints, cam_list=[0, 1, 2, 3, 4], + # number of cameras num_cameras=5, + # The range of frame indices. In Shelf, there are 3200 frames in total. + # Frames [300 - 600] are for evaluation, the rest frames are for training frame_range=list(range(0, 300)) + list(range(601, 3200)), + # the width and height of the input image width=1032, height=776, + # the indices of keypoints related to the center of a person root_id=[11, 12], + # maximum number of poses in a single training sample max_nposes=6, + # minimum number of poses in a single training sample min_nposes=1, + # number of training samples each epoch num_train_samples=3000, + # maximum number of persons in a single image maximum_person=10, + # the camera calibration file cam_file=f'{data_root}/calibration_shelf.json', + # predicted 2D poses for each image in each view, usually generated by + # 2D pose estimator trained on COCO test_pose_db_file=f'{data_root}/pred_shelf_maskrcnn_hrnet_coco.pkl', + # 3D human poses, use for generating input heatmap to train 3D model train_pose_db_file=f'{data_root}/panoptic_training_pose.pkl', + # 2D and 3D ground-truth poses gt_pose_db_file=f'{data_root}/actorsGT.mat', ) diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py index 5b896e2f10..61f7b84d90 100644 --- a/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py @@ -25,14 +25,25 @@ dict(type='TextLoggerHook'), ]) +# size of the 3D space space_size = [8000, 8000, 2000] + +# coordinate of the center of the 3D space space_center = [450, -320, 800] +# size of the input volume to the human_detector 3D CNN in voxelpose cube_size = [80, 80, 20] + +# size of the cuboid human proposal sub_space_size = [2000, 2000, 2000] +# size of the input volume to the pose_regressor 3D CNN in voxelpose sub_cube_size = [64, 64, 64] + +# input size of the network image_size = [800, 608] +# output heatmap size of the 2D model heatmap_size = [200, 152] +# number of joints num_joints = 17 data_root = 'data/shelf' @@ -44,18 +55,32 @@ heatmap_size=[heatmap_size], num_joints=num_joints, cam_list=[0, 1, 2, 3, 4], + # number of cameras num_cameras=5, + # The range of frame indices. In Shelf, there are 3200 frames in total. + # Frames [300 - 600] are for evaluation, the rest frames are for training frame_range=list(range(0, 300)) + list(range(601, 3200)), + # the width and height of the input image width=1032, height=776, + # the indices of keypoints related to the center of a person root_id=[11, 12], + # maximum number of poses in a single training sample max_nposes=6, + # minimum number of poses in a single training sample min_nposes=1, + # number of training samples each epoch num_train_samples=3000, + # maximum number of persons in a single image maximum_person=10, + # the camera calibration file cam_file=f'{data_root}/calibration_shelf.json', + # predicted 2D poses for each image in each view, usually generated by + # 2D pose estimator trained on COCO test_pose_db_file=f'{data_root}/pred_shelf_maskrcnn_hrnet_coco.pkl', + # 3D human poses, use for generating input heatmap to train 3D model train_pose_db_file=f'{data_root}/panoptic_training_pose.pkl', + # 2D and 3D ground-truth poses gt_pose_db_file=f'{data_root}/actorsGT.mat', ) From f532fe199bc81bfbbf6132ff7e4bbbd06d6c0c45 Mon Sep 17 00:00:00 2001 From: liqikai Date: Wed, 30 Mar 2022 21:29:01 +0800 Subject: [PATCH 15/19] keep _get_db method in base class and add error message in subclasses --- mmpose/core/evaluation/eval_hooks.py | 12 ++---------- .../base/kpt_3d_mview_rgb_img_direct_dataset.py | 14 +++++++++----- .../body3d/body3d_mview_direct_campus_dataset.py | 8 ++++++++ .../body3d/body3d_mview_direct_shelf_dataset.py | 8 ++++++++ 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/mmpose/core/evaluation/eval_hooks.py b/mmpose/core/evaluation/eval_hooks.py index 1cebeab65a..b35a9c6a99 100644 --- a/mmpose/core/evaluation/eval_hooks.py +++ b/mmpose/core/evaluation/eval_hooks.py @@ -5,16 +5,8 @@ from mmcv.runner import EvalHook as _EvalHook MMPOSE_GREATER_KEYS = [ - 'acc', - 'ap', - 'ar', - 'pck', - 'auc', - '3dpck', - 'p-3dpck', - '3dauc', - 'p-3dauc', - 'pcp', + 'acc', 'ap', 'ar', 'pck', 'auc', '3dpck', 'p-3dpck', '3dauc', 'p-3dauc', + 'pcp' ] MMPOSE_LESS_KEYS = ['loss', 'epe', 'nme', 'mpjpe', 'p-mpjpe', 'n-mpjpe'] diff --git a/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py b/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py index 5478e28a94..c75d8991af 100644 --- a/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py +++ b/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py @@ -1,13 +1,12 @@ # Copyright (c) OpenMMLab. All rights reserved. import copy -import json import os.path as osp import pickle from abc import ABCMeta, abstractmethod -import json_tricks as json_ +import json_tricks as json import numpy as np -import scipy.io as scio +from scipy.io import loadmat from torch.utils.data import Dataset from mmpose.datasets import DatasetInfo @@ -81,6 +80,11 @@ def __init__(self, self.pipeline = Compose(self.pipeline) + @abstractmethod + def _get_db(self): + """Load dataset.""" + raise NotImplementedError + def load_config(self, data_cfg): """Initialize dataset attributes according to the config. @@ -123,7 +127,7 @@ def _write_keypoint_results(keypoints, res_file): """Write results into a json file.""" with open(res_file, 'w') as f: - json_.dump(keypoints, f, sort_keys=True, indent=4) + json.dump(keypoints, f, sort_keys=True, indent=4) def __len__(self): """Get the size of the dataset.""" @@ -241,7 +245,7 @@ def _load_files(self): assert osp.exists(self.gt_pose_db_file), f'gt_pose_db_file ' \ f"{self.gt_pose_db_file} doesn't exist, please check again" - gt = scio.loadmat(self.gt_pose_db_file) + gt = loadmat(self.gt_pose_db_file) self.gt_pose_db = np.array(np.array( gt['actor3D'].tolist()).tolist()).squeeze() diff --git a/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py b/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py index e8fc96cd15..e91f47f9ff 100644 --- a/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py +++ b/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py @@ -135,6 +135,14 @@ def load_config(self, data_cfg): self._load_files() + def _get_db(self): + """Load dataset.""" + raise NotImplementedError( + '_get_db method is not overwritten here because of two reasons.' + 'First, the training and test samples are quite different. ' + 'Second, the training samples have some randomness which is not' + 'appropriate to collect all samples into a database one time.') + def __getitem__(self, idx): """Get the sample given index.""" diff --git a/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py b/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py index 6ce0652e94..1c0c353799 100644 --- a/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py +++ b/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py @@ -135,6 +135,14 @@ def load_config(self, data_cfg): self._load_files() + def _get_db(self): + """Load dataset.""" + raise NotImplementedError( + '_get_db method is not overwritten here because of two reasons.' + 'First, the training and test samples are quite different. ' + 'Second, the training samples have some randomness which is not' + 'appropriate to collect all samples into a database one time.') + def __getitem__(self, idx): """Get the sample given index.""" From ebb4830c0b620a7ff241d7bee8d4f544aea67aec Mon Sep 17 00:00:00 2001 From: liqikai Date: Thu, 31 Mar 2022 16:24:03 +0800 Subject: [PATCH 16/19] add explanation of some functions --- .../kpt_3d_mview_rgb_img_direct_dataset.py | 14 +++++++++----- .../body3d_mview_direct_campus_dataset.py | 18 +++++++++++++++++- .../body3d_mview_direct_shelf_dataset.py | 16 ++++++++++++++++ 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py b/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py index c75d8991af..d59deb2b84 100644 --- a/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py +++ b/mmpose/datasets/datasets/base/kpt_3d_mview_rgb_img_direct_dataset.py @@ -175,11 +175,14 @@ def _get_scale(self, raw_image_size): @staticmethod def rotate_points(points, center, rot_rad): - """ - :param points: N*2 - :param center: 2 - :param rot_rad: scalar - :return: N*2 + """Rotate the points around the center. + + Args: + points: np.ndarray, N*2 + center: np.ndarray, 2 + rot_rad: scalar + Return: + np.ndarray (N*2) """ rot_rad = rot_rad * np.pi / 180.0 rotate_mat = np.array([[np.cos(rot_rad), -np.sin(rot_rad)], @@ -193,6 +196,7 @@ def rotate_points(points, center, rot_rad): @staticmethod def calc_bbox(pose, pose_vis): + """calculate the bbox of a pose.""" index = pose_vis[:, 0] > 0 bbox = [ np.min(pose[index, 0]), diff --git a/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py b/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py index e91f47f9ff..099fb1bbab 100644 --- a/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py +++ b/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py @@ -213,8 +213,9 @@ def _prepare_test_sample(self, idx): def _prepare_train_sample(self, idx): results = {} + # To prepare a training sample, there are three steps. + # 1. Randomly sample some 3D poses from motion capture database nposes_ori = np.random.choice(range(self.min_nposes, self.max_nposes)) - select_poses = np.random.choice(self.train_pose_db, nposes_ori) joints_3d = np.array([p['pose'] for p in select_poses]) @@ -222,8 +223,10 @@ def _prepare_train_sample(self, idx): bbox_list = [] center_list = [] + # 2. Place the selected poses at random locations in the space for n in range(nposes_ori): points = joints_3d[n][:, :2].copy() + # get the location of a person's root joint center = np.mean(points[self.root_id, :2], axis=0) rot_rad = np.random.uniform(-180, 180) @@ -262,12 +265,15 @@ def _prepare_train_sample(self, idx): roots_3d = np.mean(joints_3d_u[:, self.root_id], axis=1) + # 3. Project 3D poses to all views to get the respective 2D locations for cam_id, cam_param in self.cameras.items(): joints = [] joints_vis = [] single_view_camera = SimpleCamera(cam_param) for n in range(nposes): + # project the 3D pose to the view to get 2D location pose2d = single_view_camera.world_to_pixel(joints_3d[n]) + # check the validity of joint cooridinate x_check = np.bitwise_and(pose2d[:, 0] >= 0, pose2d[:, 0] <= self.width - 1) y_check = np.bitwise_and(pose2d[:, 1] >= 0, @@ -313,6 +319,11 @@ def __len__(self): @staticmethod def get_new_center(center_list): + """Generate new center or select from the center list randomly. + + The proability and the parameters related to cooridinates can also be + tuned, just make sure that the center is within the given 3D space. + """ if len(center_list) == 0 or random.random() < 0.7: new_center = np.array([ np.random.uniform(-2500.0, 8500.0), @@ -326,6 +337,11 @@ def get_new_center(center_list): return new_center def isvalid(self, new_center, bbox, bbox_list): + """Check if the new person bbox are valid, which need to satisfies: + + 1. the center is visible in at least 2 views, and + 2. have a sufficiently small iou with all other person bboxes. + """ new_center_us = new_center.reshape(1, -1) vis = 0 for _, cam_param in self.cameras.items(): diff --git a/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py b/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py index 1c0c353799..c9b9f4d53a 100644 --- a/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py +++ b/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py @@ -213,6 +213,8 @@ def _prepare_test_sample(self, idx): def _prepare_train_sample(self, idx): results = {} + # To prepare a training sample, there are three steps. + # 1. Randomly sample some 3D poses from motion capture database nposes_ori = np.random.choice(range(self.min_nposes, self.max_nposes)) select_poses = np.random.choice(self.train_pose_db, nposes_ori) @@ -221,9 +223,11 @@ def _prepare_train_sample(self, idx): bbox_list = [] center_list = [] + # 2. Place the selected poses at random locations in the space for n in range(nposes_ori): points = joints_3d[n][:, :2].copy() + # get the location of a person's root joint center = np.mean(points[self.root_id, :2], axis=0) rot_rad = np.random.uniform(-180, 180) @@ -261,12 +265,15 @@ def _prepare_train_sample(self, idx): roots_3d = np.mean(joints_3d_u[:, self.root_id], axis=1) + # 3. Project 3D poses to all views to get the respective 2D locations for cam_id, cam_param in self.cameras.items(): joints = [] joints_vis = [] single_view_camera = SimpleCamera(cam_param) for n in range(nposes): + # project the 3D pose to the view to get 2D location pose2d = single_view_camera.world_to_pixel(joints_3d[n]) + # check the validity of joint cooridinate x_check = np.bitwise_and(pose2d[:, 0] >= 0, pose2d[:, 0] <= self.width - 1) y_check = np.bitwise_and(pose2d[:, 1] >= 0, @@ -312,6 +319,11 @@ def __len__(self): @staticmethod def get_new_center(center_list): + """Generate new center or select from the center list randomly. + + The proability and the parameters related to cooridinates can also be + tuned, just make sure that the center is within the given 3D space. + """ if len(center_list) == 0 or random.random() < 0.7: new_center = np.array([ np.random.uniform(-1000.0, 2000.0), @@ -326,6 +338,10 @@ def get_new_center(center_list): @staticmethod def isvalid(bbox, bbox_list): + """Check if the new person bbox are valid, which need to satisfies: + + have a sufficiently small iou with all other person bboxes. + """ if len(bbox_list) == 0: return True From c7683568a4f0ffe6a814e1872183828e56349970 Mon Sep 17 00:00:00 2001 From: liqikai Date: Thu, 31 Mar 2022 20:28:03 +0800 Subject: [PATCH 17/19] add explanation about dataset in doc string --- .../datasets/body3d/body3d_mview_direct_campus_dataset.py | 8 +++++++- .../datasets/body3d/body3d_mview_direct_shelf_dataset.py | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py b/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py index 099fb1bbab..332cfcaaaf 100644 --- a/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py +++ b/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py @@ -1,6 +1,5 @@ # Copyright (c) OpenMMLab. All rights reserved. import copy -# import glob import os.path as osp import random import tempfile @@ -25,6 +24,13 @@ class Body3DMviewDirectCampusDataset(Kpt3dMviewRgbImgDirectDataset): ` The dataset loads both 2D and 3D annotations as well as camera parameters. + It is worth mentioning that when training multi-view 3D pose models, + due to the limited and incomplete annotations of this dataset, we may not + use this dataset to train the model. Instead, we use the 2D pose estimator + trained on COCO, and use independent 3D human poses from the CMU Panoptic + dataset to train the 3D model. + For testing, we first estimate 2D poses and generate 2D heatmaps for this + dataset as the input to 3D model. Campus keypoint indices:: diff --git a/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py b/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py index c9b9f4d53a..7f63a0e842 100644 --- a/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py +++ b/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py @@ -1,6 +1,5 @@ # Copyright (c) OpenMMLab. All rights reserved. import copy -# import glob import os.path as osp import random import tempfile @@ -25,6 +24,13 @@ class Body3DMviewDirectShelfDataset(Kpt3dMviewRgbImgDirectDataset): ` The dataset loads both 2D and 3D annotations as well as camera parameters. + It is worth mentioning that when training multi-view 3D pose models, + due to the limited and incomplete annotations of this dataset, we may not + use this dataset to train the model. Instead, we use the 2D pose estimator + trained on COCO, and use independent 3D human poses from the CMU Panoptic + dataset to train the 3D model. + For testing, we first estimate 2D poses and generate 2D heatmaps for this + dataset as the input to 3D model. Shelf keypoint indices:: From 4f72479b7013f96550c9152f90ee2ba2f4b6be9f Mon Sep 17 00:00:00 2001 From: liqikai Date: Thu, 31 Mar 2022 20:29:39 +0800 Subject: [PATCH 18/19] modify the path for train_pose_db_file in config and docs --- .../campus/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3.py | 2 +- .../campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py | 2 +- .../shelf/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5.py | 2 +- .../shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py | 2 +- docs/en/tasks/3d_body_keypoint.md | 5 ++--- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3.py b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3.py index 44f9871f57..80d88cbf03 100644 --- a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3.py +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn32x32x32_cpn80x80x20_campus_cam3.py @@ -81,7 +81,7 @@ # 2D pose estimator trained on COCO test_pose_db_file=f'{data_root}/pred_campus_maskrcnn_hrnet_coco.pkl', # 3D human poses, use for generating input heatmap to train 3D model - train_pose_db_file=f'{data_root}/panoptic_training_pose.pkl', + train_pose_db_file='data/panoptic_training_pose.pkl', # 2D and 3D ground-truth poses gt_pose_db_file=f'{data_root}/actorsGT.mat', ) diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py index 33b8c9cf9d..11df9dc3a5 100644 --- a/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/campus/voxelpose_prn64x64x64_cpn80x80x20_campus_cam3.py @@ -81,7 +81,7 @@ # 2D pose estimator trained on COCO test_pose_db_file=f'{data_root}/pred_campus_maskrcnn_hrnet_coco.pkl', # 3D human poses, use for generating input heatmap to train 3D model - train_pose_db_file=f'{data_root}/panoptic_training_pose.pkl', + train_pose_db_file='data/panoptic_training_pose.pkl', # 2D and 3D ground-truth poses gt_pose_db_file=f'{data_root}/actorsGT.mat', ) diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5.py b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5.py index 920f47460e..0eec22a22a 100644 --- a/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5.py +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn32x32x32_cpn48x48x12_shelf_cam5.py @@ -79,7 +79,7 @@ # 2D pose estimator trained on COCO test_pose_db_file=f'{data_root}/pred_shelf_maskrcnn_hrnet_coco.pkl', # 3D human poses, use for generating input heatmap to train 3D model - train_pose_db_file=f'{data_root}/panoptic_training_pose.pkl', + train_pose_db_file='data/panoptic_training_pose.pkl', # 2D and 3D ground-truth poses gt_pose_db_file=f'{data_root}/actorsGT.mat', ) diff --git a/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py index 61f7b84d90..9e4b5c376d 100644 --- a/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py +++ b/configs/body/3d_kpt_mview_rgb_img/voxelpose/shelf/voxelpose_prn64x64x64_cpn80x80x20_shelf_cam5.py @@ -79,7 +79,7 @@ # 2D pose estimator trained on COCO test_pose_db_file=f'{data_root}/pred_shelf_maskrcnn_hrnet_coco.pkl', # 3D human poses, use for generating input heatmap to train 3D model - train_pose_db_file=f'{data_root}/panoptic_training_pose.pkl', + train_pose_db_file='data/panoptic_training_pose.pkl', # 2D and 3D ground-truth poses gt_pose_db_file=f'{data_root}/actorsGT.mat', ) diff --git a/docs/en/tasks/3d_body_keypoint.md b/docs/en/tasks/3d_body_keypoint.md index bef778d5f6..af3aeb5e9d 100644 --- a/docs/en/tasks/3d_body_keypoint.md +++ b/docs/en/tasks/3d_body_keypoint.md @@ -145,7 +145,7 @@ Please follow [voxelpose-pytorch](https://github.com/microsoft/voxelpose-pytorch 2. We directly use the processed camera parameters from [voxelpose-pytorch](https://github.com/microsoft/voxelpose-pytorch). You can download them from this repository and place in under `$MMPOSE/data/campus/calibration_campus.json` and `$MMPOSE/data/shelf/calibration_shelf.json`, respectively. -3. Like [Voxelpose](https://github.com/microsoft/voxelpose-pytorch), due to the limited and incomplete annotations of the two datasets, we don't train the model using this dataset. Instead, we directly use the 2D pose estimator trained on COCO, and use independent 3D human poses from the CMU Panoptic dataset to train our 3D model. It lies in `${MMPOSE}/data/campus/panoptic_training_pose.pkl` and `${MMPOSE}/data/shelf/panoptic_training_pose.pkl`, respectively. +3. Like [Voxelpose](https://github.com/microsoft/voxelpose-pytorch), due to the limited and incomplete annotations of the two datasets, we don't train the model using this dataset. Instead, we directly use the 2D pose estimator trained on COCO, and use independent 3D human poses from the CMU Panoptic dataset to train our 3D model. It lies in `${MMPOSE}/data/panoptic_training_pose.pkl`. 4. Like [Voxelpose](https://github.com/microsoft/voxelpose-pytorch), for testing, we first estimate 2D poses and generate 2D heatmaps for these two datasets. You can download the predicted poses from [voxelpose-pytorch](https://github.com/microsoft/voxelpose-pytorch) and place them in `$MMPOSE/data/campus/pred_campus_maskrcnn_hrnet_coco.pkl` and `$MMPOSE/data/shelf/pred_shelf_maskrcnn_hrnet_coco.pkl`, respectively. You can also use the models trained on COCO dataset (like HigherHRNet) to generate 2D heatmaps directly. @@ -159,6 +159,7 @@ mmpose ├── tools ├── configs `── data + ├── panoptic_training_pose.pkl ├── campus | ├── Camera0 | | | ├── campus4-c0-00000.png @@ -171,7 +172,6 @@ mmpose | | | ├── campus4-c2-01999.png | ├── calibration_campus.json | ├── pred_campus_maskrcnn_hrnet_coco.pkl - | ├── panoptic_training_pose.pkl | ├── actorsGT.mat ├── shelf | ├── Camera0 @@ -185,6 +185,5 @@ mmpose | | | ├── img_003199.png | ├── calibration_shelf.json | ├── pred_shelf_maskrcnn_hrnet_coco.pkl - | ├── panoptic_training_pose.pkl | ├── actorsGT.mat ``` From 2f7a37fe2c2c218d926c853375bd638f4ff693db Mon Sep 17 00:00:00 2001 From: liqikai Date: Fri, 1 Apr 2022 22:06:03 +0800 Subject: [PATCH 19/19] add code license of the dataset --- .../datasets/body3d/body3d_mview_direct_campus_dataset.py | 5 +++++ .../datasets/body3d/body3d_mview_direct_shelf_dataset.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py b/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py index 332cfcaaaf..2ade0afc06 100644 --- a/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py +++ b/mmpose/datasets/datasets/body3d/body3d_mview_direct_campus_dataset.py @@ -1,4 +1,9 @@ # Copyright (c) OpenMMLab. All rights reserved. +# ------------------------------------------------------------------------------ +# Adapted from https://github.com/microsoft/voxelpose-pytorch +# Original license: Copyright (c) Microsoft Corporation, under the MIT License. +# ------------------------------------------------------------------------------ + import copy import os.path as osp import random diff --git a/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py b/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py index 7f63a0e842..9ed553dfbc 100644 --- a/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py +++ b/mmpose/datasets/datasets/body3d/body3d_mview_direct_shelf_dataset.py @@ -1,4 +1,9 @@ # Copyright (c) OpenMMLab. All rights reserved. +# ------------------------------------------------------------------------------ +# Adapted from https://github.com/microsoft/voxelpose-pytorch +# Original license: Copyright (c) Microsoft Corporation, under the MIT License. +# ------------------------------------------------------------------------------ + import copy import os.path as osp import random