From 4bb4e512cb7cf045e83161d8111a44cf7819649c Mon Sep 17 00:00:00 2001 From: xiexinch Date: Mon, 31 Jul 2023 16:12:36 +0800 Subject: [PATCH 01/10] --init --- configs/_base_/datasets/ubody2d.py | 1152 +++++++++++++++++ ...d-hm_hrnet-w32_8xb64-210e_ubody-256x192.py | 175 +++ docs/en/dataset_zoo/2d_wholebody_keypoint.md | 82 ++ .../datasets/base/base_coco_style_dataset.py | 10 +- .../datasets/datasets/wholebody/__init__.py | 3 +- .../datasets/wholebody/ubody2d_dataset.py | 69 + .../dataset_converters/ubody_kpts_to_coco.py | 141 ++ 7 files changed, 1629 insertions(+), 3 deletions(-) create mode 100644 configs/_base_/datasets/ubody2d.py create mode 100644 configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py create mode 100644 mmpose/datasets/datasets/wholebody/ubody2d_dataset.py create mode 100644 tools/dataset_converters/ubody_kpts_to_coco.py diff --git a/configs/_base_/datasets/ubody2d.py b/configs/_base_/datasets/ubody2d.py new file mode 100644 index 0000000000..1530aad36a --- /dev/null +++ b/configs/_base_/datasets/ubody2d.py @@ -0,0 +1,1152 @@ +dataset_info = dict( + dataset_name='ubody2d', + paper_info=dict( + author='Jing Lin, Ailing Zeng, Haoqian Wang, Lei Zhang, Yu Li', + title='One-Stage 3D Whole-Body Mesh Recovery with Component Aware Transformer', + container='IEEE Computer Society Conference on Computer Vision and ' + 'Pattern Recognition (CVPR)', + year='2023', + homepage='https://github.com/IDEA-Research/OSX', + ), + keypoint_info={ + 0: + dict(name='nose', id=0, color=[51, 153, 255], type='upper', swap=''), + 1: + dict( + name='left_eye', + id=1, + color=[51, 153, 255], + type='upper', + swap='right_eye'), + 2: + dict( + name='right_eye', + id=2, + color=[51, 153, 255], + type='upper', + swap='left_eye'), + 3: + dict( + name='left_ear', + id=3, + color=[51, 153, 255], + type='upper', + swap='right_ear'), + 4: + dict( + name='right_ear', + id=4, + color=[51, 153, 255], + type='upper', + swap='left_ear'), + 5: + dict( + name='left_shoulder', + id=5, + color=[0, 255, 0], + type='upper', + swap='right_shoulder'), + 6: + dict( + name='right_shoulder', + id=6, + color=[255, 128, 0], + type='upper', + swap='left_shoulder'), + 7: + dict( + name='left_elbow', + id=7, + color=[0, 255, 0], + type='upper', + swap='right_elbow'), + 8: + dict( + name='right_elbow', + id=8, + color=[255, 128, 0], + type='upper', + swap='left_elbow'), + 9: + dict( + name='left_wrist', + id=9, + color=[0, 255, 0], + type='upper', + swap='right_wrist'), + 10: + dict( + name='right_wrist', + id=10, + color=[255, 128, 0], + type='upper', + swap='left_wrist'), + 11: + dict( + name='left_hip', + id=11, + color=[0, 255, 0], + type='lower', + swap='right_hip'), + 12: + dict( + name='right_hip', + id=12, + color=[255, 128, 0], + type='lower', + swap='left_hip'), + 13: + dict( + name='left_knee', + id=13, + color=[0, 255, 0], + type='lower', + swap='right_knee'), + 14: + dict( + name='right_knee', + id=14, + color=[255, 128, 0], + type='lower', + swap='left_knee'), + 15: + dict( + name='left_ankle', + id=15, + color=[0, 255, 0], + type='lower', + swap='right_ankle'), + 16: + dict( + name='right_ankle', + id=16, + color=[255, 128, 0], + type='lower', + swap='left_ankle'), + 17: + dict( + name='left_big_toe', + id=17, + color=[255, 128, 0], + type='lower', + swap='right_big_toe'), + 18: + dict( + name='left_small_toe', + id=18, + color=[255, 128, 0], + type='lower', + swap='right_small_toe'), + 19: + dict( + name='left_heel', + id=19, + color=[255, 128, 0], + type='lower', + swap='right_heel'), + 20: + dict( + name='right_big_toe', + id=20, + color=[255, 128, 0], + type='lower', + swap='left_big_toe'), + 21: + dict( + name='right_small_toe', + id=21, + color=[255, 128, 0], + type='lower', + swap='left_small_toe'), + 22: + dict( + name='right_heel', + id=22, + color=[255, 128, 0], + type='lower', + swap='left_heel'), + 23: + dict( + name='face-0', + id=23, + color=[255, 255, 255], + type='', + swap='face-16'), + 24: + dict( + name='face-1', + id=24, + color=[255, 255, 255], + type='', + swap='face-15'), + 25: + dict( + name='face-2', + id=25, + color=[255, 255, 255], + type='', + swap='face-14'), + 26: + dict( + name='face-3', + id=26, + color=[255, 255, 255], + type='', + swap='face-13'), + 27: + dict( + name='face-4', + id=27, + color=[255, 255, 255], + type='', + swap='face-12'), + 28: + dict( + name='face-5', + id=28, + color=[255, 255, 255], + type='', + swap='face-11'), + 29: + dict( + name='face-6', + id=29, + color=[255, 255, 255], + type='', + swap='face-10'), + 30: + dict( + name='face-7', + id=30, + color=[255, 255, 255], + type='', + swap='face-9'), + 31: + dict(name='face-8', id=31, color=[255, 255, 255], type='', swap=''), + 32: + dict( + name='face-9', + id=32, + color=[255, 255, 255], + type='', + swap='face-7'), + 33: + dict( + name='face-10', + id=33, + color=[255, 255, 255], + type='', + swap='face-6'), + 34: + dict( + name='face-11', + id=34, + color=[255, 255, 255], + type='', + swap='face-5'), + 35: + dict( + name='face-12', + id=35, + color=[255, 255, 255], + type='', + swap='face-4'), + 36: + dict( + name='face-13', + id=36, + color=[255, 255, 255], + type='', + swap='face-3'), + 37: + dict( + name='face-14', + id=37, + color=[255, 255, 255], + type='', + swap='face-2'), + 38: + dict( + name='face-15', + id=38, + color=[255, 255, 255], + type='', + swap='face-1'), + 39: + dict( + name='face-16', + id=39, + color=[255, 255, 255], + type='', + swap='face-0'), + 40: + dict( + name='face-17', + id=40, + color=[255, 255, 255], + type='', + swap='face-26'), + 41: + dict( + name='face-18', + id=41, + color=[255, 255, 255], + type='', + swap='face-25'), + 42: + dict( + name='face-19', + id=42, + color=[255, 255, 255], + type='', + swap='face-24'), + 43: + dict( + name='face-20', + id=43, + color=[255, 255, 255], + type='', + swap='face-23'), + 44: + dict( + name='face-21', + id=44, + color=[255, 255, 255], + type='', + swap='face-22'), + 45: + dict( + name='face-22', + id=45, + color=[255, 255, 255], + type='', + swap='face-21'), + 46: + dict( + name='face-23', + id=46, + color=[255, 255, 255], + type='', + swap='face-20'), + 47: + dict( + name='face-24', + id=47, + color=[255, 255, 255], + type='', + swap='face-19'), + 48: + dict( + name='face-25', + id=48, + color=[255, 255, 255], + type='', + swap='face-18'), + 49: + dict( + name='face-26', + id=49, + color=[255, 255, 255], + type='', + swap='face-17'), + 50: + dict(name='face-27', id=50, color=[255, 255, 255], type='', swap=''), + 51: + dict(name='face-28', id=51, color=[255, 255, 255], type='', swap=''), + 52: + dict(name='face-29', id=52, color=[255, 255, 255], type='', swap=''), + 53: + dict(name='face-30', id=53, color=[255, 255, 255], type='', swap=''), + 54: + dict( + name='face-31', + id=54, + color=[255, 255, 255], + type='', + swap='face-35'), + 55: + dict( + name='face-32', + id=55, + color=[255, 255, 255], + type='', + swap='face-34'), + 56: + dict(name='face-33', id=56, color=[255, 255, 255], type='', swap=''), + 57: + dict( + name='face-34', + id=57, + color=[255, 255, 255], + type='', + swap='face-32'), + 58: + dict( + name='face-35', + id=58, + color=[255, 255, 255], + type='', + swap='face-31'), + 59: + dict( + name='face-36', + id=59, + color=[255, 255, 255], + type='', + swap='face-45'), + 60: + dict( + name='face-37', + id=60, + color=[255, 255, 255], + type='', + swap='face-44'), + 61: + dict( + name='face-38', + id=61, + color=[255, 255, 255], + type='', + swap='face-43'), + 62: + dict( + name='face-39', + id=62, + color=[255, 255, 255], + type='', + swap='face-42'), + 63: + dict( + name='face-40', + id=63, + color=[255, 255, 255], + type='', + swap='face-47'), + 64: + dict( + name='face-41', + id=64, + color=[255, 255, 255], + type='', + swap='face-46'), + 65: + dict( + name='face-42', + id=65, + color=[255, 255, 255], + type='', + swap='face-39'), + 66: + dict( + name='face-43', + id=66, + color=[255, 255, 255], + type='', + swap='face-38'), + 67: + dict( + name='face-44', + id=67, + color=[255, 255, 255], + type='', + swap='face-37'), + 68: + dict( + name='face-45', + id=68, + color=[255, 255, 255], + type='', + swap='face-36'), + 69: + dict( + name='face-46', + id=69, + color=[255, 255, 255], + type='', + swap='face-41'), + 70: + dict( + name='face-47', + id=70, + color=[255, 255, 255], + type='', + swap='face-40'), + 71: + dict( + name='face-48', + id=71, + color=[255, 255, 255], + type='', + swap='face-54'), + 72: + dict( + name='face-49', + id=72, + color=[255, 255, 255], + type='', + swap='face-53'), + 73: + dict( + name='face-50', + id=73, + color=[255, 255, 255], + type='', + swap='face-52'), + 74: + dict(name='face-51', id=74, color=[255, 255, 255], type='', swap=''), + 75: + dict( + name='face-52', + id=75, + color=[255, 255, 255], + type='', + swap='face-50'), + 76: + dict( + name='face-53', + id=76, + color=[255, 255, 255], + type='', + swap='face-49'), + 77: + dict( + name='face-54', + id=77, + color=[255, 255, 255], + type='', + swap='face-48'), + 78: + dict( + name='face-55', + id=78, + color=[255, 255, 255], + type='', + swap='face-59'), + 79: + dict( + name='face-56', + id=79, + color=[255, 255, 255], + type='', + swap='face-58'), + 80: + dict(name='face-57', id=80, color=[255, 255, 255], type='', swap=''), + 81: + dict( + name='face-58', + id=81, + color=[255, 255, 255], + type='', + swap='face-56'), + 82: + dict( + name='face-59', + id=82, + color=[255, 255, 255], + type='', + swap='face-55'), + 83: + dict( + name='face-60', + id=83, + color=[255, 255, 255], + type='', + swap='face-64'), + 84: + dict( + name='face-61', + id=84, + color=[255, 255, 255], + type='', + swap='face-63'), + 85: + dict(name='face-62', id=85, color=[255, 255, 255], type='', swap=''), + 86: + dict( + name='face-63', + id=86, + color=[255, 255, 255], + type='', + swap='face-61'), + 87: + dict( + name='face-64', + id=87, + color=[255, 255, 255], + type='', + swap='face-60'), + 88: + dict( + name='face-65', + id=88, + color=[255, 255, 255], + type='', + swap='face-67'), + 89: + dict(name='face-66', id=89, color=[255, 255, 255], type='', swap=''), + 90: + dict( + name='face-67', + id=90, + color=[255, 255, 255], + type='', + swap='face-65'), + 91: + dict( + name='left_hand_root', + id=91, + color=[255, 255, 255], + type='', + swap='right_hand_root'), + 92: + dict( + name='left_thumb1', + id=92, + color=[255, 128, 0], + type='', + swap='right_thumb1'), + 93: + dict( + name='left_thumb2', + id=93, + color=[255, 128, 0], + type='', + swap='right_thumb2'), + 94: + dict( + name='left_thumb3', + id=94, + color=[255, 128, 0], + type='', + swap='right_thumb3'), + 95: + dict( + name='left_thumb4', + id=95, + color=[255, 128, 0], + type='', + swap='right_thumb4'), + 96: + dict( + name='left_forefinger1', + id=96, + color=[255, 153, 255], + type='', + swap='right_forefinger1'), + 97: + dict( + name='left_forefinger2', + id=97, + color=[255, 153, 255], + type='', + swap='right_forefinger2'), + 98: + dict( + name='left_forefinger3', + id=98, + color=[255, 153, 255], + type='', + swap='right_forefinger3'), + 99: + dict( + name='left_forefinger4', + id=99, + color=[255, 153, 255], + type='', + swap='right_forefinger4'), + 100: + dict( + name='left_middle_finger1', + id=100, + color=[102, 178, 255], + type='', + swap='right_middle_finger1'), + 101: + dict( + name='left_middle_finger2', + id=101, + color=[102, 178, 255], + type='', + swap='right_middle_finger2'), + 102: + dict( + name='left_middle_finger3', + id=102, + color=[102, 178, 255], + type='', + swap='right_middle_finger3'), + 103: + dict( + name='left_middle_finger4', + id=103, + color=[102, 178, 255], + type='', + swap='right_middle_finger4'), + 104: + dict( + name='left_ring_finger1', + id=104, + color=[255, 51, 51], + type='', + swap='right_ring_finger1'), + 105: + dict( + name='left_ring_finger2', + id=105, + color=[255, 51, 51], + type='', + swap='right_ring_finger2'), + 106: + dict( + name='left_ring_finger3', + id=106, + color=[255, 51, 51], + type='', + swap='right_ring_finger3'), + 107: + dict( + name='left_ring_finger4', + id=107, + color=[255, 51, 51], + type='', + swap='right_ring_finger4'), + 108: + dict( + name='left_pinky_finger1', + id=108, + color=[0, 255, 0], + type='', + swap='right_pinky_finger1'), + 109: + dict( + name='left_pinky_finger2', + id=109, + color=[0, 255, 0], + type='', + swap='right_pinky_finger2'), + 110: + dict( + name='left_pinky_finger3', + id=110, + color=[0, 255, 0], + type='', + swap='right_pinky_finger3'), + 111: + dict( + name='left_pinky_finger4', + id=111, + color=[0, 255, 0], + type='', + swap='right_pinky_finger4'), + 112: + dict( + name='right_hand_root', + id=112, + color=[255, 255, 255], + type='', + swap='left_hand_root'), + 113: + dict( + name='right_thumb1', + id=113, + color=[255, 128, 0], + type='', + swap='left_thumb1'), + 114: + dict( + name='right_thumb2', + id=114, + color=[255, 128, 0], + type='', + swap='left_thumb2'), + 115: + dict( + name='right_thumb3', + id=115, + color=[255, 128, 0], + type='', + swap='left_thumb3'), + 116: + dict( + name='right_thumb4', + id=116, + color=[255, 128, 0], + type='', + swap='left_thumb4'), + 117: + dict( + name='right_forefinger1', + id=117, + color=[255, 153, 255], + type='', + swap='left_forefinger1'), + 118: + dict( + name='right_forefinger2', + id=118, + color=[255, 153, 255], + type='', + swap='left_forefinger2'), + 119: + dict( + name='right_forefinger3', + id=119, + color=[255, 153, 255], + type='', + swap='left_forefinger3'), + 120: + dict( + name='right_forefinger4', + id=120, + color=[255, 153, 255], + type='', + swap='left_forefinger4'), + 121: + dict( + name='right_middle_finger1', + id=121, + color=[102, 178, 255], + type='', + swap='left_middle_finger1'), + 122: + dict( + name='right_middle_finger2', + id=122, + color=[102, 178, 255], + type='', + swap='left_middle_finger2'), + 123: + dict( + name='right_middle_finger3', + id=123, + color=[102, 178, 255], + type='', + swap='left_middle_finger3'), + 124: + dict( + name='right_middle_finger4', + id=124, + color=[102, 178, 255], + type='', + swap='left_middle_finger4'), + 125: + dict( + name='right_ring_finger1', + id=125, + color=[255, 51, 51], + type='', + swap='left_ring_finger1'), + 126: + dict( + name='right_ring_finger2', + id=126, + color=[255, 51, 51], + type='', + swap='left_ring_finger2'), + 127: + dict( + name='right_ring_finger3', + id=127, + color=[255, 51, 51], + type='', + swap='left_ring_finger3'), + 128: + dict( + name='right_ring_finger4', + id=128, + color=[255, 51, 51], + type='', + swap='left_ring_finger4'), + 129: + dict( + name='right_pinky_finger1', + id=129, + color=[0, 255, 0], + type='', + swap='left_pinky_finger1'), + 130: + dict( + name='right_pinky_finger2', + id=130, + color=[0, 255, 0], + type='', + swap='left_pinky_finger2'), + 131: + dict( + name='right_pinky_finger3', + id=131, + color=[0, 255, 0], + type='', + swap='left_pinky_finger3'), + 132: + dict( + name='right_pinky_finger4', + id=132, + color=[0, 255, 0], + type='', + swap='left_pinky_finger4') + }, + skeleton_info={ + 0: + dict(link=('left_ankle', 'left_knee'), id=0, color=[0, 255, 0]), + 1: + dict(link=('left_knee', 'left_hip'), id=1, color=[0, 255, 0]), + 2: + dict(link=('right_ankle', 'right_knee'), id=2, color=[255, 128, 0]), + 3: + dict(link=('right_knee', 'right_hip'), id=3, color=[255, 128, 0]), + 4: + dict(link=('left_hip', 'right_hip'), id=4, color=[51, 153, 255]), + 5: + dict(link=('left_shoulder', 'left_hip'), id=5, color=[51, 153, 255]), + 6: + dict(link=('right_shoulder', 'right_hip'), id=6, color=[51, 153, 255]), + 7: + dict( + link=('left_shoulder', 'right_shoulder'), + id=7, + color=[51, 153, 255]), + 8: + dict(link=('left_shoulder', 'left_elbow'), id=8, color=[0, 255, 0]), + 9: + dict( + link=('right_shoulder', 'right_elbow'), id=9, color=[255, 128, 0]), + 10: + dict(link=('left_elbow', 'left_wrist'), id=10, color=[0, 255, 0]), + 11: + dict(link=('right_elbow', 'right_wrist'), id=11, color=[255, 128, 0]), + 12: + dict(link=('left_eye', 'right_eye'), id=12, color=[51, 153, 255]), + 13: + dict(link=('nose', 'left_eye'), id=13, color=[51, 153, 255]), + 14: + dict(link=('nose', 'right_eye'), id=14, color=[51, 153, 255]), + 15: + dict(link=('left_eye', 'left_ear'), id=15, color=[51, 153, 255]), + 16: + dict(link=('right_eye', 'right_ear'), id=16, color=[51, 153, 255]), + 17: + dict(link=('left_ear', 'left_shoulder'), id=17, color=[51, 153, 255]), + 18: + dict( + link=('right_ear', 'right_shoulder'), id=18, color=[51, 153, 255]), + 19: + dict(link=('left_ankle', 'left_big_toe'), id=19, color=[0, 255, 0]), + 20: + dict(link=('left_ankle', 'left_small_toe'), id=20, color=[0, 255, 0]), + 21: + dict(link=('left_ankle', 'left_heel'), id=21, color=[0, 255, 0]), + 22: + dict( + link=('right_ankle', 'right_big_toe'), id=22, color=[255, 128, 0]), + 23: + dict( + link=('right_ankle', 'right_small_toe'), + id=23, + color=[255, 128, 0]), + 24: + dict(link=('right_ankle', 'right_heel'), id=24, color=[255, 128, 0]), + 25: + dict( + link=('left_hand_root', 'left_thumb1'), id=25, color=[255, 128, + 0]), + 26: + dict(link=('left_thumb1', 'left_thumb2'), id=26, color=[255, 128, 0]), + 27: + dict(link=('left_thumb2', 'left_thumb3'), id=27, color=[255, 128, 0]), + 28: + dict(link=('left_thumb3', 'left_thumb4'), id=28, color=[255, 128, 0]), + 29: + dict( + link=('left_hand_root', 'left_forefinger1'), + id=29, + color=[255, 153, 255]), + 30: + dict( + link=('left_forefinger1', 'left_forefinger2'), + id=30, + color=[255, 153, 255]), + 31: + dict( + link=('left_forefinger2', 'left_forefinger3'), + id=31, + color=[255, 153, 255]), + 32: + dict( + link=('left_forefinger3', 'left_forefinger4'), + id=32, + color=[255, 153, 255]), + 33: + dict( + link=('left_hand_root', 'left_middle_finger1'), + id=33, + color=[102, 178, 255]), + 34: + dict( + link=('left_middle_finger1', 'left_middle_finger2'), + id=34, + color=[102, 178, 255]), + 35: + dict( + link=('left_middle_finger2', 'left_middle_finger3'), + id=35, + color=[102, 178, 255]), + 36: + dict( + link=('left_middle_finger3', 'left_middle_finger4'), + id=36, + color=[102, 178, 255]), + 37: + dict( + link=('left_hand_root', 'left_ring_finger1'), + id=37, + color=[255, 51, 51]), + 38: + dict( + link=('left_ring_finger1', 'left_ring_finger2'), + id=38, + color=[255, 51, 51]), + 39: + dict( + link=('left_ring_finger2', 'left_ring_finger3'), + id=39, + color=[255, 51, 51]), + 40: + dict( + link=('left_ring_finger3', 'left_ring_finger4'), + id=40, + color=[255, 51, 51]), + 41: + dict( + link=('left_hand_root', 'left_pinky_finger1'), + id=41, + color=[0, 255, 0]), + 42: + dict( + link=('left_pinky_finger1', 'left_pinky_finger2'), + id=42, + color=[0, 255, 0]), + 43: + dict( + link=('left_pinky_finger2', 'left_pinky_finger3'), + id=43, + color=[0, 255, 0]), + 44: + dict( + link=('left_pinky_finger3', 'left_pinky_finger4'), + id=44, + color=[0, 255, 0]), + 45: + dict( + link=('right_hand_root', 'right_thumb1'), + id=45, + color=[255, 128, 0]), + 46: + dict( + link=('right_thumb1', 'right_thumb2'), id=46, color=[255, 128, 0]), + 47: + dict( + link=('right_thumb2', 'right_thumb3'), id=47, color=[255, 128, 0]), + 48: + dict( + link=('right_thumb3', 'right_thumb4'), id=48, color=[255, 128, 0]), + 49: + dict( + link=('right_hand_root', 'right_forefinger1'), + id=49, + color=[255, 153, 255]), + 50: + dict( + link=('right_forefinger1', 'right_forefinger2'), + id=50, + color=[255, 153, 255]), + 51: + dict( + link=('right_forefinger2', 'right_forefinger3'), + id=51, + color=[255, 153, 255]), + 52: + dict( + link=('right_forefinger3', 'right_forefinger4'), + id=52, + color=[255, 153, 255]), + 53: + dict( + link=('right_hand_root', 'right_middle_finger1'), + id=53, + color=[102, 178, 255]), + 54: + dict( + link=('right_middle_finger1', 'right_middle_finger2'), + id=54, + color=[102, 178, 255]), + 55: + dict( + link=('right_middle_finger2', 'right_middle_finger3'), + id=55, + color=[102, 178, 255]), + 56: + dict( + link=('right_middle_finger3', 'right_middle_finger4'), + id=56, + color=[102, 178, 255]), + 57: + dict( + link=('right_hand_root', 'right_ring_finger1'), + id=57, + color=[255, 51, 51]), + 58: + dict( + link=('right_ring_finger1', 'right_ring_finger2'), + id=58, + color=[255, 51, 51]), + 59: + dict( + link=('right_ring_finger2', 'right_ring_finger3'), + id=59, + color=[255, 51, 51]), + 60: + dict( + link=('right_ring_finger3', 'right_ring_finger4'), + id=60, + color=[255, 51, 51]), + 61: + dict( + link=('right_hand_root', 'right_pinky_finger1'), + id=61, + color=[0, 255, 0]), + 62: + dict( + link=('right_pinky_finger1', 'right_pinky_finger2'), + id=62, + color=[0, 255, 0]), + 63: + dict( + link=('right_pinky_finger2', 'right_pinky_finger3'), + id=63, + color=[0, 255, 0]), + 64: + dict( + link=('right_pinky_finger3', 'right_pinky_finger4'), + id=64, + color=[0, 255, 0]) + }, + joint_weights=[1.] * 133, + # 'https://github.com/jin-s13/COCO-WholeBody/blob/master/' + # 'evaluation/myeval_wholebody.py#L175' + sigmas=[ + 0.026, 0.025, 0.025, 0.035, 0.035, 0.079, 0.079, 0.072, 0.072, 0.062, + 0.062, 0.107, 0.107, 0.087, 0.087, 0.089, 0.089, 0.068, 0.066, 0.066, + 0.092, 0.094, 0.094, 0.042, 0.043, 0.044, 0.043, 0.040, 0.035, 0.031, + 0.025, 0.020, 0.023, 0.029, 0.032, 0.037, 0.038, 0.043, 0.041, 0.045, + 0.013, 0.012, 0.011, 0.011, 0.012, 0.012, 0.011, 0.011, 0.013, 0.015, + 0.009, 0.007, 0.007, 0.007, 0.012, 0.009, 0.008, 0.016, 0.010, 0.017, + 0.011, 0.009, 0.011, 0.009, 0.007, 0.013, 0.008, 0.011, 0.012, 0.010, + 0.034, 0.008, 0.008, 0.009, 0.008, 0.008, 0.007, 0.010, 0.008, 0.009, + 0.009, 0.009, 0.007, 0.007, 0.008, 0.011, 0.008, 0.008, 0.008, 0.01, + 0.008, 0.029, 0.022, 0.035, 0.037, 0.047, 0.026, 0.025, 0.024, 0.035, + 0.018, 0.024, 0.022, 0.026, 0.017, 0.021, 0.021, 0.032, 0.02, 0.019, + 0.022, 0.031, 0.029, 0.022, 0.035, 0.037, 0.047, 0.026, 0.025, 0.024, + 0.035, 0.018, 0.024, 0.022, 0.026, 0.017, 0.021, 0.021, 0.032, 0.02, + 0.019, 0.022, 0.031 + ]) diff --git a/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py b/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py new file mode 100644 index 0000000000..e52667dc8a --- /dev/null +++ b/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py @@ -0,0 +1,175 @@ +_base_ = ['../../../_base_/default_runtime.py'] + +# runtime +train_cfg = dict(max_epochs=210, val_interval=10) + +# optimizer +optim_wrapper = dict(optimizer=dict( + type='Adam', + lr=5e-4, +)) + +# learning policy +param_scheduler = [ + dict( + type='LinearLR', begin=0, end=500, start_factor=0.001, + by_epoch=False), # warm-up + dict( + type='MultiStepLR', + begin=0, + end=210, + milestones=[170, 200], + gamma=0.1, + by_epoch=True) +] + +# automatically scaling LR based on the actual training batch size +auto_scale_lr = dict(base_batch_size=512) + +# hooks +default_hooks = dict( + checkpoint=dict(save_best='coco-wholebody/AP', rule='greater')) + +# codec settings +codec = dict( + type='MSRAHeatmap', input_size=(192, 256), heatmap_size=(48, 64), sigma=2) + +# model settings +model = dict( + type='TopdownPoseEstimator', + data_preprocessor=dict( + type='PoseDataPreprocessor', + mean=[123.675, 116.28, 103.53], + std=[58.395, 57.12, 57.375], + bgr_to_rgb=True), + backbone=dict( + type='HRNet', + in_channels=3, + extra=dict( + stage1=dict( + num_modules=1, + num_branches=1, + block='BOTTLENECK', + num_blocks=(4, ), + num_channels=(64, )), + stage2=dict( + num_modules=1, + num_branches=2, + block='BASIC', + num_blocks=(4, 4), + num_channels=(32, 64)), + stage3=dict( + num_modules=4, + num_branches=3, + block='BASIC', + num_blocks=(4, 4, 4), + num_channels=(32, 64, 128)), + stage4=dict( + num_modules=3, + num_branches=4, + block='BASIC', + num_blocks=(4, 4, 4, 4), + num_channels=(32, 64, 128, 256))), + init_cfg=dict( + type='Pretrained', + checkpoint='https://download.openmmlab.com/mmpose/' + 'pretrain_models/hrnet_w32-36af842e.pth'), + ), + head=dict( + type='HeatmapHead', + in_channels=32, + out_channels=133, + deconv_out_channels=None, + loss=dict(type='KeypointMSELoss', use_target_weight=True), + decoder=codec), + test_cfg=dict( + flip_test=True, + flip_mode='heatmap', + shift_heatmap=True, + )) + +# base dataset settings +dataset_type = 'UBody2dDataset' +data_mode = 'topdown' +data_root = 'data/UBody/' + +scenes = [ + 'Magic_show', 'Entertainment', + 'ConductMusic', 'Online_class', 'TalkShow', + 'Speech', 'Fitness', 'Interview', 'Olympic', 'TVShow', 'Singing', + 'SignLanguage', 'Movie', 'LiveVlog', 'VideoConference' +] + +train_datasets = [] +val_datasets = [] + +for scene in scenes: + train_dataset = dict( + type=dataset_type, + data_root=data_root, + data_mode=data_mode, + ann_file=f'annotations/{scene}/train_annotations.json', + data_prefix=dict(img='images/'), + pipeline=[]) + train_datasets.append(train_dataset) + + val_dataset = dict( + type=dataset_type, + data_root=data_root, + data_mode=data_mode, + ann_file=f'annotations/{scene}/val_annotations.json', + data_prefix=dict(img='images/'), + pipeline=[]) + val_datasets.append(val_dataset) + +# pipelines +train_pipeline = [ + dict(type='LoadImage'), + dict(type='GetBBoxCenterScale'), + dict(type='RandomFlip', direction='horizontal'), + dict(type='RandomHalfBody'), + dict(type='RandomBBoxTransform'), + dict(type='TopdownAffine', input_size=codec['input_size']), + dict(type='GenerateTarget', encoder=codec), + dict(type='PackPoseInputs') +] +val_pipeline = [ + dict(type='LoadImage'), + dict(type='GetBBoxCenterScale'), + dict(type='TopdownAffine', input_size=codec['input_size']), + dict(type='PackPoseInputs') +] + +# data loaders +train_dataloader = dict( + batch_size=64, + num_workers=2, + persistent_workers=True, + sampler=dict(type='DefaultSampler', shuffle=True), + dataset=dict( + type='CombinedDataset', + metainfo=dict(from_file='configs/_base_/datasets/coco_wholebody.py'), + datasets=train_datasets, + pipeline=train_pipeline, + test_mode=False, + )) +val_dataloader = dict( + batch_size=32, + num_workers=2, + persistent_workers=True, + drop_last=False, + sampler=dict(type='DefaultSampler', shuffle=False, round_up=False), + dataset=dict( + type='UBody2dDataset', + ann_file=data_root + 'annotations/val_annotation.json', + data_prefix=dict(img=data_root + 'images/'), + pipeline=val_pipeline, + test_mode=True, + )) +test_dataloader = val_dataloader + +val_evaluator = dict( + type='CocoWholeBodyMetric', + use_area=False, + ann_file=data_root + 'annotations/val_annotation.json') +test_evaluator = val_evaluator diff --git a/docs/en/dataset_zoo/2d_wholebody_keypoint.md b/docs/en/dataset_zoo/2d_wholebody_keypoint.md index a082c657c6..720ccf169a 100644 --- a/docs/en/dataset_zoo/2d_wholebody_keypoint.md +++ b/docs/en/dataset_zoo/2d_wholebody_keypoint.md @@ -131,3 +131,85 @@ mmpose Please also install the latest version of [Extended COCO API](https://github.com/jin-s13/xtcocoapi) (version>=1.5) to support Halpe evaluation: `pip install xtcocotools` + +## UBody + + + +
+UBody (CVPR'2023) + +```bibtex +@article{lin2023one, + title={One-Stage 3D Whole-Body Mesh Recovery with Component Aware Transformer}, + author={Lin, Jing and Zeng, Ailing and Wang, Haoqian and Zhang, Lei and Li, Yu}, + booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, + year={2023}, +} +``` + +
+ +
+ +
+ +For [Ubody](https://github.com/IDEA-Research/OSX) dataset, videos and annotations can be downloaded from [OSX homepage](https://github.com/IDEA-Research/OSX). + +Download and extract them under $MMPOSE/data, and make them look like this: + +```text +mmpose +├── mmpose +├── docs +├── tests +├── tools +├── configs +`── data + │── UBody + ├── annotations + │   ├── ConductMusic + │   ├── Entertainment + │   ├── Fitness + │   ├── Interview + │   ├── LiveVlog + │   ├── Magic_show + │   ├── Movie + │   ├── Olympic + │   ├── Online_class + │   ├── SignLanguage + │   ├── Singing + │   ├── Speech + │   ├── TVShow + │   ├── TalkShow + │   └── VideoConference + ├── splits + │   ├── inter_scene_test_list.npy + │   └── intra_scene_test_list.npy + ├── videos + │   ├── ConductMusic + │   ├── Entertainment + │   ├── Fitness + │   ├── Interview + │   ├── LiveVlog + │   ├── Magic_show + │   ├── Movie + │   ├── Olympic + │   ├── Online_class + │   ├── SignLanguage + │   ├── Singing + │   ├── Speech + │   ├── TVShow + │   ├── TalkShow + │   └── VideoConference +``` + +Convert videos to images then split them into train/val set: + +```shell +python tools/dataset_converters/ubody_kpts_to_coco.py +``` + +Please also install the latest version of [Extended COCO API](https://github.com/jin-s13/xtcocoapi) (version>=1.5) to support COCO-WholeBody evaluation: + +`pip install xtcocotools` diff --git a/mmpose/datasets/datasets/base/base_coco_style_dataset.py b/mmpose/datasets/datasets/base/base_coco_style_dataset.py index 3b592813d8..e2bdecbe49 100644 --- a/mmpose/datasets/datasets/base/base_coco_style_dataset.py +++ b/mmpose/datasets/datasets/base/base_coco_style_dataset.py @@ -56,6 +56,8 @@ class BaseCocoStyleDataset(BaseDataset): max_refetch (int, optional): If ``Basedataset.prepare_data`` get a None img. The maximum extra number of cycles to get a valid image. Default: 1000. + sample_interval (int, optional): The sample interval of the dataset. + Default: 1. """ METAINFO: dict = dict() @@ -73,7 +75,8 @@ def __init__(self, pipeline: List[Union[dict, Callable]] = [], test_mode: bool = False, lazy_init: bool = False, - max_refetch: int = 1000): + max_refetch: int = 1000, + sample_interval: int = 1): if data_mode not in {'topdown', 'bottomup'}: raise ValueError( @@ -94,7 +97,8 @@ def __init__(self, 'while "bbox_file" is only ' 'supported when `test_mode==True`.') self.bbox_file = bbox_file - + self.sample_interval = sample_interval + super().__init__( ann_file=ann_file, metainfo=metainfo, @@ -207,6 +211,8 @@ def _load_annotations(self) -> Tuple[List[dict], List[dict]]: image_list = [] for img_id in self.coco.getImgIds(): + if img_id % self.sample_interval != 0: + continue img = self.coco.loadImgs(img_id)[0] img.update({ 'img_id': diff --git a/mmpose/datasets/datasets/wholebody/__init__.py b/mmpose/datasets/datasets/wholebody/__init__.py index 156094c2b0..b3934fc225 100644 --- a/mmpose/datasets/datasets/wholebody/__init__.py +++ b/mmpose/datasets/datasets/wholebody/__init__.py @@ -1,5 +1,6 @@ # Copyright (c) OpenMMLab. All rights reserved. from .coco_wholebody_dataset import CocoWholeBodyDataset from .halpe_dataset import HalpeDataset +from .ubody2d_dataset import UBody2dDataset -__all__ = ['CocoWholeBodyDataset', 'HalpeDataset'] +__all__ = ['CocoWholeBodyDataset', 'HalpeDataset', 'UBody2dDataset'] diff --git a/mmpose/datasets/datasets/wholebody/ubody2d_dataset.py b/mmpose/datasets/datasets/wholebody/ubody2d_dataset.py new file mode 100644 index 0000000000..6f78da134b --- /dev/null +++ b/mmpose/datasets/datasets/wholebody/ubody2d_dataset.py @@ -0,0 +1,69 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy +import os.path as osp +from typing import Optional + +import numpy as np + +from mmpose.registry import DATASETS +from ..base import BaseCocoStyleDataset +from .coco_wholebody_dataset import CocoWholeBodyDataset + + +@DATASETS.register_module() +class UBody2dDataset(CocoWholeBodyDataset): + """Ubody2d dataset for pose estimation. + + "One-Stage 3D Whole-Body Mesh Recovery with Component Aware Transformer", CVPR'2023. + More details can be found in the `paper + `__ . + + Ubody2D keypoints:: + + 0-16: 17 body keypoints, + 17-22: 6 foot keypoints, + 23-90: 68 face keypoints, + 91-132: 42 hand keypoints + + In total, we have 133 keypoints for wholebody pose estimation. + + Args: + ann_file (str): Annotation file path. Default: ''. + bbox_file (str, optional): Detection result file path. If + ``bbox_file`` is set, detected bboxes loaded from this file will + be used instead of ground-truth bboxes. This setting is only for + evaluation, i.e., ignored when ``test_mode`` is ``False``. + Default: ``None``. + data_mode (str): Specifies the mode of data samples: ``'topdown'`` or + ``'bottomup'``. In ``'topdown'`` mode, each data sample contains + one instance; while in ``'bottomup'`` mode, each data sample + contains all instances in a image. Default: ``'topdown'`` + metainfo (dict, optional): Meta information for dataset, such as class + information. Default: ``None``. + data_root (str, optional): The root directory for ``data_prefix`` and + ``ann_file``. Default: ``None``. + data_prefix (dict, optional): Prefix for training data. Default: + ``dict(img=None, ann=None)``. + filter_cfg (dict, optional): Config for filter data. Default: `None`. + indices (int or Sequence[int], optional): Support using first few + data in annotation file to facilitate training/testing on a smaller + dataset. Default: ``None`` which means using all ``data_infos``. + serialize_data (bool, optional): Whether to hold memory using + serialized objects, when enabled, data loader workers can use + shared RAM from master process instead of making a copy. + Default: ``True``. + pipeline (list, optional): Processing pipeline. Default: []. + test_mode (bool, optional): ``test_mode=True`` means in test phase. + Default: ``False``. + lazy_init (bool, optional): Whether to load annotation during + instantiation. In some cases, such as visualization, only the meta + information of the dataset is needed, which is not necessary to + load annotation file. ``Basedataset`` can skip load annotations to + save time by set ``lazy_init=False``. Default: ``False``. + max_refetch (int, optional): If ``Basedataset.prepare_data`` get a + None img. The maximum extra number of cycles to get a valid + image. Default: 1000. + """ + + METAINFO: dict = dict( + from_file='configs/_base_/datasets/ubody2d.py') diff --git a/tools/dataset_converters/ubody_kpts_to_coco.py b/tools/dataset_converters/ubody_kpts_to_coco.py new file mode 100644 index 0000000000..548ea61fb7 --- /dev/null +++ b/tools/dataset_converters/ubody_kpts_to_coco.py @@ -0,0 +1,141 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import argparse +import os +from copy import deepcopy +from multiprocessing import Pool + +import mmengine +import numpy as np +from pycocotools.coco import COCO + +from multiprocessing import Pool + + +def findAllFile(base): + file_path = [] + for root, ds, fs in os.walk(base): + for f in fs: + fullname = os.path.join(root, f) + file_path.append(fullname) + return file_path + + +def convert(video_path: str): + video_name = video_path.split('/')[-1] + image_path = video_path.replace(video_name, video_name.split('.')[0]) + image_path = image_path.replace('/videos/', '/images/') + os.makedirs(image_path, exist_ok=True) + print( + f'ffmpeg -i {video_path} -f image2 -r 30 -b:v 5626k {image_path}/%06d.png' # noqa + ) + os.system( + f'ffmpeg -i {video_path} -f image2 -r 30 -b:v 5626k {image_path}/%06d.png' # noqa + ) + + +def split_dataset(annotation_path: str, split_path: str): + folders = os.listdir(annotation_path) + splits = np.load(split_path) + train_annos = [] + val_annos = [] + train_imgs = [] + val_imgs = [] + t_id = 0 + v_id = 0 + categories = [{'supercategory': 'person', 'id': 1, 'name': 'person'}] + + for scene in folders: + scene_train_anns = [] + scene_val_anns = [] + scene_train_imgs = [] + scene_val_imgs = [] + data = COCO( + os.path.join(annotation_path, scene, 'keypoint_annotation.json')) + print(f'Processing {scene}.........') + progress_bar = mmengine.ProgressBar(len(data.anns.keys())) + for aid in data.anns.keys(): + ann = data.anns[aid] + img = data.loadImgs(ann['image_id'])[0] + + if img['file_name'].startswith('/'): + file_name = img['file_name'][1:] # [1:] means delete '/' + else: + file_name = img['file_name'] + video_name = file_name.split('/')[-2] + if 'Trim' in video_name: + video_name = video_name.split('_Trim')[0] + + img_path = os.path.join( + annotation_path.replace('annotations', 'images'), scene, + file_name) + if not os.path.exists(img_path): + progress_bar.update() + continue + + img['file_name'] = os.path.join(scene, file_name) + ann_ = deepcopy(ann) + img_ = deepcopy(img) + if video_name in splits: + scene_val_anns.append(ann) + scene_val_imgs.append(img) + ann_['id'] = v_id + ann_['image_id'] = v_id + img_['id'] = v_id + val_annos.append(ann_) + val_imgs.append(img_) + v_id += 1 + else: + scene_train_anns.append(ann) + scene_train_imgs.append(img) + ann_['id'] = t_id + ann_['image_id'] = t_id + img_['id'] =t_id + train_annos.append(ann_) + train_imgs.append(img_) + t_id += 1 + + progress_bar.update() + + scene_train_data = dict( + images=scene_train_imgs, + annotations=scene_train_anns, + categories=categories) + scene_val_data = dict( + images=scene_val_imgs, + annotations=scene_val_anns, + categories=categories) + + mmengine.dump( + scene_train_data, + os.path.join(annotation_path, scene, 'train_annotations.json')) + mmengine.dump( + scene_val_data, + os.path.join(annotation_path, scene, 'val_annotations.json')) + + train_data = dict( + images=train_imgs, annotations=train_annos, categories=categories) + val_data = dict( + images=val_imgs, annotations=val_annos, categories=categories) + + + mmengine.dump(train_data, + os.path.join(annotation_path, 'train_annotation.json')) + mmengine.dump(val_data, os.path.join(annotation_path, + 'val_annotation.json')) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--data-root', type=str, default='data/UBody') + args = parser.parse_args() + video_root = f'{args.data_root}/videos' + split_path = f'{args.data_root}/splits/intra_scene_test_list.npy' + annotation_path = f'{args.data_root}/annotations' + + video_paths = findAllFile(video_root) + pool = Pool(processes=1) + pool.map(convert, video_paths) + pool.close() + pool.join() + + split_dataset(annotation_path, split_path) From 6fedd07307847e7e65f228813e49200d5ea899ba Mon Sep 17 00:00:00 2001 From: xiexinch Date: Mon, 31 Jul 2023 16:19:16 +0800 Subject: [PATCH 02/10] --other=fix lint --- configs/_base_/datasets/ubody2d.py | 3 ++- .../td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py | 3 +-- docs/en/dataset_zoo/2d_wholebody_keypoint.md | 2 +- .../datasets/base/base_coco_style_dataset.py | 2 +- .../datasets/datasets/wholebody/ubody2d_dataset.py | 14 +++----------- tools/dataset_converters/ubody_kpts_to_coco.py | 5 +---- 6 files changed, 9 insertions(+), 20 deletions(-) diff --git a/configs/_base_/datasets/ubody2d.py b/configs/_base_/datasets/ubody2d.py index 1530aad36a..8486db05ab 100644 --- a/configs/_base_/datasets/ubody2d.py +++ b/configs/_base_/datasets/ubody2d.py @@ -2,7 +2,8 @@ dataset_name='ubody2d', paper_info=dict( author='Jing Lin, Ailing Zeng, Haoqian Wang, Lei Zhang, Yu Li', - title='One-Stage 3D Whole-Body Mesh Recovery with Component Aware Transformer', + title='One-Stage 3D Whole-Body Mesh Recovery with Component Aware' + 'Transformer', container='IEEE Computer Society Conference on Computer Vision and ' 'Pattern Recognition (CVPR)', year='2023', diff --git a/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py b/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py index e52667dc8a..2b458829e8 100644 --- a/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py +++ b/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py @@ -94,8 +94,7 @@ data_root = 'data/UBody/' scenes = [ - 'Magic_show', 'Entertainment', - 'ConductMusic', 'Online_class', 'TalkShow', + 'Magic_show', 'Entertainment', 'ConductMusic', 'Online_class', 'TalkShow', 'Speech', 'Fitness', 'Interview', 'Olympic', 'TVShow', 'Singing', 'SignLanguage', 'Movie', 'LiveVlog', 'VideoConference' ] diff --git a/docs/en/dataset_zoo/2d_wholebody_keypoint.md b/docs/en/dataset_zoo/2d_wholebody_keypoint.md index 720ccf169a..b572ec69ce 100644 --- a/docs/en/dataset_zoo/2d_wholebody_keypoint.md +++ b/docs/en/dataset_zoo/2d_wholebody_keypoint.md @@ -207,7 +207,7 @@ mmpose Convert videos to images then split them into train/val set: ```shell -python tools/dataset_converters/ubody_kpts_to_coco.py +python tools/dataset_converters/ubody_kpts_to_coco.py ``` Please also install the latest version of [Extended COCO API](https://github.com/jin-s13/xtcocoapi) (version>=1.5) to support COCO-WholeBody evaluation: diff --git a/mmpose/datasets/datasets/base/base_coco_style_dataset.py b/mmpose/datasets/datasets/base/base_coco_style_dataset.py index e2bdecbe49..60c24d7c71 100644 --- a/mmpose/datasets/datasets/base/base_coco_style_dataset.py +++ b/mmpose/datasets/datasets/base/base_coco_style_dataset.py @@ -98,7 +98,7 @@ def __init__(self, 'supported when `test_mode==True`.') self.bbox_file = bbox_file self.sample_interval = sample_interval - + super().__init__( ann_file=ann_file, metainfo=metainfo, diff --git a/mmpose/datasets/datasets/wholebody/ubody2d_dataset.py b/mmpose/datasets/datasets/wholebody/ubody2d_dataset.py index 6f78da134b..cf2f6b3428 100644 --- a/mmpose/datasets/datasets/wholebody/ubody2d_dataset.py +++ b/mmpose/datasets/datasets/wholebody/ubody2d_dataset.py @@ -1,12 +1,5 @@ # Copyright (c) OpenMMLab. All rights reserved. -import copy -import os.path as osp -from typing import Optional - -import numpy as np - from mmpose.registry import DATASETS -from ..base import BaseCocoStyleDataset from .coco_wholebody_dataset import CocoWholeBodyDataset @@ -14,8 +7,8 @@ class UBody2dDataset(CocoWholeBodyDataset): """Ubody2d dataset for pose estimation. - "One-Stage 3D Whole-Body Mesh Recovery with Component Aware Transformer", CVPR'2023. - More details can be found in the `paper + "One-Stage 3D Whole-Body Mesh Recovery with Component Aware Transformer", + CVPR'2023. More details can be found in the `paper `__ . Ubody2D keypoints:: @@ -65,5 +58,4 @@ class UBody2dDataset(CocoWholeBodyDataset): image. Default: 1000. """ - METAINFO: dict = dict( - from_file='configs/_base_/datasets/ubody2d.py') + METAINFO: dict = dict(from_file='configs/_base_/datasets/ubody2d.py') diff --git a/tools/dataset_converters/ubody_kpts_to_coco.py b/tools/dataset_converters/ubody_kpts_to_coco.py index 548ea61fb7..070fe814de 100644 --- a/tools/dataset_converters/ubody_kpts_to_coco.py +++ b/tools/dataset_converters/ubody_kpts_to_coco.py @@ -8,8 +8,6 @@ import numpy as np from pycocotools.coco import COCO -from multiprocessing import Pool - def findAllFile(base): file_path = [] @@ -89,7 +87,7 @@ def split_dataset(annotation_path: str, split_path: str): scene_train_imgs.append(img) ann_['id'] = t_id ann_['image_id'] = t_id - img_['id'] =t_id + img_['id'] = t_id train_annos.append(ann_) train_imgs.append(img_) t_id += 1 @@ -117,7 +115,6 @@ def split_dataset(annotation_path: str, split_path: str): val_data = dict( images=val_imgs, annotations=val_annos, categories=categories) - mmengine.dump(train_data, os.path.join(annotation_path, 'train_annotation.json')) mmengine.dump(val_data, os.path.join(annotation_path, From 221d41cd5cc8464caf43c13b2a2afe3c215976c1 Mon Sep 17 00:00:00 2001 From: xiexinch Date: Mon, 31 Jul 2023 16:21:38 +0800 Subject: [PATCH 03/10] --config=update sample interval --- .../ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py b/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py index 2b458829e8..b5749c5f35 100644 --- a/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py +++ b/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py @@ -109,7 +109,8 @@ data_mode=data_mode, ann_file=f'annotations/{scene}/train_annotations.json', data_prefix=dict(img='images/'), - pipeline=[]) + pipeline=[], + sample_interval=10) train_datasets.append(train_dataset) val_dataset = dict( @@ -164,7 +165,7 @@ data_prefix=dict(img=data_root + 'images/'), pipeline=val_pipeline, test_mode=True, - )) + sample_interval=10)) test_dataloader = val_dataloader val_evaluator = dict( From 656a2705cd484202051bd5c2b67da12323195ca9 Mon Sep 17 00:00:00 2001 From: xiexinch Date: Tue, 1 Aug 2023 11:40:16 +0800 Subject: [PATCH 04/10] --other=fix whole body metric --- .../evaluation/metrics/coco_wholebody_metric.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/mmpose/evaluation/metrics/coco_wholebody_metric.py b/mmpose/evaluation/metrics/coco_wholebody_metric.py index c5675f54c8..74dc52c2ad 100644 --- a/mmpose/evaluation/metrics/coco_wholebody_metric.py +++ b/mmpose/evaluation/metrics/coco_wholebody_metric.py @@ -245,7 +245,7 @@ def _do_python_keypoint_eval(self, outfile_prefix: str) -> list: coco_det, 'keypoints_body', sigmas[cuts[0]:cuts[1]], - use_area=True) + use_area=self.use_area) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() @@ -256,7 +256,7 @@ def _do_python_keypoint_eval(self, outfile_prefix: str) -> list: coco_det, 'keypoints_foot', sigmas[cuts[1]:cuts[2]], - use_area=True) + use_area=self.use_area) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() @@ -267,7 +267,7 @@ def _do_python_keypoint_eval(self, outfile_prefix: str) -> list: coco_det, 'keypoints_face', sigmas[cuts[2]:cuts[3]], - use_area=True) + use_area=self.use_area) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() @@ -278,7 +278,7 @@ def _do_python_keypoint_eval(self, outfile_prefix: str) -> list: coco_det, 'keypoints_lefthand', sigmas[cuts[3]:cuts[4]], - use_area=True) + use_area=self.use_area) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() @@ -289,14 +289,18 @@ def _do_python_keypoint_eval(self, outfile_prefix: str) -> list: coco_det, 'keypoints_righthand', sigmas[cuts[4]:cuts[5]], - use_area=True) + use_area=self.use_area) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() coco_eval = COCOeval( - self.coco, coco_det, 'keypoints_wholebody', sigmas, use_area=True) + self.coco, + coco_det, + 'keypoints_wholebody', + sigmas, + use_area=self.use_area) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() From ecc9c5f64c0769386c85df287a72e7363a131153 Mon Sep 17 00:00:00 2001 From: xiexinch Date: Tue, 1 Aug 2023 11:41:45 +0800 Subject: [PATCH 05/10] --other=remove val datasets --- .../td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py b/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py index b5749c5f35..dfa2d4d02e 100644 --- a/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py +++ b/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py @@ -100,7 +100,6 @@ ] train_datasets = [] -val_datasets = [] for scene in scenes: train_dataset = dict( @@ -113,15 +112,6 @@ sample_interval=10) train_datasets.append(train_dataset) - val_dataset = dict( - type=dataset_type, - data_root=data_root, - data_mode=data_mode, - ann_file=f'annotations/{scene}/val_annotations.json', - data_prefix=dict(img='images/'), - pipeline=[]) - val_datasets.append(val_dataset) - # pipelines train_pipeline = [ dict(type='LoadImage'), From b01849b917e9b0ae51e2bc2cb656efc57236c8e7 Mon Sep 17 00:00:00 2001 From: xiexinch Date: Tue, 1 Aug 2023 14:14:24 +0800 Subject: [PATCH 06/10] --other=minor change --- .../ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py | 7 +++---- tools/dataset_converters/ubody_kpts_to_coco.py | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py b/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py index dfa2d4d02e..5382917d8e 100644 --- a/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py +++ b/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py @@ -151,15 +151,14 @@ sampler=dict(type='DefaultSampler', shuffle=False, round_up=False), dataset=dict( type='UBody2dDataset', - ann_file=data_root + 'annotations/val_annotation.json', + ann_file=data_root + 'annotations/val_annotations.json', data_prefix=dict(img=data_root + 'images/'), pipeline=val_pipeline, - test_mode=True, - sample_interval=10)) + test_mode=True)) test_dataloader = val_dataloader val_evaluator = dict( type='CocoWholeBodyMetric', use_area=False, - ann_file=data_root + 'annotations/val_annotation.json') + ann_file=data_root + 'annotations/val_annotations.json') test_evaluator = val_evaluator diff --git a/tools/dataset_converters/ubody_kpts_to_coco.py b/tools/dataset_converters/ubody_kpts_to_coco.py index 070fe814de..9d927da7ca 100644 --- a/tools/dataset_converters/ubody_kpts_to_coco.py +++ b/tools/dataset_converters/ubody_kpts_to_coco.py @@ -116,9 +116,9 @@ def split_dataset(annotation_path: str, split_path: str): images=val_imgs, annotations=val_annos, categories=categories) mmengine.dump(train_data, - os.path.join(annotation_path, 'train_annotation.json')) - mmengine.dump(val_data, os.path.join(annotation_path, - 'val_annotation.json')) + os.path.join(annotation_path, 'train_annotations.json')) + mmengine.dump(val_data, + os.path.join(annotation_path, 'val_annotations.json')) if __name__ == '__main__': From b185fa3e00f5aec93e6ac2fba43fcc1ad227f40e Mon Sep 17 00:00:00 2001 From: xiexinch Date: Mon, 7 Aug 2023 14:36:42 +0800 Subject: [PATCH 07/10] --other=add training result --- .../ubody2d/hrnet_ubody-coco-wholebody.md | 38 +++++++++++++++++++ ...d-hm_hrnet-w32_8xb64-210e_ubody-256x192.py | 21 +++++++--- docs/src/papers/datasets/ubody.md | 17 +++++++++ .../datasets/wholebody/ubody2d_dataset.py | 2 + tools/test.py | 5 ++- 5 files changed, 76 insertions(+), 7 deletions(-) create mode 100644 configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/hrnet_ubody-coco-wholebody.md create mode 100644 docs/src/papers/datasets/ubody.md diff --git a/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/hrnet_ubody-coco-wholebody.md b/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/hrnet_ubody-coco-wholebody.md new file mode 100644 index 0000000000..1d05a323c4 --- /dev/null +++ b/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/hrnet_ubody-coco-wholebody.md @@ -0,0 +1,38 @@ + + +
+HRNet (CVPR'2019) + +```bibtex +@inproceedings{sun2019deep, + title={Deep high-resolution representation learning for human pose estimation}, + author={Sun, Ke and Xiao, Bin and Liu, Dong and Wang, Jingdong}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5693--5703}, + year={2019} +} +``` + +
+ + + +
+UBody (CVPR'2023) + +```bibtex +@article{lin2023one, + title={One-Stage 3D Whole-Body Mesh Recovery with Component Aware Transformer}, + author={Lin, Jing and Zeng, Ailing and Wang, Haoqian and Zhang, Lei and Li, Yu}, + booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, + year={2023}, +} +``` + +
+ +Results on COCO-WholeBody v1.0 val with detector having human AP of 54.9 on COCO val2017 dataset + +| Arch | Input Size | Body AP | Body AR | Foot AP | Foot AR | Face AP | Face AR | Hand AP | Hand AR | Whole AP | Whole AR | ckpt | log | +| :-------------------------------------- | :--------: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :------: | :------: | :--------------------------------------: | :-------------------------------------: | +| [pose_hrnet_w32](/configs/wholebody_2d_keypoint/topdown_heatmap/ubody/td-hm_hrnet-w32_8xb64-210e_coco-wholebody-256x192.py) | 256x192 | 0.685 | 0.759 | 0.564 | 0.675 | 0.625 | 0.705 | 0.516 | 0.609 | 0.549 | 0.646 | [ckpt](https://download.openmmlab.com/mmpose/v1/wholebody_2d_keypoint/ubody/td-hm_hrnet-w32_8xb64-210e_ubody-coco-256x192-7c227391_20230807.pth) | [log](https://download.openmmlab.com/mmpose/v1/wholebody_2d_keypoint/ubody/td-hm_hrnet-w32_8xb64-210e_ubody-coco-256x192-7c227391_20230807.json) | diff --git a/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py b/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py index 5382917d8e..055484d009 100644 --- a/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py +++ b/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py @@ -99,7 +99,15 @@ 'SignLanguage', 'Movie', 'LiveVlog', 'VideoConference' ] -train_datasets = [] +train_datasets = [ + dict( + type='CocoWholeBodyDataset', + data_root='data/coco/', + data_mode=data_mode, + ann_file='annotations/coco_wholebody_train_v1.0.json', + data_prefix=dict(img='train2017/'), + pipeline=[]) +] for scene in scenes: train_dataset = dict( @@ -150,15 +158,16 @@ drop_last=False, sampler=dict(type='DefaultSampler', shuffle=False, round_up=False), dataset=dict( - type='UBody2dDataset', - ann_file=data_root + 'annotations/val_annotations.json', - data_prefix=dict(img=data_root + 'images/'), + type='CocoWholeBodyDataset', + ann_file='data/coco/annotations/coco_wholebody_val_v1.0.json', + data_prefix=dict(img='data/coco/val2017/'), pipeline=val_pipeline, + bbox_file='data/coco/person_detection_results/' + 'COCO_val2017_detections_AP_H_56_person.json', test_mode=True)) test_dataloader = val_dataloader val_evaluator = dict( type='CocoWholeBodyMetric', - use_area=False, - ann_file=data_root + 'annotations/val_annotations.json') + ann_file='data/coco/annotations/coco_wholebody_val_v1.0.json') test_evaluator = val_evaluator diff --git a/docs/src/papers/datasets/ubody.md b/docs/src/papers/datasets/ubody.md new file mode 100644 index 0000000000..319eabe2b4 --- /dev/null +++ b/docs/src/papers/datasets/ubody.md @@ -0,0 +1,17 @@ +# One-Stage 3D Whole-Body Mesh Recovery with Component Aware Transformer + + + +
+UBody (CVPR'2023) + +```bibtex +@article{lin2023one, + title={One-Stage 3D Whole-Body Mesh Recovery with Component Aware Transformer}, + author={Lin, Jing and Zeng, Ailing and Wang, Haoqian and Zhang, Lei and Li, Yu}, + booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, + year={2023}, +} +``` + +
diff --git a/mmpose/datasets/datasets/wholebody/ubody2d_dataset.py b/mmpose/datasets/datasets/wholebody/ubody2d_dataset.py index cf2f6b3428..9a0cb1711a 100644 --- a/mmpose/datasets/datasets/wholebody/ubody2d_dataset.py +++ b/mmpose/datasets/datasets/wholebody/ubody2d_dataset.py @@ -56,6 +56,8 @@ class UBody2dDataset(CocoWholeBodyDataset): max_refetch (int, optional): If ``Basedataset.prepare_data`` get a None img. The maximum extra number of cycles to get a valid image. Default: 1000. + sample_interval (int, optional): The sample interval of the dataset. + Default: 1. """ METAINFO: dict = dict(from_file='configs/_base_/datasets/ubody2d.py') diff --git a/tools/test.py b/tools/test.py index 5dc0110260..513c95a6db 100644 --- a/tools/test.py +++ b/tools/test.py @@ -51,7 +51,10 @@ def parse_args(): choices=['none', 'pytorch', 'slurm', 'mpi'], default='none', help='job launcher') - parser.add_argument('--local_rank', type=int, default=0) + # When using PyTorch version >= 2.0.0, the `torch.distributed.launch` + # will pass the `--local-rank` parameter to `tools/test.py` instead + # of `--local_rank`. + parser.add_argument('--local_rank', '--local-rank', type=int, default=0) args = parser.parse_args() if 'LOCAL_RANK' not in os.environ: os.environ['LOCAL_RANK'] = str(args.local_rank) From a3aa0a8cd7cb87441103f41234740b0fc2d9be31 Mon Sep 17 00:00:00 2001 From: xiexinch Date: Mon, 7 Aug 2023 15:20:59 +0800 Subject: [PATCH 08/10] --other=update ubody val result and add yml --- .../topdown_heatmap/README.md | 9 ++++++++ .../ubody2d/hrnet_coco-wholebody.yml | 23 +++++++++++++++++++ .../ubody2d/hrnet_ubody-coco-wholebody.md | 2 +- 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/hrnet_coco-wholebody.yml diff --git a/configs/wholebody_2d_keypoint/topdown_heatmap/README.md b/configs/wholebody_2d_keypoint/topdown_heatmap/README.md index 23ee1ed315..71837c998b 100644 --- a/configs/wholebody_2d_keypoint/topdown_heatmap/README.md +++ b/configs/wholebody_2d_keypoint/topdown_heatmap/README.md @@ -18,9 +18,18 @@ Results on COCO-WholeBody v1.0 val with detector having human AP of 56.4 on COCO | HRNet-w32+Dark | 256x192 | 0.582 | 0.671 | [hrnet_dark_coco-wholebody.md](./coco-wholebody/hrnet_dark_coco-wholebody.md) | | HRNet-w48 | 256x192 | 0.579 | 0.681 | [hrnet_coco-wholebody.md](./coco-wholebody/hrnet_coco-wholebody.md) | | CSPNeXt-m | 256x192 | 0.567 | 0.641 | [cspnext_udp_coco-wholebody.md](./coco-wholebody/cspnext_udp_coco-wholebody.md) | +| HRNet-w32 | 256x192 | 0.549 | 0.646 | [hrnet_ubody-coco-wholebody.md](./ubody2d/hrnet_ubody-coco-wholebody.md) | | ResNet-152 | 256x192 | 0.548 | 0.661 | [resnet_coco-wholebody.md](./coco-wholebody/resnet_coco-wholebody.md) | | HRNet-w32 | 256x192 | 0.536 | 0.636 | [hrnet_coco-wholebody.md](./coco-wholebody/hrnet_coco-wholebody.md) | | ResNet-101 | 256x192 | 0.531 | 0.645 | [resnet_coco-wholebody.md](./coco-wholebody/resnet_coco-wholebody.md) | | S-ViPNAS-Res50+Dark | 256x192 | 0.528 | 0.632 | [vipnas_dark_coco-wholebody.md](./coco-wholebody/vipnas_dark_coco-wholebody.md) | | ResNet-50 | 256x192 | 0.521 | 0.633 | [resnet_coco-wholebody.md](./coco-wholebody/resnet_coco-wholebody.md) | | S-ViPNAS-Res50 | 256x192 | 0.495 | 0.607 | [vipnas_coco-wholebody.md](./coco-wholebody/vipnas_coco-wholebody.md) | + +### UBody2D Dataset + +Result on UBody val set, computed with gt keypoints. + +| Model | Input Size | Whole AP | Whole AR | Details and Download | +| :-------: | :--------: | :------: | :------: | :----------------------------------------------------------------------: | +| HRNet-w32 | 256x192 | 0.690 | 0.729 | [hrnet_ubody-coco-wholebody.md](./ubody2d/hrnet_ubody-coco-wholebody.md) | diff --git a/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/hrnet_coco-wholebody.yml b/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/hrnet_coco-wholebody.yml new file mode 100644 index 0000000000..d51126cab8 --- /dev/null +++ b/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/hrnet_coco-wholebody.yml @@ -0,0 +1,23 @@ +Models: +- Config: configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/td-hm_hrnet-w32_8xb64-210e_ubody-256x192.py + In Collection: HRNet + Metadata: + Architecture: &id001 + - HRNet + Training Data: UBody-COCO-WholeBody + Name: td-hm_hrnet-w32_8xb64-210e_ubody-256x192 + Results: + - Dataset: COCO-WholeBody + Metrics: + Body AP: 0.678 + Body AR: 0.755 + Face AP: 0.630 + Face AR: 0.708 + Foot AP: 0.543 + Foot AR: 0.661 + Hand AP: 0.467 + Hand AR: 0.566 + Whole AP: 0.536 + Whole AR: 0.636 + Task: Wholebody 2D Keypoint + Weights: https://download.openmmlab.com/mmpose/v1/wholebody_2d_keypoint/ubody/td-hm_hrnet-w32_8xb64-210e_ubody-coco-256x192-7c227391_20230807.pth diff --git a/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/hrnet_ubody-coco-wholebody.md b/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/hrnet_ubody-coco-wholebody.md index 1d05a323c4..bd62073847 100644 --- a/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/hrnet_ubody-coco-wholebody.md +++ b/configs/wholebody_2d_keypoint/topdown_heatmap/ubody2d/hrnet_ubody-coco-wholebody.md @@ -31,7 +31,7 @@ -Results on COCO-WholeBody v1.0 val with detector having human AP of 54.9 on COCO val2017 dataset +Results on COCO-WholeBody v1.0 val with detector having human AP of 56.4 on COCO val2017 dataset | Arch | Input Size | Body AP | Body AR | Foot AP | Foot AR | Face AP | Face AR | Hand AP | Hand AR | Whole AP | Whole AR | ckpt | log | | :-------------------------------------- | :--------: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :-----: | :------: | :------: | :--------------------------------------: | :-------------------------------------: | From 0ab9ac55fb8027718b5ba7e93441771b9be0b568 Mon Sep 17 00:00:00 2001 From: xiexinch Date: Mon, 7 Aug 2023 15:26:00 +0800 Subject: [PATCH 09/10] --other=update img --- docs/en/dataset_zoo/2d_wholebody_keypoint.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/dataset_zoo/2d_wholebody_keypoint.md b/docs/en/dataset_zoo/2d_wholebody_keypoint.md index b572ec69ce..1c1e9f75d1 100644 --- a/docs/en/dataset_zoo/2d_wholebody_keypoint.md +++ b/docs/en/dataset_zoo/2d_wholebody_keypoint.md @@ -151,7 +151,7 @@ Please also install the latest version of [Extended COCO API](https://github.com
- +
For [Ubody](https://github.com/IDEA-Research/OSX) dataset, videos and annotations can be downloaded from [OSX homepage](https://github.com/IDEA-Research/OSX). From 3bf88bc5f46c6ef1ec682b4f462d28e85a9f80d3 Mon Sep 17 00:00:00 2001 From: xiexinch Date: Mon, 7 Aug 2023 15:49:49 +0800 Subject: [PATCH 10/10] --other=add dataset tool doc --- docs/en/dataset_zoo/dataset_tools.md | 78 ++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/docs/en/dataset_zoo/dataset_tools.md b/docs/en/dataset_zoo/dataset_tools.md index 44a7c96b2b..94c2b9d012 100644 --- a/docs/en/dataset_zoo/dataset_tools.md +++ b/docs/en/dataset_zoo/dataset_tools.md @@ -396,3 +396,81 @@ For example, ```shell python tools/dataset_converters/labelstudio2coco.py config.xml project-1-at-2023-05-13-09-22-91b53efa.json output/result.json ``` + +## UBody2D + +
+UBody (CVPR'2023) + +```bibtex +@article{lin2023one, + title={One-Stage 3D Whole-Body Mesh Recovery with Component Aware Transformer}, + author={Lin, Jing and Zeng, Ailing and Wang, Haoqian and Zhang, Lei and Li, Yu}, + booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, + year={2023}, +} +``` + +
+ +For [Ubody](https://github.com/IDEA-Research/OSX) dataset, videos and annotations can be downloaded from [OSX homepage](https://github.com/IDEA-Research/OSX). + +Download and extract them under $MMPOSE/data, and make them look like this: + +```text +mmpose +├── mmpose +├── docs +├── tests +├── tools +├── configs +`── data + │── UBody + ├── annotations + │   ├── ConductMusic + │   ├── Entertainment + │   ├── Fitness + │   ├── Interview + │   ├── LiveVlog + │   ├── Magic_show + │   ├── Movie + │   ├── Olympic + │   ├── Online_class + │   ├── SignLanguage + │   ├── Singing + │   ├── Speech + │   ├── TVShow + │   ├── TalkShow + │   └── VideoConference + ├── splits + │   ├── inter_scene_test_list.npy + │   └── intra_scene_test_list.npy + ├── videos + │   ├── ConductMusic + │   ├── Entertainment + │   ├── Fitness + │   ├── Interview + │   ├── LiveVlog + │   ├── Magic_show + │   ├── Movie + │   ├── Olympic + │   ├── Online_class + │   ├── SignLanguage + │   ├── Singing + │   ├── Speech + │   ├── TVShow + │   ├── TalkShow + │   └── VideoConference +``` + +We provide a script to convert vidoes to images and split annotations to train/val sets. It can be used by running the following command: + +```shell +python tools/dataset_converters/ubody_kpts_to_coco.py --data-root ${UBODY_DATA_ROOT} +``` + +For example, + +```shell +python tools/dataset_converters/ubody_kpts_to_coco.py --data-root data/UBody +```