diff --git a/configs/_base_/models/point_rcnn.py b/configs/_base_/models/point_rcnn.py
index 7a9667e73..02a1414f7 100644
--- a/configs/_base_/models/point_rcnn.py
+++ b/configs/_base_/models/point_rcnn.py
@@ -91,7 +91,8 @@
                     pos_iou_thr=0.55,
                     neg_iou_thr=0.55,
                     min_pos_iou=0.55,
-                    ignore_iof_thr=-1),
+                    ignore_iof_thr=-1,
+                    match_low_quality=False),
                 dict(  # for Pedestrian
                     type='MaxIoUAssigner',
                     iou_calculator=dict(
@@ -99,7 +100,8 @@
                     pos_iou_thr=0.55,
                     neg_iou_thr=0.55,
                     min_pos_iou=0.55,
-                    ignore_iof_thr=-1),
+                    ignore_iof_thr=-1,
+                    match_low_quality=False),
                 dict(  # for Cyclist
                     type='MaxIoUAssigner',
                     iou_calculator=dict(
@@ -107,7 +109,8 @@
                     pos_iou_thr=0.55,
                     neg_iou_thr=0.55,
                     min_pos_iou=0.55,
-                    ignore_iof_thr=-1)
+                    ignore_iof_thr=-1,
+                    match_low_quality=False)
             ],
             sampler=dict(
                 type='IoUNegPiecewiseSampler',
diff --git a/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py b/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py
index c0de845f6..cbd8483ca 100644
--- a/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py
+++ b/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py
@@ -60,9 +60,7 @@ def _sample_neg(self, assign_result, num_expected, **kwargs):
         if neg_inds.numel() != 0:
             neg_inds = neg_inds.squeeze(1)
         if len(neg_inds) <= 0:
-            raise NotImplementedError(
-                'Not support sampling the negative samples when the length '
-                'of negative samples is 0')
+            return neg_inds.squeeze(1)
         else:
             neg_inds_choice = neg_inds.new_zeros([0])
             extend_num = 0
diff --git a/mmdet3d/models/dense_heads/point_rpn_head.py b/mmdet3d/models/dense_heads/point_rpn_head.py
index 50cc9d552..48ed1324e 100644
--- a/mmdet3d/models/dense_heads/point_rpn_head.py
+++ b/mmdet3d/models/dense_heads/point_rpn_head.py
@@ -272,7 +272,8 @@ def get_bboxes(self,
             bbox3d = self.bbox_coder.decode(bbox_preds[b], points[b, ..., :3],
                                             object_class[b])
             bbox_selected, score_selected, labels, cls_preds_selected = \
-                self.class_agnostic_nms(obj_scores[b], sem_scores[b], bbox3d)
+                self.class_agnostic_nms(obj_scores[b], sem_scores[b], bbox3d,
+                                        points[b, ..., :3], input_metas[b])
             bbox = input_metas[b]['box_type_3d'](
                 bbox_selected.clone(),
                 box_dim=bbox_selected.shape[-1],
@@ -280,7 +281,8 @@ def get_bboxes(self,
             results.append((bbox, score_selected, labels, cls_preds_selected))
         return results
 
-    def class_agnostic_nms(self, obj_scores, sem_scores, bbox):
+    def class_agnostic_nms(self, obj_scores, sem_scores, bbox, points,
+                           input_meta):
         """Class agnostic nms.
 
         Args:
@@ -298,6 +300,29 @@ def class_agnostic_nms(self, obj_scores, sem_scores, bbox):
         else:
             nms_func = nms_normal_gpu
 
+        num_bbox = bbox.shape[0]
+        bbox = input_meta['box_type_3d'](
+            bbox.clone(),
+            box_dim=bbox.shape[-1],
+            with_yaw=True,
+            origin=(0.5, 0.5, 0.5))
+
+        if isinstance(bbox, LiDARInstance3DBoxes):
+            box_idx = bbox.points_in_boxes(points)
+            box_indices = box_idx.new_zeros([num_bbox + 1])
+            box_idx[box_idx == -1] = num_bbox
+            box_indices.scatter_add_(0, box_idx.long(),
+                                     box_idx.new_ones(box_idx.shape))
+            box_indices = box_indices[:-1]
+            nonempty_box_mask = box_indices >= 0
+        elif isinstance(bbox, DepthInstance3DBoxes):
+            box_indices = bbox.points_in_boxes(points)
+            nonempty_box_mask = box_indices.T.sum(1) >= 0
+        else:
+            raise NotImplementedError('Unsupported bbox type!')
+
+        bbox = bbox.tensor[nonempty_box_mask]
+
         if self.test_cfg.score_thr is not None:
             score_thr = self.test_cfg.score_thr
             keep = (obj_scores >= score_thr)