Extracting mAP for each class when evaluating.

[Jinksi]

❓ Questions and Help

Is there some way to extract a mAP metric for each individual class/label when evaluating?

I am using a COCO-like custom dataset.

Any help or advice is appreciated, thanks!

[chengyangfu]

@Jinksi
You just need to insert few lines here.
In COCO evaluation code, you could specify which category and images are evaluated.

    for catId in coco_gt.getCatIds():
        coco_eval = COCOeval(coco_gt, coco_dt, iou_type)
        coco_eval.params.catIds = [catId]
        coco_eval.evaluate()
        coco_eval.accumulate()
        coco_eval.summarize()

[Jinksi]

Thanks @chengyangfu !

I've made a few changes to coco_eval.py to suit my needs.
If anyone is interested, the changes I've made are:

https://github.com/Jinksi/maskrcnn-benchmark/blob/d6a669e4e94ca23f3c190c4b378002b57c9095ee/maskrcnn_benchmark/data/datasets/evaluation/coco/coco_eval.py#L60-L68

https://github.com/Jinksi/maskrcnn-benchmark/blob/d6a669e4e94ca23f3c190c4b378002b57c9095ee/maskrcnn_benchmark/data/datasets/evaluation/coco/coco_eval.py#L318-L339

https://github.com/Jinksi/maskrcnn-benchmark/blob/d6a669e4e94ca23f3c190c4b378002b57c9095ee/maskrcnn_benchmark/data/datasets/evaluation/coco/coco_eval.py#L377-L388

This adds eval metrics for each category to the final results

[austinmw]

HI @chengyangfu , I notice that this reruns evaluate() each time. If you have many images this can take several minutes per class. Is there any way to re-use the previous all-class results to speed this up?

For example I'd like to print:

<Overall metrics>
<Class 1 metrics>
<Class 2 metrics>
<Class 3 metrics>
...
<Class 50 metrics>

But currently each one takes about 12 minutes for my dataset.

---

Edit:
Ok, figured it out:

# Add more IoUs and larger detect counts
cocoEval.params.iouThrs = np.array([0.25, 0.3 , 0.5 , 0.55, 0.6 , 0.65, 0.7 , 0.75, 0.8 , 0.85, 0.9 , 0.95])
cocoEval.params.maxDets = [1, 10, 100, 200, 250, 300]

# Modify summarize to include category argument
def summarize_extended(categoryId=None):
    '''
    Compute and display summary metrics for evaluation results.
    Note this functin can *only* be applied on the default parameter setting
    '''        
    
    def _summarize(ap=1, iouThr=None, areaRng='all', maxDets=300, categoryId=None):
        p = cocoEval.params
        if categoryId:
            iStr = ' {:<18} {} @[ CategoryId={:>3d} | IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}'
        else:
            iStr = ' {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}'
        
        titleStr = 'Average Precision' if ap == 1 else 'Average Recall'
        typeStr = '(AP)' if ap==1 else '(AR)'
        iouStr = '{:0.2f}:{:0.2f}'.format(p.iouThrs[2], p.iouThrs[-1]) \
            if iouThr is None else '{:0.2f}'.format(iouThr)

        aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng]
        mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets]

        if ap == 1:
            # dimension of precision: [TxRxKxAxM]
            s = cocoEval.eval['precision']
            # IoU
            if iouThr is not None:
                t = np.where(iouThr == p.iouThrs)[0]
                s = s[t]
                s = s[:,:,:,aind,mind]
            else:
                s = s[2:,:,:,aind,mind]
            if categoryId is not None:
                category_index = [i for i, i_catId in enumerate(p.catIds) if i_catId == categoryId]
                s = s[:,:,category_index,:]
        else:
            # dimension of recall: [TxKxAxM]
            s = cocoEval.eval['recall']
            if iouThr is not None:
                t = np.where(iouThr == p.iouThrs)[0]
                s = s[t]
                s = s[:,:,aind,mind]
            else:
                s = s[2:,:,aind,mind]
            if categoryId is not None:
                category_index = [i for i, i_catId in enumerate(p.catIds) if i_catId == categoryId]
                s = s[:,category_index,:]

        if len(s[s>-1])==0:
            mean_s = -1
        else:
            mean_s = np.mean(s[s>-1])
        if categoryId:
            print(iStr.format(titleStr, typeStr, categoryId, iouStr, areaRng, maxDets, mean_s))
        else:
            print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s))
        return mean_s
    
    def _summarizeDets(categoryId=None):
        stats = np.zeros((27,))
        stats[0] = _summarize(1, maxDets=cocoEval.params.maxDets[-1], categoryId=categoryId)
        stats[1] = _summarize(1, iouThr=.25, maxDets=cocoEval.params.maxDets[-1], categoryId=categoryId)
        stats[2] = _summarize(1, iouThr=.30, maxDets=cocoEval.params.maxDets[-1], categoryId=categoryId)
        stats[3] = _summarize(1, iouThr=.5, maxDets=cocoEval.params.maxDets[-1], categoryId=categoryId)
        stats[4] = _summarize(1, iouThr=.75, maxDets=cocoEval.params.maxDets[-1], categoryId=categoryId)
        stats[5] = _summarize(1, areaRng='small', maxDets=cocoEval.params.maxDets[-1], categoryId=categoryId)
        stats[6] = _summarize(1, iouThr=.25, areaRng='small', maxDets=cocoEval.params.maxDets[-1], categoryId=categoryId)
        stats[8] = _summarize(1, iouThr=.30, areaRng='small', maxDets=cocoEval.params.maxDets[-1], categoryId=categoryId)
        stats[9] = _summarize(1, iouThr=.5, areaRng='small', maxDets=cocoEval.params.maxDets[-1], categoryId=categoryId)
        stats[10] = _summarize(1, areaRng='medium', maxDets=cocoEval.params.maxDets[-1], categoryId=categoryId)
        stats[11] = _summarize(1, iouThr=.25, areaRng='medium', maxDets=cocoEval.params.maxDets[-1], categoryId=categoryId)
        stats[12] = _summarize(1, iouThr=.30, areaRng='medium', maxDets=cocoEval.params.maxDets[-1], categoryId=categoryId)
        stats[13] = _summarize(1, iouThr=.5, areaRng='medium', maxDets=cocoEval.params.maxDets[-1], categoryId=categoryId)
        stats[14] = _summarize(1, areaRng='large', maxDets=cocoEval.params.maxDets[-1], categoryId=categoryId)
        stats[15] = _summarize(1, iouThr=.25, areaRng='large', maxDets=cocoEval.params.maxDets[-1], categoryId=categoryId)
        stats[16] = _summarize(1, iouThr=.30, areaRng='large', maxDets=cocoEval.params.maxDets[-1], categoryId=categoryId)
        stats[17] = _summarize(1, iouThr=.5, areaRng='large', maxDets=cocoEval.params.maxDets[-1], categoryId=categoryId)
        stats[18] = _summarize(0, maxDets=cocoEval.params.maxDets[0], categoryId=categoryId)
        stats[19] = _summarize(0, maxDets=cocoEval.params.maxDets[1], categoryId=categoryId)
        stats[20] = _summarize(0, maxDets=cocoEval.params.maxDets[2], categoryId=categoryId)
        stats[21] = _summarize(0, maxDets=cocoEval.params.maxDets[3], categoryId=categoryId)
        stats[22] = _summarize(0, maxDets=cocoEval.params.maxDets[4], categoryId=categoryId)
        stats[23] = _summarize(0, maxDets=cocoEval.params.maxDets[5], categoryId=categoryId)
        stats[24] = _summarize(0, areaRng='small', maxDets=cocoEval.params.maxDets[-1], categoryId=categoryId)
        stats[25] = _summarize(0, areaRng='medium', maxDets=cocoEval.params.maxDets[-1], categoryId=categoryId)
        stats[26] = _summarize(0, areaRng='large', maxDets=cocoEval.params.maxDets[-1], categoryId=categoryId)
        return stats

    def _summarizeKps():
        stats = np.zeros((10,))
        stats[0] = _summarize(1, maxDets=20)
        stats[1] = _summarize(1, maxDets=20, iouThr=.5)
        stats[2] = _summarize(1, maxDets=20, iouThr=.75)
        stats[3] = _summarize(1, maxDets=20, areaRng='medium')
        stats[4] = _summarize(1, maxDets=20, areaRng='large')
        stats[5] = _summarize(0, maxDets=20)
        stats[6] = _summarize(0, maxDets=20, iouThr=.5)
        stats[7] = _summarize(0, maxDets=20, iouThr=.75)
        stats[8] = _summarize(0, maxDets=20, areaRng='medium')
        stats[9] = _summarize(0, maxDets=20, areaRng='large')
        return stats
    if not cocoEval.eval:
        raise Exception('Please run accumulate() first')
    iouType = cocoEval.params.iouType
    if iouType == 'segm' or iouType == 'bbox':
        summarize = _summarizeDets
    elif iouType == 'keypoints':
        summarize = _summarizeKps
    cocoEval.stats = summarize(categoryId)

# Replace function and run for all, catid 1, catid 2
cocoEval.summarize = summarize_extended
cocoEval.summarize()
cocoEval.summarize(1)
cocoEval.summarize(2)