How to visualize the att_map

[bbgua85776540]

How to visualize the att_map like you showed in the prediction result.

[yw0nam]

you can visualize attention map using these codes.

def normalize_img(value, vmax=None, vmin=None):
    vmin = value.min() if vmin is None else vmin
    vmax = value.max() if vmax is None else vmax
    if not (vmax - vmin) == 0:
        value = (value - vmin) / (vmax - vmin)  # vmin..vmax

    return value

model = avobject_model().cuda()


score, att_map, vid_sync, aud_sync, label = model.predict(data[batch])

batch = 0
idx = 0

S_av = att_map[idx, label, :, :, :].cpu()
aggregate_S_av = S_av.mean(0)
a = aggregate_S_av.unsqueeze(0)
a = a.unsqueeze(0)
interpolate_s_av = F.interpolate(a, 
                  size=(270, 480), mode='area')

interpolate_s_av = interpolate_s_av.squeeze()

original = data[batch]['video'][idx, :, 0, :, :].permute(1, 2, 0).int()

cam = normalize_img(-interpolate_s_av)
heatmap = cv2.applyColorMap(np.uint8(255 * cam), cv2.COLORMAP_JET)
heatmap = np.float32(heatmap) / 255
heatmap = np.uint8(255 * heatmap)
dst = cv2.addWeighted(original.numpy(), 0.5, heatmap, 0.5, 0, dtype = cv2.CV_32F)

fig = plt.figure(figsize=(16, 12))
plt.subplot(1,3,1)
plt.title('original')
plt.imshow(original)

plt.subplot(1,3,2)
plt.title('Attention Map')
plt.imshow(heatmap)


plt.subplot(1,3,3)
plt.title('Attention Map with original')
plt.imshow(dst.astype(np.uint8))