openpose模型在AI challenge人体骨骼关键点检测的表现
因为之前正好看了CMU在CVPR2017上的论文《Realtime Multi-Person 2D Pose Estimation using Part Affinity Fields》,而且他们提供了训练好的模型。所以就直接用CMU训练的模型在AI challenge的数据集上做了测试。最后没有使用AI challenge训练集训练的模型在AI challenge上的得分是0.1667,可以看作是一个baseline。
以下是预处理的说明以及加入预处理程序的源代码。openpose的源代码使用#openpose ##openpose标注出来了,剩下的就是AI challenge的预处理程序。
在Google Cloud 上使用1片NVIDIA Tesla K80 跑完AI challenge的测试集大约需要24小时,4秒左右处理一副图。
AI challenge测试要求的关键点顺序是:1右肩,2右肘,3右腕,4左肩,5左肘,6左腕,7右髋,8右膝,9右踝,10左髋,11左膝,12左踝,13头顶,14脖子
openpose源码中subset输出的关键点顺序是:1鼻子,2脖子,3右肩,4右肘,5右腕,6左肩,7左肘,8左腕,9右髋,10右膝,11右踝,12左髋,13左膝,14左踝,15左眼,16右眼,17左耳,18右耳,19 pt19
函数 subset2AIsubset, all_peaks2all_peaks_1d, listMultiKeypoints 负责把openpose的关键点转换成AI challenge 的关键点。
当然还得按照官网上的要求输出特定格式的JSON文件,如下所示:
[
{
"image_id": "a0f6bdc065a602b7b84a67fb8d14ce403d902e0d",
"keypoint_annotations": {
"human1": [261, 294, 1, 281, 328, 1, 0, 0, 0, 213, 295, 1, 208, 346, 1, 192, 335, 1, 245, 375, 1, 255, 432, 1, 244, 494, 1, 221, 379, 1, 219, 442, 1, 226, 491, 1, 226, 256, 1, 231, 284, 1],
"human2": [313, 301, 1, 305, 337, 1, 321, 345, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 359, 1, 320, 409, 1, 311, 454, 1, 0, 0, 0, 330, 409, 1, 324, 446, 1, 337, 284, 1, 327, 302, 1],
"human3": [373, 304, 1, 346, 286, 1, 332, 263, 1, 0, 0, 0, 0, 0, 0, 345, 313, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 386, 1, 361, 424, 1, 361, 475, 1, 365, 273, 1, 369, 297, 1],
...
}
}
...
]
#import numpy as np
import json
import os
#openpose
import keras
from keras.models import Sequential
from keras.models import Model
from keras.layers import Input, Dense, Activation
from keras.layers.convolutional import Conv2D
from keras.layers.pooling import MaxPooling2D
from keras.layers.normalization import BatchNormalization
from keras.layers.merge import Concatenate
from config_reader import config_reader
import scipy import cv2
import numpy as np
np.seterr(divide='ignore', invalid='ignore')
import util
import math
from numpy import ma
from scipy.ndimage.filters import gaussian_filter
##openpose
#openpose
def relu(x):
return Activation('relu')(x) def conv(x, nf, ks, name):
x1 = Conv2D(nf, (ks, ks), padding='same', name=name)(x)
return x1 def pooling(x, ks, st, name):
x = MaxPooling2D((ks, ks), strides=(st, st), name=name)(x)
return x def vgg_block(x): # Block 1
x = conv(x, 64, 3, "conv1_1")
x = relu(x)
x = conv(x, 64, 3, "conv1_2")
x = relu(x)
x = pooling(x, 2, 2, "pool1_1") # Block 2
x = conv(x, 128, 3, "conv2_1")
x = relu(x)
x = conv(x, 128, 3, "conv2_2")
x = relu(x)
x = pooling(x, 2, 2, "pool2_1") # Block 3
x = conv(x, 256, 3, "conv3_1")
x = relu(x)
x = conv(x, 256, 3, "conv3_2")
x = relu(x)
x = conv(x, 256, 3, "conv3_3")
x = relu(x)
x = conv(x, 256, 3, "conv3_4")
x = relu(x)
x = pooling(x, 2, 2, "pool3_1") # Block 4
x = conv(x, 512, 3, "conv4_1")
x = relu(x)
x = conv(x, 512, 3, "conv4_2")
x = relu(x) # Additional non vgg layers
x = conv(x, 256, 3, "conv4_3_CPM")
x = relu(x)
x = conv(x, 128, 3, "conv4_4_CPM")
x = relu(x) return x def stage1_block(x, num_p, branch): # Block 1
x = conv(x, 128, 3, "conv5_1_CPM_L%d" % branch)
x = relu(x)
x = conv(x, 128, 3, "conv5_2_CPM_L%d" % branch)
x = relu(x)
x = conv(x, 128, 3, "conv5_3_CPM_L%d" % branch)
x = relu(x)
x = conv(x, 512, 1, "conv5_4_CPM_L%d" % branch)
x = relu(x)
x = conv(x, num_p, 1, "conv5_5_CPM_L%d" % branch) return x def stageT_block(x, num_p, stage, branch): # Block 1
x = conv(x, 128, 7, "Mconv1_stage%d_L%d" % (stage, branch))
x = relu(x)
x = conv(x, 128, 7, "Mconv2_stage%d_L%d" % (stage, branch))
x = relu(x)
x = conv(x, 128, 7, "Mconv3_stage%d_L%d" % (stage, branch))
x = relu(x)
x = conv(x, 128, 7, "Mconv4_stage%d_L%d" % (stage, branch))
x = relu(x)
x = conv(x, 128, 7, "Mconv5_stage%d_L%d" % (stage, branch))
x = relu(x)
x = conv(x, 128, 1, "Mconv6_stage%d_L%d" % (stage, branch))
x = relu(x)
x = conv(x, num_p, 1, "Mconv7_stage%d_L%d" % (stage, branch)) return x
##openpose def subset2AIsubset(t, numPersons):
AIsubset=[]
for j in xrange(numPersons):
tempsubset=[]
for i in xrange(12):
#
#print(i+2)
tempsubset.append(t[j][i+2]) tempsubset.append(t[j][0])
tempsubset.append(t[j][1])
#print(AIsubset)
AIsubset.append(tempsubset)
return AIsubset def all_peaks2all_peaks_1d(all_peaks):
all_peaks_1d=[]
for item in all_peaks:
for item1 in item:
all_peaks_1d.append(item1)
return all_peaks_1d def listMultiKeypoints(all_peaks_1d, numPersons):
multi_keypoints=[]
for i in xrange(numPersons):
sp_keypoints=[]
for j in xrange(14):
if(AIsubset[i][j]== -1.):
sp_keypoints.append(0)
sp_keypoints.append(0)
sp_keypoints.append(0)
else:
sp_keypoints.append(all_peaks_1d[int(AIsubset[i][j])][0])
sp_keypoints.append(all_peaks_1d[int(AIsubset[i][j])][1])
sp_keypoints.append(1)
#print(sp_keypoints)
multi_keypoints.append(sp_keypoints)
return multi_keypoints def nPersons(t):
return len(t) def listHuman(nPersons):
list_human=[]
for i in xrange(numPersons):
list_human.append('human'+str(i+1))
return list_human #openpose
weights_path = "model/keras/model.h5" input_shape = (None,None,3) img_input = Input(shape=input_shape) stages = 6
np_branch1 = 38
np_branch2 = 19 # VGG
stage0_out = vgg_block(img_input) # stage 1
stage1_branch1_out = stage1_block(stage0_out, np_branch1, 1)
stage1_branch2_out = stage1_block(stage0_out, np_branch2, 2)
x = Concatenate()([stage1_branch1_out, stage1_branch2_out, stage0_out]) # stage t >= 2
for sn in range(2, stages + 1):
stageT_branch1_out = stageT_block(x, np_branch1, sn, 1)
stageT_branch2_out = stageT_block(x, np_branch2, sn, 2)
if (sn < stages):
x = Concatenate()([stageT_branch1_out, stageT_branch2_out, stage0_out]) model = Model(img_input, [stageT_branch1_out, stageT_branch2_out])
model.load_weights(weights_path)
##openpose #openpose
# find connection in the specified sequence, center 29 is in the position 15
limbSeq = [[2,3], [2,6], [3,4], [4,5], [6,7], [7,8], [2,9], [9,10], \
[10,11], [2,12], [12,13], [13,14], [2,1], [1,15], [15,17], \
[1,16], [16,18], [3,17], [6,18]]
# the middle joints heatmap correpondence
mapIdx = [[31,32], [39,40], [33,34], [35,36], [41,42], [43,44], [19,20], [21,22], \
[23,24], [25,26], [27,28], [29,30], [47,48], [49,50], [53,54], [51,52], \
[55,56], [37,38], [45,46]]
##openpose path = "./test0"
files = os.listdir(path)
list_image_names=[]
final_results=[]
num_processed_images=0.
total_images=30000.
for file in files:
num_processed_images+=1
print('file:',file)
print('number of image:',num_processed_images)
print('%.2f%%'%(num_processed_images/total_images*100))
list_image_names.append(str(file)[:-4])
#openpose
test_image = './test0/'+file
#test_image = 'sample_images/000a902c8674739c97f188157c63d709b45b7595.jpg'
oriImg = cv2.imread(test_image) param, model_params = config_reader()
multiplier = [x * model_params['boxsize'] / oriImg.shape[0] for x in param['scale_search']]
heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 19))
paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38)) for m in range(len(multiplier)):
scale = multiplier[m]
imageToTest = cv2.resize(oriImg, (0,0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
imageToTest_padded, pad = util.padRightDownCorner(imageToTest, model_params['stride'], model_params['padValue']) input_img = np.transpose(np.float32(imageToTest_padded[:,:,:,np.newaxis]), (3,0,1,2))/256 - 0.5; # required shape (1, width, height, channels)
print("Input shape: " + str(input_img.shape)) output_blobs = model.predict(input_img)
print("Output shape (heatmap): " + str(output_blobs[1].shape)) # extract outputs, resize, and remove padding
heatmap = np.squeeze(output_blobs[1]) # output 1 is heatmaps
heatmap = cv2.resize(heatmap, (0,0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC)
heatmap = heatmap[:imageToTest_padded.shape[0]-pad[2], :imageToTest_padded.shape[1]-pad[3], :]
heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC) paf = np.squeeze(output_blobs[0]) # output 0 is PAFs
paf = cv2.resize(paf, (0,0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC)
paf = paf[:imageToTest_padded.shape[0]-pad[2], :imageToTest_padded.shape[1]-pad[3], :]
paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC) heatmap_avg = heatmap_avg + heatmap / len(multiplier)
paf_avg = paf_avg + paf / len(multiplier)
##openpose
#openpose
U = paf_avg[:,:,16] * -1
V = paf_avg[:,:,17]
X, Y = np.meshgrid(np.arange(U.shape[1]), np.arange(U.shape[0]))
M = np.zeros(U.shape, dtype='bool')
M[U**2 + V**2 < 0.5 * 0.5] = True
U = ma.masked_array(U, mask=M)
V = ma.masked_array(V, mask=M) all_peaks = []
peak_counter = 0 for part in range(19-1):
map_ori = heatmap_avg[:,:,part]
map = gaussian_filter(map_ori, sigma=3) map_left = np.zeros(map.shape)
map_left[1:,:] = map[:-1,:]
map_right = np.zeros(map.shape)
map_right[:-1,:] = map[1:,:]
map_up = np.zeros(map.shape)
map_up[:,1:] = map[:,:-1]
map_down = np.zeros(map.shape)
map_down[:,:-1] = map[:,1:] peaks_binary = np.logical_and.reduce((map>=map_left, map>=map_right, map>=map_up, map>=map_down, map > param['thre1']))
peaks = list(zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse
peaks_with_score = [x + (map_ori[x[1],x[0]],) for x in peaks]
id = range(peak_counter, peak_counter + len(peaks))
peaks_with_score_and_id = [peaks_with_score[i] + (id[i],) for i in range(len(id))] all_peaks.append(peaks_with_score_and_id)
peak_counter += len(peaks)
##openpose
#openpose
connection_all = []
special_k = []
mid_num = 10 for k in range(len(mapIdx)):
score_mid = paf_avg[:,:,[x-19 for x in mapIdx[k]]]
candA = all_peaks[limbSeq[k][0]-1]
candB = all_peaks[limbSeq[k][1]-1]
nA = len(candA)
nB = len(candB)
indexA, indexB = limbSeq[k]
if(nA != 0 and nB != 0):
connection_candidate = []
for i in range(nA):
for j in range(nB):
vec = np.subtract(candB[j][:2], candA[i][:2])
#
#print('vec0:',vec[0],'vec1:',vec[1])
# #
norm = math.sqrt(vec[0]*vec[0] + vec[1]*vec[1]+0.1)
vec = np.divide(vec, norm) startend = list(zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \
np.linspace(candA[i][1], candB[j][1], num=mid_num))) vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
for I in range(len(startend))])
vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
for I in range(len(startend))]) score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1])
#
#print('norm',norm)
# #
score_with_dist_prior = sum(score_midpts)/len(score_midpts) + min(0.5*oriImg.shape[0]/norm-1, 0) criterion1 = len(np.nonzero(score_midpts > param['thre2'])[0]) > 0.8 * len(score_midpts)
criterion2 = score_with_dist_prior > 0
if criterion1 and criterion2:
connection_candidate.append([i, j, score_with_dist_prior, score_with_dist_prior+candA[i][2]+candB[j][2]]) connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True)
connection = np.zeros((0,5))
for c in range(len(connection_candidate)):
i,j,s = connection_candidate[c][0:3]
if(i not in connection[:,3] and j not in connection[:,4]):
connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]])
if(len(connection) >= min(nA, nB)):
break connection_all.append(connection)
else:
special_k.append(k)
connection_all.append([])
##openpose
#openpose
# last number in each row is the total parts number of that person
# the second last number in each row is the score of the overall configuration
subset = -1 * np.ones((0, 20))
candidate = np.array([item for sublist in all_peaks for item in sublist]) for k in range(len(mapIdx)):
if k not in special_k:
partAs = connection_all[k][:,0]
partBs = connection_all[k][:,1]
indexA, indexB = np.array(limbSeq[k]) - 1 for i in range(len(connection_all[k])): #= 1:size(temp,1)
found = 0
subset_idx = [-1, -1]
for j in range(len(subset)): #1:size(subset,1):
if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]:
subset_idx[found] = j
found += 1 if found == 1:
j = subset_idx[0]
if(subset[j][indexB] != partBs[i]):
subset[j][indexB] = partBs[i]
subset[j][-1] += 1
subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
elif found == 2: # if found 2 and disjoint, merge them
j1, j2 = subset_idx
print ("found = 2")
membership = ((subset[j1]>=0).astype(int) + (subset[j2]>=0).astype(int))[:-2]
if len(np.nonzero(membership == 2)[0]) == 0: #merge
subset[j1][:-2] += (subset[j2][:-2] + 1)
subset[j1][-2:] += subset[j2][-2:]
subset[j1][-2] += connection_all[k][i][2]
subset = np.delete(subset, j2, 0)
else: # as like found == 1
subset[j1][indexB] = partBs[i]
subset[j1][-1] += 1
subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2] # if find no partA in the subset, create a new subset
elif not found and k < 17:
row = -1 * np.ones(20)
row[indexA] = partAs[i]
row[indexB] = partBs[i]
row[-1] = 2
row[-2] = sum(candidate[connection_all[k][i,:2].astype(int), 2]) + connection_all[k][i][2]
subset = np.vstack([subset, row])
# delete some rows of subset which has few parts occur
deleteIdx = [];
for i in range(len(subset)):
if subset[i][-1] < 4 or subset[i][-2]/subset[i][-1] < 0.4:
deleteIdx.append(i)
subset = np.delete(subset, deleteIdx, axis=0)
##openpose
numPersons= nPersons(subset)
#print(subset2AIsubset(subset, numPersons))
AIsubset = subset2AIsubset(subset,numPersons)
#print(all_peaks[i][numPersons][3]==[int(AIsubset[0][0])])
#all_peaks->all_peaks_1d
all_peaks_1d=all_peaks2all_peaks_1d(all_peaks)
#print('numPersons:',numPersons)
#print('multi_keypoints:',listMultiKeypoints(all_peaks_1d, numPersons))
keys=['image_id','keypoint_annotations']
values=[]
image_id=str(file)[:-4] keypoint_annotations = dict(zip(listHuman(numPersons), listMultiKeypoints(all_peaks_1d, numPersons)))
values.append(image_id)
values.append(keypoint_annotations) d = dict(zip(keys, values)) final_results.append(d)
print(final_results)
with open('data.json', 'w') as f:
json.dump(final_results, f) print(list_image_names)
[
{
"image_id": "a0f6bdc065a602b7b84a67fb8d14ce403d902e0d",
"keypoint_annotations": {
"human1": [261, 294, 1, 281, 328, 1, 0, 0, 0, 213, 295, 1, 208, 346, 1, 192, 335, 1, 245, 375, 1, 255, 432, 1, 244, 494, 1, 221, 379, 1, 219, 442, 1, 226, 491, 1, 226, 256, 1, 231, 284, 1],
"human2": [313, 301, 1, 305, 337, 1, 321, 345, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 359, 1, 320, 409, 1, 311, 454, 1, 0, 0, 0, 330, 409, 1, 324, 446, 1, 337, 284, 1, 327, 302, 1],
"human3": [373, 304, 1, 346, 286, 1, 332, 263, 1, 0, 0, 0, 0, 0, 0, 345, 313, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 386, 1, 361, 424, 1, 361, 475, 1, 365, 273, 1, 369, 297, 1],
...
}
}
...
]
openpose模型在AI challenge人体骨骼关键点检测的表现的更多相关文章
- dlib人脸关键点检测的模型分析与压缩
本文系原创,转载请注明出处~ 小喵的博客:https://www.miaoerduo.com 博客原文(排版更精美):https://www.miaoerduo.com/c/dlib人脸关键点检测的模 ...
- 用华为HMS ML kit人体骨骼识别技术,Android快速实现人体姿势动作抓拍
你有没有过这种体验,拍照时对着镜头,脑子一片空白.表情僵硬.手和脚无处安放,最后拍出来的照片很是奇怪.拍照软件中的固定姿势抓拍功能可以帮助你:选择一个你想要的姿势模板,当你摆出同款姿势时,软件会进 ...
- 『关键点检测』CPN:Cascaded Pyramid Network for Multi-Person Pose Estimation
论文连接 网络简介 face++2017年coco keypoint benchmark 数据集冠军的文章,发表于CVPR201 1 提出了一种金字塔型的串接模型,即CPN(cascaded pyr ...
- 『计算机视觉』Mask-RCNN_从服装关键点检测看KeyPoints分支
下图Github地址:Mask_RCNN Mask_RCNN_KeyPoints『计算机视觉』Mask-RCNN_论文学习『计算机视觉』Mask-RCNN_项目文档翻译『计算机视觉』Mas ...
- 用keras实现人脸关键点检测(2)
上一个代码只能实现小数据的读取与训练,在大数据训练的情况下.会造内存紧张,于是我根据keras的官方文档,对上一个代码进行了改进. 用keras实现人脸关键点检测 数据集:https://pan.ba ...
- keras实现简单CNN人脸关键点检测
用keras实现人脸关键点检测 改良版:http://www.cnblogs.com/ansang/p/8583122.html 第一步:准备好需要的库 tensorflow 1.4.0 h5py ...
- OpenCV实战:人脸关键点检测(FaceMark)
Summary:利用OpenCV中的LBF算法进行人脸关键点检测(Facial Landmark Detection) Author: Amusi Date: 2018-03-20 ...
- PCL—低层次视觉—关键点检测(iss&Trajkovic)
关键点检测往往需要和特征提取联合在一起,关键点检测的一个重要性质就是旋转不变性,也就是说,物体旋转后还能够检测出对应的关键点.不过说实话我觉的这个要求对机器人视觉来说是比较鸡肋的.因为机器人采集到的三 ...
- PCL—关键点检测(iss&Trajkovic)低层次点云处理
博客转载自:http://www.cnblogs.com/ironstark/p/5069311.html 关键点检测往往需要和特征提取联合在一起,关键点检测的一个重要性质就是旋转不变性,也就是说,物 ...
随机推荐
- 委托、事件、Observer观察者模式的使用解析二
一.设计模式-Observer观察者模式 Observer设计模式是为了定义对象间的一种一对多的依赖关系,以便于当一个对象的状态改变时,其他依赖于它的对象会被自动告知并更新.Observer模式是一种 ...
- java远程备份mysql数据库关键问题(限windows环境,亲测解决)
其它环境同理也可解决. 条件:为了使用mysql命令,本机要安装mysql ,我本机安装的是mysql 5.5. 错误1:使用命令 mysqldump -h192.168.1.50 -u root - ...
- 基于RTKLIB构建高并发通信测试工具
1. RTKLIB基础动态库生成 RTKLIB是全球导航卫星系统GNSS(global navigation satellite system)的标准&精密定位开源程序包,由日本东京海洋大学的 ...
- C# winform小票打印
(1)自定义纸张设置 控制面板->打印机和传真->右键->服务器属性->创建新的格式 (2)自定义纸张使用 this.printDocument1.DefaultPageSet ...
- 数组去重方法(ES6)
let arrayBefore = [1,3,3,2,1,5,2,1]; //去重之前的数组 Array.prototype.dedupe = function (){ //去重函数 返回去重后的数组 ...
- YYHS-NOIP模拟赛-gcd
题解 这道题题解里说用莫比乌斯反演做(我这个蒟蒻怎么会做呢) 但是不会,所以我们另想方法,这里我们用容斥来做 我们先把500000以内的所有质数筛出来 每次读入编号的时候,先把编号对应的这个数分解质因 ...
- log4j2配置文件解读
log4j2可以按照开发人员预先的设定,在指定的位置和情况下打印log语句,并且可以酌情关闭某些log语句,如开发阶段debug类型的语句等.并且,可以使用layout来定义输出语句的格式. 使用前需 ...
- RT-thread 利用Scons 工具编译提示python编码错误解决办法
错误信息: scons: Reading SConscript files ...UnicodeDecodeError: 'ascii' codec can't decode byte 0xbd in ...
- ubuntu下codeblocks安装与中文化
什么是Code::Blocks Code::Blocks是一个免费.开源.跨平台的集成开发环境,使用C++开发,并且使用wxWidgets做为GUI库.Code::Blocks使用了插件架构,其功能可 ...
- LeetCode 381. Insert Delete GetRandom O(1) - Duplicates allowed (插入删除和获得随机数 常数时间 允许重复项)
Design a data structure that supports all following operations in average O(1) time. Note: Duplicate ...