RoboMasters 2016 SkyEye Vision Framework

RoboMasters 2016 SkyEye Vision Framework

SkyEye系统是基于RoboMasters 2016赛季的基地战车和步兵战车自主识别的需求而搭建的。采用目标检测领域当时较为突出优秀的 YOLO v1 算法:You Only Look Once: Unified, Real-Time Object Detection进行有针对性的训练和检验。

针对RoboMasters 2016 步兵车红蓝方训练集算法训练

请确保高性能主机安装好Opencv和cuda,最好推荐安装成功cudnn以此来加速训练,安装详情请见博主另一篇博文:Ubuntu16.04 配置 CUDA8.0 + cudnn5.1 + Opencv2.4.13

1. 设定检测目标的标签 BlueCar / RedCar

编辑 SkyEye 系统文件夹中../data/labels/make_labels.py为以下内容:

import os
l = ["BlueCar", "RedCar"]
for word in l:
os.system("convert -fill black -background white -bordercolor white -border 4 -font futura-normal -pointsize 18 label:\"%s\" \"%s.png\""%(word, word))

终端进入../data/labels文件夹中输入以下命令:

./make_labels.py

在同一层次目录下生成相应的标签图片
 BlueCar.png /RedCar.png

2. 使用 自行制作的RoboMasters Vision 软件对训练集图片中的目标根据提示要求进行框选


3. 构建 Robo 训练集文件结构

4. 代码生成train.txt其中包含详细的训练集图片的完整路径

/home/ubuntu/Desktop/Robo/1.jpg
/home/ubuntu/Desktop/Robo/10.jpg
/home/ubuntu/Desktop/Robo/100.jpg
/home/ubuntu/Desktop/Robo/101.jpg
...... .......

5. 修改../src/yolo.c程序文件中相关代码和参数

(1)全局变量声明:

/* Change class number here */
#define CLASSNUM 2

/* Change class names here */
char *voc_names[] = {"BlueCar", "RedCar"};

(2)train_yolo函数更新:

/* Change training folders here */
char *train_images = "/home/ubuntu/Desktop/Robo/train.txt";

/* Change output weight folders here */
char *backup_directory = "/home/ubuntu/SkyEye/backup/";

6. 修改../cfg/yolo-tiny.cfg配置文件中相关代码和参数

(1)配置文件前端,其中更新最大迭代次数为40000

[net]
max_batches = 40000 

(2)配置文件后端,其中classes更新为2,然后更新output = side*side*(num*5+classes)

[connected]
output= 588
activation=linear

[detection]
classes=2
coords=4
rescore=1
side=7
num=2
softmax=0
sqrt=1
jitter=.2

7. 修改../src/yolo_kernels.cu程序cuda文件中相关代码和参数

(1)全局变量声明:

/* Change class number here */
#define CLS_NUM 2

(2)detect_in_thread函数更新:

draw_detections(det, l.side*l.side*l.n, demo_thresh, boxes, probs, voc_names, voc_labels, CLS_NUM);

8. 下载预训练模型 extraction.conv.weights (darknet主页更新YOLO v2之后该预训练模型改名为darknet.conv.weights)

9. 修改Makefile文件为以下内容:(其中cuda-6.5更改为主机对应的cuda版本文件夹名)

GPU=1
CUDNN=0
OPENCV=1
DEBUG=0

ARCH= --gpu-architecture=compute_20 --gpu-code=compute_20

VPATH=./src/
EXEC=darknet
OBJDIR=./obj/

CC=gcc
NVCC=nvcc
OPTS=-Ofast
LDFLAGS= -lm -pthread -lstdc++
COMMON=
CFLAGS=-Wall -Wfatal-errors

ifeq ($(DEBUG), 1)
OPTS=-O0 -g
endif

CFLAGS+=$(OPTS)

ifeq ($(OPENCV), 1)
COMMON+= -DOPENCV
CFLAGS+= -DOPENCV
LDFLAGS+= `pkg-config --libs opencv`
COMMON+= `pkg-config --cflags opencv`
endif

ifeq ($(GPU), 1)
COMMON+= -DGPU -I/usr/local/cuda-6.5/include/
CFLAGS+= -DGPU
LDFLAGS+= -L/usr/local/cuda-6.5/lib -lcuda -lcudart -lcublas -lcurand
endif

ifeq ($(CUDNN), 1)
COMMON+= -DCUDNN -I/usr/local/cuda-6.5/include/
CFLAGS+= -DCUDNN
LDFLAGS+= -L/usr/local/cuda-6.5/lib -lcudnn
endif
...... ......

如果NVCC有效[Nvidia Jetson TK1 无法正常使用cudnn进行显卡运算加速],则置

CUDNN = 1
NVCC = /usr/local/cuda-6.5/bin/nvcc

10. 重新编译SkyEye系统并进行训练

sudo make clean
sudo make -j4
./darknet yolo train cfg/yolo-tiny.cfg extraction.conv.weights

针对RoboMasters 2016 步兵车红蓝方检测并整合步兵发射云台系统

1. 保存好训练结果权重文件yolo-tiny_final.weights,此为训练好的权重模型。若想观察检测效果直接运行以下指令即可。无需进行后续操作

./darknet yolo demo cfg/yolo-tiny.cfg yolo-tiny_final.weights

2. 修改../src/yolo.c程序文件中相关代码和参数:

(1)demo_yolo函数声明:

void demo_yolo(char *cfgfile, char *weightfile, float thresh, int target, float segthresh, float size, int ErrorMinX, int ErrorMinY, int ErrorMaxX, int ErrorMaxY, int yuntaiX, int yuntaiY, int cam_index, char* filename);
#ifndef GPU
void demo_yolo(char *cfgfile, char *weightfile, float thresh, int target, float segthresh, float size, int ErrorMinX, int ErrorMinY, int ErrorMaxX, int ErrorMaxY, int yuntaiX, int yuntaiY, int cam_index, char* filename)
{
    fprintf(stderr, "Darknet must be compiled with CUDA for YOLO demo.\n");
}
#endif

(2)run_yolo函数更新:

void run_yolo(int argc, char **argv)
{
int i;
for(i = 0; i < CLASSNUM; ++i){
char buff[256];
sprintf(buff, "data/labels/%s.png", voc_names[i]);
voc_labels[i] = load_image_color(buff, 0, 0);
}

float thresh = find_float_arg(argc, argv, "-thresh", .25);
int target = find_int_arg(argc, argv, "-target", 0);
float segthresh = find_float_arg(argc, argv, "-segthresh", .35);
float size = find_float_arg(argc, argv, "-size", 100.0);
int errorminX = find_int_arg(argc, argv, "-ErrMinX", 0);
int errorminY = find_int_arg(argc, argv, "-ErrMinY", 0);
int errormaxX = find_int_arg(argc, argv, "-ErrMaxX", 0);
int errormaxY = find_int_arg(argc, argv, "-ErrMaxY", 0);
int yuntaiX = find_int_arg(argc, argv, "-yuntaiX", 0);
int yuntaiY = find_int_arg(argc, argv, "-yuntaiY", 0);
int cam_index = find_int_arg(argc, argv, "-c", 0);
if(argc < 4){
fprintf(stderr, "usage: %s %s [train/test/valid] [cfg] [weights (optional)]\n", argv[0], argv[1]);
return;
}

char *cfg = argv[3];
char *weights = (argc > 4) ? argv[4] : 0;
char *filename = (argc > 5) ? argv[5]: 0;
if(0==strcmp(argv[2], "test")) test_yolo(cfg, weights, filename, thresh);
else if(0==strcmp(argv[2], "train")) train_yolo(cfg, weights);
else if(0==strcmp(argv[2], "valid")) validate_yolo(cfg, weights);
else if(0==strcmp(argv[2], "recall")) validate_yolo_recall(cfg, weights);
else if(0==strcmp(argv[2], "skyeye")) demo_yolo(cfg, weights, thresh, target, segthresh, size, errorminX, errorminY, errormaxX, errormaxY, yuntaiX, yuntaiY, cam_index, "NULL");
//else if(0==strcmp(argv[2], "demo_vid")) demo_yolo(cfg, weights, thresh, -1, filename);
}

3. 修改../src/yolo_kernels.cu程序cuda文件中相关代码和参数:

(1)全局变量声明:

extern "C" {
#include "network.h"
#include "detection_layer.h"
#include "cost_layer.h"
#include "utils.h"
#include "parser.h"
#include "box.h"
#include "image.h"
#include <sys/time.h>
}

Comm RM;
int offsetTagX;
int offsetTagY;

static float **probs;
static box *boxes;
static network net;
static image in ;
static image in_s ;
static image det ;
static image det_s;
static image disp ;
static cv::VideoCapture cap;
static cv::VideoWriter cap_out;
static float fps = 0;
static float demo_thresh = 0;
static float demo_segthresh = 0;
static int demo_target = 0;
static int demo_errorminX = 0;
static int demo_errorminY = 0;
static int demo_errormaxX = 0;
static int demo_errormaxY = 0;
static int demo_yuntaiX = 0;
static int demo_yuntaiY = 0;
static float demo_size = 0;
static int w, h, depth, c, step= 0;
static int MODE = -1;

(2)加入串口初始化函数:详情请见:RoboMasters 与步兵控制整合的串口发送函数

(3)采用跳帧的方式提高实时处理速度,并确保跳帧过程中发射云台保持原位不动。fetch_in_thread函数更新:

void *fetch_in_thread(void *ptr)
{
cv::Mat frame_m; 
 przTurn(RM,offsetTagX,offsetTagY);
 cap >> frame_m;
 przTurn(RM,offsetTagX,offsetTagY);
 cap >> frame_m;
 przTurn(RM,offsetTagX,offsetTagY);
 cap >> frame_m;
 przTurn(RM,offsetTagX,offsetTagY);

IplImage frame = frame_m;

if(step == 0)
{
w = frame.width;
h = frame.height;
c = frame.nChannels;
depth= frame.depth;
step = frame.widthStep;
}

in = ipl_to_image(&frame);
rgbgr_image(in);
in_s = resize_image(in, net.w, net.h);
return 0;
}

(4)更新detect_in_thread函数中的fps显示和draw_detections函数声明:

printf("\nTruth FPS:%.0f  Real-Time FPS:%.0f\n",fps , fps*3);

if (demo_target == 0)
 printf("Target: BlueCar\n");
 else if (demo_target == 1)
 printf("Target: RedCar\n");
 printf(" Objects:\n\n");
 draw_detections(RM, det, l.side*l.side*l.n, demo_thresh, demo_target, demo_segthresh, demo_size, demo_errorminX, demo_errorminY, demo_errormaxX, demo_errormaxY, demo_yuntaiX, demo_yuntaiY, boxes, probs, voc_names, voc_labels, CLS_NUM);

(5)demo_yolo函数更新并加入串口初始化函数:


extern "C" void demo_yolo(char *cfgfile, char *weightfile, float thresh, int target, float segthresh, float size, int ErrorMinX, int ErrorMinY, int ErrorMaxX, int ErrorMaxY, int yuntaiX, int yuntaiY, int cam_index, char *videofile)
{
demo_thresh = thresh;
demo_target = target;
 demo_segthresh = segthresh;
 demo_size = size;
 demo_errorminX = ErrorMinX;
 demo_errorminY = ErrorMinY;
 demo_errormaxX = ErrorMaxX;
 demo_errormaxY = ErrorMaxY;
 demo_yuntaiX = yuntaiX;
 demo_yuntaiY = yuntaiY;
 offsetTagX = 0; 
 offsetTagY = 0;
 PortInit();
 //printf("OpenCCOMPort: %d",RM.fd);
 printf("*************************SkyEye*************************\n");
...... ......

4. 修改../src/image.c程序文件中相关代码和参数:

(1)全局变量声明:

int DetTag = 0;//Had Found The Objects

int Loc1offsetX = 0 , Loc1offsetY = 0;
int Loc2offsetX = 0 , Loc2offsetY = 0;

box preBox;

extern Comm RM;
extern int offsetTagX;
extern int offsetTagY;

(2)加入串口发送函数:详情请见:RoboMasters 与步兵控制整合的串口发送函数
(3)根据情况修改draw_detections函数并建立与步兵云台和发射系统通信:

void draw_detections(Comm com,image im, int num, float thresh, int target, float segthresh, float size, int ErrorMinX, int ErrorMinY, int ErrorMaxX, int ErrorMaxY,int yuntaiX, int yuntaiY, box *boxes, float **probs, char **names, image *labels, int classes)
{
    int i;
    int offsetX = 0;
    int offsetY = 0;

    int Tag = 0 ;
    int Car[5];
    float Prop[5];
    float prob = 0;

    for(i = 0; i < num; ++i) {//Find All The Target int class = max_index(probs[i], classes); prob = probs[i][class]; if((prob > thresh) && (class == target))
        {
            Car[Tag] = i;
            Prop[Tag] = prob;
     	    Tag = Tag + 1;
            
	}
    }

	box b;

  //Find The Suitable One to Draw The Detections.  
    if(Tag == 0)
    {//No Target 
      DetTag = 0;
      przTurn(com,0,0);
      offsetTagX = 0;
      offsetTagY = 0;	
    }else{
        DetTag = DetTag + 1;
        

       	
       	if(Tag == 1)
        {//Target One 
	   int index = Car[0];
	   b = boxes[index];
           prob = probs[index][target];
	   offsetX = (b.x-0.5)*250;
	   offsetY = (b.y-0.5)*250;
	   //przTurn(com,offsetX,offsetY);
    	}else{//Target More Than One
           if(Tag == 2)
           {
              if(Prop[0] >= Prop[1])
              {//Two Target [If With The Same Prop.Choose the First One] 
                  int index = Car[0];
	          b = boxes[index];
		  prob = probs[index][target];
	          offsetX = (b.x-0.5)*250;
	          offsetY = (b.y-0.5)*250;
	          //przTurn(com,offsetX,offsetY);
              }else{
                  int index = Car[1];
	          b = boxes[index];
                  prob = probs[index][target];
	          offsetX = (b.x-0.5)*250;
	          offsetY = (b.y-0.5)*250;
	          //przTurn(com,offsetX,offsetY);
              }
           }else{//Target More Than Two 
              int tempint = 0;
              int tempfloat = 0;
              int proppair = 1;
              int PropPairIndex[5];
              int k;
              
              for(k = 0; k < Tag; k++) { if(Prop[k] > tempfloat){//Find The MAX Prop
                      tempfloat = Prop[k];
                      tempint = k;
		      
                  }else if(Prop[k] == tempfloat){//If The Props Are Same.Put Them Into PropPair
                      proppair = proppair + 1;     
                      PropPairIndex[proppair - 1] = k;
                  }
	      } 
              PropPairIndex[0] = tempint;

              if(proppair == 1)
              {//No Props Are Same
              	  int index = Car[tempint];
		  b = boxes[index];
		  prob = probs[index][target];
	          offsetX = (b.x-0.5)*250;
	          offsetY = (b.y-0.5)*250;
	          //przTurn(com,offsetX,offsetY);
              }else{//PropPair Is Existed.
                  float S = 0;
                  int Maxsizeindex = 0;
                  int l;
                  for(l = 0; l < proppair; l++){ int index = PropPairIndex[l]; b = boxes[index]; float Starget = b.w*b.h*im.w*im.h; if((Starget > size) && (Starget > S))
                      {//Find The MAX Size In PropPair
			  S = Starget;
                          Maxsizeindex = index;
		      } 
                  }
                  b = boxes[Maxsizeindex];
                  prob = probs[Maxsizeindex][target];
	          offsetX = (b.x-0.5)*250;
	          offsetY = (b.y-0.5)*250;
	          //przTurn(com,offsetX,offsetY);
              }
            }
         }

   //Find The Suitable One.
		float Tsize = b.w*b.h*im.w*im.h;
	     	if (Tsize <= 21000.0)
	     	{
			offsetY = offsetY - 4;
		}
		else if ( Tsize <= 26000.0 ) 
		{
			offsetY = offsetY - 3;
		}
		else if ( Tsize <= 31000.0 )
		{
			offsetY = offsetY - 2;
		}
		else if ( ( Tsize <= 51000.0) && ( Tsize >= 36000.0 ) )
		{
			offsetY = offsetY + 1;
		}
		else if ( Tsize <= 85000.0 )
		{
			offsetY = offsetY + 2;
		}
		else if ( Tsize <= 115000.0 )
		{
			offsetY = offsetY + 3;
		}
		else if ( Tsize <= 155000.0 ) { offsetY = offsetY + 4; } przTurn(com,offsetX,offsetY); offsetTagX = offsetX; offsetTagY = offsetY; int width = pow(prob, 1./2.)*10+1; printf("%s : %.3f\n", names[target], prob); printf(" Total Target %s : %d\n", names[target], Tag); printf(" Loc( b.x %.2f , b.y %.2f ) \n" , b.x ,b.y ); if(Tag > 1){
                  int m;
                  for(m = 1; m <= Tag; m++){
                        box bb = boxes[Car[m-1]];
                  	printf("           Target %s %d: [Prop] %.3f [Size] %.3f\n", names[target], m, Prop[m-1], bb.w*bb.h*im.w*im.h );  
                  } 
	     }else{
                  printf("           Target %s : [Prop] %.3f [Size] %.3f\n", names[target], prob, b.w*b.h*im.w*im.h );  
             }
	     int offset = target*17 % classes;
	     float red = get_color(0,offset,classes);
	     float green = get_color(1,offset,classes);
	     float blue = get_color(2,offset,classes);
	     float rgb[3];
	     rgb[0] = red;
	     rgb[1] = green;
	     rgb[2] = blue;

	     int left  = (b.x-b.w/2.)*im.w;
	     int right = (b.x+b.w/2.)*im.w;
	     int top   = (b.y-b.h/2.)*im.h;
	     int bot   = (b.y+b.h/2.)*im.h;


	     if(left < 0) left = 0; if(right > im.w-1) right = im.w-1;
	     if(top < 0) top = 0; if(bot > im.h-1) bot = im.h-1;

	     draw_box_width(im, left, top, right, bot, width, red, green, blue);
	     if (labels) draw_label(im, top + width, left, labels[target], rgb);
             /*if (DetTag == 1)
             {
		przTurn(com,offsetX,offsetY);
             	Loc1offsetX = offsetX;
             	Loc1offsetY = offsetY;
                
             }else if (DetTag == 2)
  	     { 
		Loc2offsetX = offsetX;
                Loc2offsetY = offsetY;

               printf("\n*********Enter Prediction Of Target!\n");
	       printf("SETTING\n");
               printf("         ErrorMin Offset( %d , %d )\n",ErrorMinX,ErrorMinY);
               printf("         ErrorMax Offset( %d , %d )\n",ErrorMaxX,ErrorMaxY);
	       printf("DETAILS\n");
               printf("           Loc1 Offset   ( %d , %d )\n", Loc1offsetX,Loc1offsetY);
               printf("           Loc2 Offset   ( %d , %d )\n", Loc2offsetX,Loc2offsetY);

   	       int deltaX = Loc2offsetX - Loc1offsetX;
               int deltaY = Loc2offsetY - Loc1offsetY;
              
    	       if ((deltaX <= ErrorMinX) && (deltaX >= -ErrorMinX)) 
               {  
                  deltaX = 0;
               }
               if ((deltaY <= ErrorMinY) && (deltaY >= -ErrorMinY)) 
               {  
                  deltaY = 0;
               }

               if ((deltaX >= ErrorMaxX) && (deltaX <= -ErrorMaxX)) { deltaX = 0; } if ((deltaY >= ErrorMaxY) && (deltaY <= -ErrorMaxY)) 
               {  
                  deltaY = 0;
               }

	       printf("          Delta Offset  ( %d , %d )\n",deltaX,deltaY);

               float Q1param = 4 * 0.125;
	       float Q2param = 2 * 0.125;
               float Mparam = 1.7;

               offsetX = (int) ( (2 * Q1param - 1)* Mparam - Q1param + 1 ) * Loc1offsetX + ( Mparam - 1) * Loc2offsetX;
	       offsetY = (int) ( (2 * Q2param - 1)* Mparam - Q2param + 1 ) * Loc1offsetY + ( Mparam - 1) * Loc2offsetY;

               if (deltaX = 0) offsetX = Loc2offsetX;
               if (deltaY = 0) offsetY = Loc2offsetY;
               przTurn(com,offsetX,offsetY);
 	       printf("         Newest Offset  ( %d , %d )\n",offsetX,offsetY);
	       printf("         BoxOriginal Loc( %.2f , %.2f , %.2f, %.2f )\n",b.x,b.y,b.w,b.h);
               int preleft  = ((offsetX/250.)+0.5-b.w/2.)*im.w;
	       int preright = ((offsetX/250.)+0.5+b.w/2.)*im.w;
	       int pretop   = ((offsetY/250.)+0.5-b.h/2.)*im.h;
	       int prebot   = ((offsetY/250.)+0.5+b.h/2.)*im.h;

               if(preleft < 0) preleft = 0; if(preright > im.w-1) preright = im.w-1;
	       if(pretop < 0) pretop = 0; if(prebot > im.h-1) prebot = im.h-1;
               
               float prered,pregreen,preblue;
               if (target == 0)
               {
                   int preoffset = 1*17 % classes;
	           prered = get_color(0,preoffset,classes);
	           pregreen = get_color(1,preoffset,classes);
	           preblue = get_color(2,preoffset,classes);
               }else if (target == 1)
               {   
                   int preoffset = 0*17 % classes;
	           prered = get_color(0,preoffset,classes);
	           pregreen = get_color(1,preoffset,classes);
	           preblue = get_color(2,preoffset,classes);
               }
               draw_box_width(im, preleft, pretop, preright, prebot, width, prered, pregreen, preblue);
		//if (labels) draw_label(im, pretop + width, preleft, labels[target], rgb);
              //}else if (DetTag == 8){
                DetTag = 0;
	      }
             //DetTag = DetTag + 1;
               //DetTag = 1;
             */
             
     }

发表评论