Отслеживание движений головы в Linux

Один из принципов отслеживания головы (трекинг) состоит в том, что на головной убор или наушники крепится маркер (светящийся или отражающий). Видеокамера перед игроком отслеживает положение маркера и управляет положением курсора мыши.

Была найдена веб-камера с форматом пикселей YUV и разрешением 640 на 480. В качестве маркера - светодиодный фонарик на козырек кепки. Базовый пример для захвата изображения с устройства V4L2 - из документации к API V4L2. Пример был упрощен для компактности (убраны все методы работы с камерой кроме mmap), совмещен с кодом предыдущего поста об управлении мышью, а функция обработки изображения process_image переписана.

unsigned long sx = 0, sy = 0, sa = 0;
unsigned char v = 0;
 
for (int y = 0, pos = 0; y < 480; y++) {
    for (int x = 0; x < 640; x++, pos += 2) {
        // Y component of YUYV
        v = ((unsigned char *)p)[pos];
        if (v < threshold) continue;
        v -= (threshold - 1);
        sx += v * x;
        sy += v * y;
        sa += v;
    }
}
// empty screen
if (sa == 0) sa = 1;
 
double dx = (double)sx / (double)sa;
double dy = (double)sy / (double)sa;
 
//printf("A=%ld, x=%f, y=%f\n", sa, dx, dy);
 
// fill window 
xp[bp] = dx;
yp[bp] = dy;
// window size
bp += 1;
if (bp == window) bp = 0;
 
// sum of window
double sumx = 0, sumy = 0;
for (int j = 0; j < window; j++) {
    sumx += xp[j];
    sumy += yp[j];
}
 
// send to mouse
int lx = -(sumx - prev_x);
int ly = +(sumy - prev_y);
if (lx != 0 || ly != 0) {
    emit(fdu, EV_REL, REL_X, lx);
    emit(fdu, EV_REL, REL_Y, ly);
    emit(fdu, EV_SYN, SYN_REPORT, 0);
}
 
// save last
prev_x = sumx;
prev_y = sumy;

Каждый кадр захваченного изображения обрабатывается следующим образом (функция process_image):

В целом концепт работает. Положение кусора мыши управляется обычным светодиодным фонариком с точностью до пиксела. При работе приложения с игрой необходима адаптация для управления игровым процессом с трекингом, без подготовки вызывает тошноту.

Предварительные итоги особенностей работы трекера:

/*
 *  V4L2 video capture example
 *
 *  This program can be used and distributed without restrictions.
 *
 *      This program is provided with the V4L2 API
 * see http://linuxtv.org/docs.php for more information
 */
 
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
 
#include <getopt.h>             /* getopt_long() */
 
#include <fcntl.h>              /* low-level i/o */
#include <unistd.h>
#include <errno.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/mman.h>
#include <sys/ioctl.h>
 
#include <linux/videodev2.h>
 
#include <linux/uinput.h>
 
#define CLEAR(x) memset(&(x), 0, sizeof(x))
 
#ifndef V4L2_PIX_FMT_H264
#define V4L2_PIX_FMT_H264     v4l2_fourcc('H', '2', '6', '4') /* H264 with start codes */
#endif
 
 
void emit(int fd, int type, int code, int val) {
   struct input_event ie;
 
   ie.type = type;
   ie.code = code;
   ie.value = val;
   /* timestamp values below are ignored */
   ie.time.tv_sec = 0;
   ie.time.tv_usec = 0;
 
   write(fd, &ie, sizeof(ie));
}
 
 
struct buffer {
        void   *start;
        size_t  length;
};
 
static char            *dev_name;
static int              fd = -1;
struct buffer          *buffers;
static unsigned int     n_buffers;
static int              out_buf;
static int              force_format;
static int              frame_count = 200;
static int              frame_number = 0;
 
static int fdu = 0;
 
static void errno_exit(const char *s)
{
        fprintf(stderr, "%s error %d, %s\n", s, errno, strerror(errno));
        exit(EXIT_FAILURE);
}
 
static int xioctl(int fh, int request, void *arg) {
        int r;
        do {
                r = ioctl(fh, request, arg);
        } while (-1 == r && EINTR == errno);
        return r;
}
 
 
static int threshold = 240;
static int window = 2;
 
static unsigned int bp = 0;
static double xp[64], yp[64];
static double prev_x = 0, prev_y = 0;
 
static void process_image(const void *p, int size) {
    frame_number++;
 
    unsigned long sx = 0, sy = 0, sa = 0;
    unsigned char v = 0;
 
    for (int y = 0, pos = 0; y < 480; y++) {
        for (int x = 0; x < 640; x++, pos += 2) {
            // Y component of YUYV
            v = ((unsigned char *)p)[pos];
            if (v < threshold) continue;
            v -= (threshold - 1);
            sx += v * x;
            sy += v * y;
            sa += v;
        }
    }
    // empty screen
    if (sa == 0) sa = 1;
 
    double dx = (double)sx / (double)sa;
    double dy = (double)sy / (double)sa;
 
    //printf("A=%ld, x=%f, y=%f\n", sa, dx, dy);
 
    // fill window 
    xp[bp] = dx;
    yp[bp] = dy;
    // window size
    bp += 1;
    if (bp == window) bp = 0;
 
    // sum of window
    double sumx = 0, sumy = 0;
    for (int j = 0; j < window; j++) {
        sumx += xp[j];
        sumy += yp[j];
    }
 
    // send to mouse
    int lx = -(sumx - prev_x);
    int ly = +(sumy - prev_y);
    if (lx != 0 || ly != 0) {
        emit(fdu, EV_REL, REL_X, lx);
        emit(fdu, EV_REL, REL_Y, ly);
        emit(fdu, EV_SYN, SYN_REPORT, 0);
    }
 
    // save last
    prev_x = sumx;
    prev_y = sumy;
}
 
static int read_frame(void)
{
        struct v4l2_buffer buf;
        unsigned int i;
 
        CLEAR(buf);
 
        buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
        buf.memory = V4L2_MEMORY_MMAP;
 
        if (-1 == xioctl(fd, VIDIOC_DQBUF, &buf)) {
                switch (errno) {
                case EAGAIN:
                        return 0;
 
                case EIO:
                        /* Could ignore EIO, see spec. */
 
                        /* fall through */
 
                default:
                        errno_exit("VIDIOC_DQBUF");
                }
        }
 
        assert(buf.index < n_buffers);
 
        process_image(buffers[buf.index].start, buf.bytesused);
 
        if (-1 == xioctl(fd, VIDIOC_QBUF, &buf))
                errno_exit("VIDIOC_QBUF");
 
        return 1;
}
 
static void mainloop(void)
{
        unsigned int count;
 
        count = frame_count;
 
        while (count-- > 0) {
                for (;;) {
                        fd_set fds;
                        struct timeval tv;
                        int r;
 
                        FD_ZERO(&fds);
                        FD_SET(fd, &fds);
 
                        /* Timeout. */
                        tv.tv_sec = 2;
                        tv.tv_usec = 0;
 
                        r = select(fd + 1, &fds, NULL, NULL, &tv);
 
                        if (-1 == r) {
                                if (EINTR == errno)
                                        continue;
                                errno_exit("select");
                        }
 
                        if (0 == r) {
                                fprintf(stderr, "select timeout\n");
                                exit(EXIT_FAILURE);
                        }
 
                        if (read_frame())
                                break;
                        /* EAGAIN - continue select loop. */
                }
        }
}
 
static void stop_capturing(void)
{
        enum v4l2_buf_type type;
 
        type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
        if (-1 == xioctl(fd, VIDIOC_STREAMOFF, &type))
                errno_exit("VIDIOC_STREAMOFF");
}
 
static void start_capturing(void)
{
        unsigned int i;
        enum v4l2_buf_type type;
 
        for (i = 0; i < n_buffers; ++i) {
                struct v4l2_buffer buf;
 
                CLEAR(buf);
                buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
                buf.memory = V4L2_MEMORY_MMAP;
                buf.index = i;
 
                if (-1 == xioctl(fd, VIDIOC_QBUF, &buf))
                        errno_exit("VIDIOC_QBUF");
        }
        type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
        if (-1 == xioctl(fd, VIDIOC_STREAMON, &type))
                errno_exit("VIDIOC_STREAMON");
}
 
static void uninit_device(void) {
        unsigned int i;
 
        for (i = 0; i < n_buffers; ++i)
                if (-1 == munmap(buffers[i].start, buffers[i].length))
                        errno_exit("munmap");
 
        free(buffers);
}
 
static void init_mmap(void)
{
        struct v4l2_requestbuffers req;
 
        CLEAR(req);
 
        req.count = 4;
        req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
        req.memory = V4L2_MEMORY_MMAP;
 
        if (-1 == xioctl(fd, VIDIOC_REQBUFS, &req)) {
                if (EINVAL == errno) {
                        fprintf(stderr, "%s does not support "
                                 "memory mapping\n", dev_name);
                        exit(EXIT_FAILURE);
                } else {
                        errno_exit("VIDIOC_REQBUFS");
                }
        }
 
        if (req.count < 2) {
                fprintf(stderr, "Insufficient buffer memory on %s\n",
                         dev_name);
                exit(EXIT_FAILURE);
        }
 
        buffers = calloc(req.count, sizeof(*buffers));
 
        if (!buffers) {
                fprintf(stderr, "Out of memory\n");
                exit(EXIT_FAILURE);
        }
 
        for (n_buffers = 0; n_buffers < req.count; ++n_buffers) {
                struct v4l2_buffer buf;
 
                CLEAR(buf);
 
                buf.type        = V4L2_BUF_TYPE_VIDEO_CAPTURE;
                buf.memory      = V4L2_MEMORY_MMAP;
                buf.index       = n_buffers;
 
                if (-1 == xioctl(fd, VIDIOC_QUERYBUF, &buf))
                        errno_exit("VIDIOC_QUERYBUF");
 
                buffers[n_buffers].length = buf.length;
                buffers[n_buffers].start =
                        mmap(NULL /* start anywhere */,
                              buf.length,
                              PROT_READ | PROT_WRITE /* required */,
                              MAP_SHARED /* recommended */,
                              fd, buf.m.offset);
 
                if (MAP_FAILED == buffers[n_buffers].start)
                        errno_exit("mmap");
        }
}
 
static void init_device(void)
{
        struct v4l2_capability cap;
        struct v4l2_cropcap cropcap;
        struct v4l2_crop crop;
        struct v4l2_format fmt;
        unsigned int min;
 
        if (-1 == xioctl(fd, VIDIOC_QUERYCAP, &cap)) {
                if (EINVAL == errno) {
                        fprintf(stderr, "%s is no V4L2 device\n",
                                 dev_name);
                        exit(EXIT_FAILURE);
                } else {
                        errno_exit("VIDIOC_QUERYCAP");
                }
        }
 
        if (!(cap.capabilities & V4L2_CAP_VIDEO_CAPTURE)) {
                fprintf(stderr, "%s is no video capture device\n",
                         dev_name);
                exit(EXIT_FAILURE);
        }
 
        if (!(cap.capabilities & V4L2_CAP_STREAMING)) {
                fprintf(stderr, "%s does not support streaming i/o\n",
                         dev_name);
                exit(EXIT_FAILURE);
        }
 
 
        /* Select video input, video standard and tune here. */
 
 
        CLEAR(cropcap);
 
        cropcap.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
 
        if (0 == xioctl(fd, VIDIOC_CROPCAP, &cropcap)) {
                crop.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
                crop.c = cropcap.defrect; /* reset to default */
 
                if (-1 == xioctl(fd, VIDIOC_S_CROP, &crop)) {
                        switch (errno) {
                        case EINVAL:
                                /* Cropping not supported. */
                                break;
                        default:
                                /* Errors ignored. */
                                break;
                        }
                }
        } else {
                /* Errors ignored. */
        }
 
 
        CLEAR(fmt);
 
        fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
        if (force_format) {
	fprintf(stderr, "Set H264\r\n");
                fmt.fmt.pix.width       = 640; //replace
                fmt.fmt.pix.height      = 480; //replace
                fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_H264; //replace
                fmt.fmt.pix.field       = V4L2_FIELD_ANY;
 
                if (-1 == xioctl(fd, VIDIOC_S_FMT, &fmt))
                        errno_exit("VIDIOC_S_FMT");
 
                /* Note VIDIOC_S_FMT may change width and height. */
        } else {
                /* Preserve original settings as set by v4l2-ctl for example */
                if (-1 == xioctl(fd, VIDIOC_G_FMT, &fmt))
                        errno_exit("VIDIOC_G_FMT");
        }
 
        /* Buggy driver paranoia. */
        min = fmt.fmt.pix.width * 2;
        if (fmt.fmt.pix.bytesperline < min)
                fmt.fmt.pix.bytesperline = min;
        min = fmt.fmt.pix.bytesperline * fmt.fmt.pix.height;
        if (fmt.fmt.pix.sizeimage < min)
                fmt.fmt.pix.sizeimage = min;
 
        init_mmap();
}
 
static void close_device(void)
{
        if (-1 == close(fd))
                errno_exit("close");
 
        fd = -1;
}
 
static void open_device(void)
{
        struct stat st;
 
        if (-1 == stat(dev_name, &st)) {
                fprintf(stderr, "Cannot identify '%s': %d, %s\n",
                         dev_name, errno, strerror(errno));
                exit(EXIT_FAILURE);
        }
 
        if (!S_ISCHR(st.st_mode)) {
                fprintf(stderr, "%s is no device\n", dev_name);
                exit(EXIT_FAILURE);
        }
 
        fd = open(dev_name, O_RDWR /* required */ | O_NONBLOCK, 0);
 
        if (-1 == fd) {
                fprintf(stderr, "Cannot open '%s': %d, %s\n",
                         dev_name, errno, strerror(errno));
                exit(EXIT_FAILURE);
        }
}
 
static void usage(FILE *fp, int argc, char **argv)
{
        fprintf(fp,
                 "Usage: %s [options]\n\n"
                 "Version 1.3\n"
                 "Options:\n"
                 "-d | --device name   Video device name [%s]\n"
                 "-h | --help          Print this message\n"
                 "-o | --output        Outputs stream to stdout\n"
                 "-f | --format        Force format to 640x480 YUYV\n"
                 "-c | --count         Number of frames to grab [%i]\n"
                 "",
                 argv[0], dev_name, frame_count);
}
 
static const char short_options[] = "d:hofc:";
 
static const struct option
long_options[] = {
        { "device", required_argument, NULL, 'd' },
        { "help",   no_argument,       NULL, 'h' },
        { "output", no_argument,       NULL, 'o' },
        { "format", no_argument,       NULL, 'f' },
        { "count",  required_argument, NULL, 'c' },
        { 0, 0, 0, 0 }
};
 
int main(int argc, char **argv)
{
        dev_name = "/dev/video0";
 
        for (;;) {
                int idx;
                int c;
 
                c = getopt_long(argc, argv,
                                short_options, long_options, &idx);
 
                if (-1 == c)
                        break;
 
                switch (c) {
                case 0: /* getopt_long() flag */
                        break;
 
                case 'd':
                        dev_name = optarg;
                        break;
 
                case 'h':
                        usage(stdout, argc, argv);
                        exit(EXIT_SUCCESS);
 
                case 'o':
                        out_buf++;
                        break;
 
                case 'f':
                        force_format++;
                        break;
 
                case 'c':
                        errno = 0;
                        frame_count = strtol(optarg, NULL, 0);
                        if (errno)
                                errno_exit(optarg);
                        break;
 
                default:
                        usage(stderr, argc, argv);
                        exit(EXIT_FAILURE);
                }
        }
 
        open_device();
        init_device();
        start_capturing();
 
        // start mouse control
        struct uinput_setup usetup;
        fdu = open("/dev/uinput", O_WRONLY | O_NONBLOCK);
 
        /* enable mouse button left and relative events */
        ioctl(fdu, UI_SET_EVBIT, EV_KEY);
        ioctl(fdu, UI_SET_KEYBIT, BTN_LEFT);
 
        ioctl(fdu, UI_SET_EVBIT, EV_REL);
        ioctl(fdu, UI_SET_RELBIT, REL_X);
        ioctl(fdu, UI_SET_RELBIT, REL_Y);
 
        memset(&usetup, 0, sizeof(usetup));
        usetup.id.bustype = BUS_USB;
        usetup.id.vendor = 0x1234; /* sample vendor */
        usetup.id.product = 0x5678; /* sample product */
        strcpy(usetup.name, "Example device");
 
        ioctl(fdu, UI_DEV_SETUP, &usetup);
        ioctl(fdu, UI_DEV_CREATE);
        sleep(1);
 
        // loop capturing
        mainloop();
 
        // stop mouse control
        sleep(1);
        ioctl(fdu, UI_DEV_DESTROY);
        close(fdu);
 
        // stop capturing
        stop_capturing();
        uninit_device();
        close_device();
        fprintf(stderr, "\n");
        return 0;
}
 

2018-10-24