使用TensorFlow.js从网络摄像头进行实时AI情感检测

我们将使用视频标签来使用来自实时摄像机的图像，而不是使用图像或画布标签。

我们还需要“脾气暴躁”和“中性”按钮，才能将实况视频的图像添加到我们的训练数据中：

Grumpy
Neutral

让我们添加另一个标签，以在页面而不是控制台上输出我们的预测。

我们最终的HTML文件如下所示：

    
        
         
        
        
    
    
        

            Real-Time Grumpiness detection using Tensorflow.js
        
            

            
            
 
            How are you feeling?
            Grumpy
            Neutral
 
            

 
            
        
    

let knn;
let model;
 
const classes = ['Grumpy', 'Neutral'];
const video = document.getElementById('webcam');

我们还需要加载模型并创建KNN分类器的实例：

let knn;
let model;
 
const classes = ['Grumpy', 'Neutral'];
const video = document.getElementById('webcam');

我们现在可以设置网络摄像头以获取视频图片：

const setupWebcam = async () => {
    return new Promise((resolve, reject) => {
        const _navigator = navigator;
        navigator.getUserMedia = navigator.getUserMedia ||
        _navigator.webkitGetUserMedia || _navigator.mozGetUserMedia ||
        _navigator.msGetUserMedia;
        if (navigator.getUserMedia) {
            navigator.getUserMedia({video: true},
                stream => {
                    video.srcObject = stream;
                    video.addEventListener('loadeddata', () => resolve(), false);
                },
            error => reject());
        }
    });
}

const addExample = label => {
    // getting the intermediate activation of MobileNet 'conv_preds' and passing that to the KNN classifier.
    const feature = model.infer(video, 'conv_preds');

    // Pass the intermediate activation to the classifier
    knn.addExample(feature, label);
};

// add an example to the specified class on button click
document.getElementById('grumpy').addEventListener('click', () => addExample(0));
document.getElementById('neutral').addEventListener('click', () => addExample(1));

let knn;
let model;
 
knn = knnClassifier.create();
const classes = ['Grumpy', 'Neutral'];
const video = document.getElementById('webcam');
 
async function loadKnnClassifier() {
 
    console.log('Model is Loading..');
    model = await mobilenet.load();
    console.log('Model loaded successfully!');
 
    await setupWebcam();
 
    // Reading the image from the webcam and associate the image with a specific class 
    const addExample = label => {
        // getting the intermediate activation of MobileNet 'conv_preds' and passing that to the KNN classifier.
        const feature = model.infer(video, 'conv_preds');
 
        // Pass the intermediate activation to the classifier
        knn.addExample(feature, label);
    };
 
    // add an example to the specified class on button click
    document.getElementById('grumpy').addEventListener('click', () => addExample(0));
    document.getElementById('neutral').addEventListener('click', () => addExample(1));
 
    while(true) {
        if (knn.getNumClasses() > 0) {
            // Getting activation from mobilenet for the webcam video
            const feature = model.infer(video, 'conv_preds');
            // getting the top prediction from the classifier module
            const prediction = await knn.predictClass(feature);
 
            // printing the prediction with confidence score on the screen
            document.getElementById('prediction').innerText = `
                Predicted emotion: ${classes[prediction.classIndex]}\n
                Probability of prediction: ${prediction.confidences[prediction.classIndex].toFixed(2)}
                `;
        }
 
        // wait for the next animation frame 
        await tf.nextFrame();
    }
}
 
const setupWebcam = async () => {
    return new Promise((resolve, reject) => {
        const _navigator = navigator;
        navigator.getUserMedia = navigator.getUserMedia ||
        _navigator.webkitGetUserMedia || _navigator.mozGetUserMedia ||
        _navigator.msGetUserMedia;
        if (navigator.getUserMedia) {
            navigator.getUserMedia({video: true},
                stream => {
                    video.srcObject = stream;
                    video.addEventListener('loadeddata', () => resolve(), false);
                },
            error => reject());
        }
    });
}
 
loadKnnClassifier();