python - 使用 WebRTC 和 Python 服务器进行人脸情绪分析

Question

我有一个项目，我想在其中执行实时面部情绪识别。

一个客户打开一个网页，他的面部图像被一个摄像头(内置笔记本电脑摄像头)捕获，然后这个视频被绘制在 Canvas (HTML 元素)上，然后转换成 BLOB 并通过 Python 发送Websocket 到 Python 服务器后端。在那里 - 我需要使用另一个 Python 脚本(将图像作为输入)执行情感分析 - 所以我需要将这个 BLOB 转换回图像，但我不知道如何正确地进行，我一个接一个地收到错误，因为我是 Python 的新手，所以我不知道如何解决这个任务。

到目前为止，我设法打开了 websocket，实时发送了这个 BLOB，但随后发生了一些神奇的事情，一切都崩溃了——可能我将函数放入循环的方式以及我转换图像的方式也有问题数据。

HTML 片段:

<div class="booth">
            <video id="video" width="320" height="240"></video>
            <canvas width="320" id="canvas" height="240" style="display: inline;"></canvas>
        </div>
    <script src="canvas.js"></script>

Canvas .js:

(function () {
    var video = document.getElementsByTagName("video")[0];
    var canvas = document.getElementsByTagName("canvas");
    var ctx = canvas[0].getContext('2d');

    navigator.getMedia = navigator.getUserMedia ||
        navigator.webkitGetUserMedia ||
        navigator.mozGetUserMedia ||
        navigator.msGetUserMedia;

    navigator.getMedia({
        video: true,
        audio: false,
    }, function (stream) {
        console.log("I'm in live function")
        console.log(stream);
        video.srcObject = stream;
        video.play();
    }, function (error) {
        console.log("Error in live" + error)
        error.code
    });
    function dataURItoBlob(dataURI) {
        // convert base64/URLEncoded data component to raw binary data held in a string
        var byteString;
        if (dataURI.split(',')[0].indexOf('base64') >= 0)
            byteString = atob(dataURI.split(',')[1]);
        else
            byteString = unescape(dataURI.split(',')[1]);

        // separate out the mime component
        var mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0];

        // write the bytes of the string to a typed array
        var ia = new Uint8Array(byteString.length);
        for (var i = 0; i < byteString.length; i++) {
            ia[i] = byteString.charCodeAt(i);
        }

        return new Blob([ia], { type: mimeString });
    }
    var ws = new WebSocket(" ws://127.0.0.1:5678/");
        ws.onopen = function () {
              console.log("Openened connection to websocket");
    }
    timer = setInterval(
        function () {
            ctx.drawImage(video, 0, 0, 320, 240);
            var data = canvas[0].toDataURL('image/jpeg', 1.0);
            newblob = dataURItoBlob(data);
            ws.send(newblob);
        }, 100);


})();

web_socket_server.py :

async def time(websocket, path):
    detection_model_path = r'C:\Users\karol\face_recognition\haarcascade_frontalface_default.xml'

    emotion_model_path = r'C:\Users\karol\face_recognition\models_mini_XCEPTION.88-0.65.hdf5'

    face_detection = cv2.CascadeClassifier(detection_model_path)

    emotion_classifier = load_model(emotion_model_path, compile=False)

    EMOTIONS = ["angry", "disgust", "scared", "happy", "sad", "surprised",

                "neutral"]
    # starting video streaming
    while True:
        message = await websocket.recv()
        print(f"We got message from the client!")
        print (message)
        #images = message.decode('base64')
        #message = base64.encodebytes(message)
        #print(message)
        face_rec.emotion_detection(message, face_detection, emotion_classifier, EMOTIONS)

start_server = websockets.serve(time, "127.0.0.1", 5678)

asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()

人脸记录.py:

def decode_base64(data, altchars=b'+/'):
    data = re.sub(rb'[^a-zA-Z0-9%s]+' % altchars, b'', data)  # normalize
    missing_padding = len(data) % 4
    if missing_padding:
        data += b'='* (4 - missing_padding)
    return base64.b64decode(data, altchars)

def stringToRGB(base64_string):
    imgdata = decode_base64(base64_string)
    image = Image.open(io.BytesIO(imgdata)) //HERE I GET THE ERROR "cannot identify image file %r(filename if filename else fp))" 
    return cv2.cvtColor(np.array(image), cv2.COLOR_BGR2RGB)

def emotion_detection(bytes, face_detection, emotion_classifier, EMOTIONS):

    while True:

        image = stringToRGB(bytes)
        frame = imutils.resize(image, width=400) 
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

        faces = face_detection.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=5, minSize=(30, 30),
                                                flags=cv2.CASCADE_SCALE_IMAGE)

        canvas = np.zeros((250, 300, 3), dtype="uint8")

        frameClone = frame.copy()

        if len(faces) > 0:
            faces = sorted(faces, reverse=True,

                           key=lambda x: (x[2] - x[0]) * (x[3] - x[1]))[0]

            (fX, fY, fW, fH) = faces

            # Extract the ROI of the face from the grayscale image, resize it to a fixed 48x48 pixels, and then prepare

            # the ROI for classification via the CNN

            roi = gray[fY:fY + fH, fX:fX + fW]

            roi = cv2.resize(roi, (48, 48))

            roi = roi.astype("float") / 255.0

            roi = img_to_array(roi)

            roi = np.expand_dims(roi, axis=0)

            preds = emotion_classifier.predict(roi)[0]

            emotion_probability = np.max(preds)

            label = EMOTIONS[preds.argmax()]

        for (i, (emotion, prob)) in enumerate(zip(EMOTIONS, preds)):
            # construct the label text

            text = "{}: {:.2f}%".format(emotion, prob * 100)

            w = int(prob * 300)

            cv2.rectangle(canvas, (7, (i * 35) + 5),

                          (w, (i * 35) + 35), (0, 0, 255), -1)

            cv2.putText(canvas, text, (10, (i * 35) + 23),

                        cv2.FONT_HERSHEY_SIMPLEX, 0.45,

                        (255, 255, 255), 2)

            cv2.putText(frameClone, label, (fX, fY - 10),

                        cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 0, 255), 2)

            cv2.rectangle(frameClone, (fX, fY), (fX + fW, fY + fH),

                          (0, 0, 255), 2)

        cv2.imshow('your_face', frameClone)

        cv2.imshow("Probabilities", canvas)

        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    #camera.release()

cv2.destroyAllWindows()

我知道代码很多，但我想更广泛地说明我想做什么。拜托，如果有人能给我提示我应该如何正确处理这种转变，请告诉我 ;)

Answer 1

我得到了一些有用的东西，这可能对你有帮助(它将二进制数据从 javascript websocket 发送到 python 服务器，并将接收到的数据转换为你似乎在使用的枕头库的图像)。试试这个作为客户端站点脚本:

(async () => {
    const useFrameRate = 30;
    const stream = await navigator.mediaDevices.getUserMedia({video: true});
    const capture = new ImageCapture(stream.getVideoTracks()[0]);
    const socket = new WebSocket('ws://localhost:5678');
    const options = {imageWidth: 640, imageHeight: 480};
    socket.addEventListener('open', () => {
        const send = () => capture.takePhoto(options).then(blob => socket.send(blob)).catch(() => {});
        const sendloop = setInterval(send ,1000/useFrameRate);
    });
 })();

然后在你的服务器上是这样的:

import asyncio
import websockets
import io
from PIL import Image, ImageMode

async def time(websocket, path):
    while True:
        message = await websocket.recv()
        image = Image.open(io.BytesIO(message))
        # now do with your images whatever you want. I used image.show to check it, it was spamming my monitor

start_server = websockets.serve(time, "127.0.0.1", 5678)

asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()

解释:
起初，我想绕过 canvas 方法，直接从 getUserMedia-Stream 中抓取一个 Blob。我为此使用了 ImageCapture-API，尤其是 takePhoto()，它返回一个 Blob。
如果你愿意，你可以保留你的 Canvas 方法并调用.toBlob()。在你的 Canvas 上。
标准的 websocket 不仅接受字符串，还接受像 Blob 这样的字节对象，参见 here .它会自动将它们转换为二进制帧，您可以在 python 中将其作为二进制字符串处理，因为您已经正确地尝试过 io.BytesIO。