added stream sanity checks (sound, singe color)

Pipfile

@@ -36,6 +36,9 @@ python-socketio = {version = "*",extras = ["client"]}
 socketio-client = "*"
 websocket-client = "*"
 apscheduler = "*"
+pillow = "*"
+pydub = "*"
+simpleaudio = "*"
 
 [dev-packages]
 

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "83c5fbc0a5b6c72ecdd8842e12b7c647e6d43d94a83623a8c0268c08d8a835b7"
+            "sha256": "e84f3cf0b0e1452298555dc95f7493f8ec43bbff5471f088a1a9ebe64a9db44e"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -517,6 +517,34 @@
             ],
             "version": "==5.4.4"
         },
+        "pillow": {
+            "hashes": [
+                "sha256:0a628977ac2e01ca96aaae247ec2bd38e729631ddf2221b4b715446fd45505be",
+                "sha256:4d9ed9a64095e031435af120d3c910148067087541131e82b3e8db302f4c8946",
+                "sha256:54ebae163e8412aff0b9df1e88adab65788f5f5b58e625dc5c7f51eaf14a6837",
+                "sha256:5bfef0b1cdde9f33881c913af14e43db69815c7e8df429ceda4c70a5e529210f",
+                "sha256:5f3546ceb08089cedb9e8ff7e3f6a7042bb5b37c2a95d392fb027c3e53a2da00",
+                "sha256:5f7ae9126d16194f114435ebb79cc536b5682002a4fa57fa7bb2cbcde65f2f4d",
+                "sha256:62a889aeb0a79e50ecf5af272e9e3c164148f4bd9636cc6bcfa182a52c8b0533",
+                "sha256:7406f5a9b2fd966e79e6abdaf700585a4522e98d6559ce37fc52e5c955fade0a",
+                "sha256:8453f914f4e5a3d828281a6628cf517832abfa13ff50679a4848926dac7c0358",
+                "sha256:87269cc6ce1e3dee11f23fa515e4249ae678dbbe2704598a51cee76c52e19cda",
+                "sha256:875358310ed7abd5320f21dd97351d62de4929b0426cdb1eaa904b64ac36b435",
+                "sha256:8ac6ce7ff3892e5deaab7abaec763538ffd011f74dc1801d93d3c5fc541feee2",
+                "sha256:91b710e3353aea6fc758cdb7136d9bbdcb26b53cefe43e2cba953ac3ee1d3313",
+                "sha256:9d2ba4ed13af381233e2d810ff3bab84ef9f18430a9b336ab69eaf3cd24299ff",
+                "sha256:a62ec5e13e227399be73303ff301f2865bf68657d15ea50b038d25fc41097317",
+                "sha256:ab76e5580b0ed647a8d8d2d2daee170e8e9f8aad225ede314f684e297e3643c2",
+                "sha256:bf4003aa538af3f4205c5fac56eacaa67a6dd81e454ffd9e9f055fff9f1bc614",
+                "sha256:bf598d2e37cf8edb1a2f26ed3fb255191f5232badea4003c16301cb94ac5bdd0",
+                "sha256:c18f70dc27cc5d236f10e7834236aff60aadc71346a5bc1f4f83a4b3abee6386",
+                "sha256:c5ed816632204a2fc9486d784d8e0d0ae754347aba99c811458d69fcdfd2a2f9",
+                "sha256:dc058b7833184970d1248135b8b0ab702e6daa833be14035179f2acb78ff5636",
+                "sha256:ff3797f2f16bf9d17d53257612da84dd0758db33935777149b3334c01ff68865"
+            ],
+            "index": "pypi",
+            "version": "==7.0.0"
+        },
         "pyasn1": {
             "hashes": [
                 "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d",
@@ -532,38 +560,46 @@
         },
         "pycryptodomex": {
             "hashes": [
-                "sha256:04646e40ef5788bad6d415e52862ffcdf2ac2d888ba4a5c82d5cb44607a042f7",
+                "sha256:1537d2d15b604b303aef56e7f440895a1c81adbee786b91f1f06eddc34da5314",
-                "sha256:132f1e5fa84921f25695a313a6d4988847dfaee7fb1fd0d1fbe03ef678836f58",
+                "sha256:1d20ab8369b7558168fc014a0745c678613f9f486dae468cca2d68145196b8a4",
-                "sha256:17ad1ebaa00806305d34550fe5d3c776e38a27b8a2678dfb7871ef0209d64e46",
+                "sha256:1ecc9db7409db67765eb008e558879d298406642d33ade43a6488224d23e8081",
-                "sha256:27736fa02a2d3502e1ca4b150457e56ce3b98f132462f540073498884e5f8975",
+                "sha256:37033976f72af829fe15f7fe5fe1dbed308cc43a98d9dd9d2a0a76de8ca5ee78",
-                "sha256:38050b3fd86c74c6c79e40bbe824bec6431c3e4e36f6080ed544673ba2dc133a",
+                "sha256:3c3dd9d4c9c1e279d3945ae422895c901f98987333acc132dc094faf52afec35",
-                "sha256:3b9306b360bddbc8e098b16eab7adacf49389d212db9c3739588ab840a1ca868",
+                "sha256:3c9b3fba037ea52c626060c5a87ee6de7e86c99e8a7c6ee07302539985d2bd64",
-                "sha256:466e36ba74a7e725625e717fad3f36e0b9293c247b7d0439c66528026ef2834f",
+                "sha256:45ee555fc5e28c119a46d44ce373f5237e54a35c61b750fb3a94446b09855dbc",
-                "sha256:4f77360b23a21db32a4c35dacffac33dc30ac6a5a77162a34e99ab11ab631516",
+                "sha256:4c93038ac011b36512cb0bf2ee3e2aec774e8bc81021d015917c89fe02bb0ee5",
-                "sha256:5002388178845683c330a02f4faeddfe7cd477b87824987cca4718fa0c4f2085",
+                "sha256:50163324834edd0c9ce3e4512ded3e221c969086e10fdd5d3fdcaadac5e24a78",
-                "sha256:51be76756abfc1ddc97e1e2e3c38f4e62fb940161162368308ea9e5919e86c34",
+                "sha256:59b0ea9cda5490f924771456912a225d8d9e678891f9f986661af718534719b2",
-                "sha256:544628ae67d61c31c28a60e621dadd738b303c5266492355d5ebdb6e7dd1e78f",
+                "sha256:5cf306a17cccc327a33cdc3845629fa13f4573a4ec620ed607c79cf6785f2e27",
-                "sha256:6ff9d4a06bc40211eee05cd88436740d698a01233f4aaff9eb70d8a90e578966",
+                "sha256:5fff8da399af16a1855f58771223acbbdac720b9969cd03fc5013d2e9a7bd9a4",
-                "sha256:718329c6ca60260f1c27b8392e372dd51e4e691f7dcb88adc53eb3b76af6363c",
+                "sha256:68650ce5b9f7152b8283302a4617269f821695a612692640dd247bd12ab21c0b",
-                "sha256:918bc5a0170fe8ed7b72f202245b34f84a1997f5ca1520b9c7db71126e5acd62",
+                "sha256:6b3a9a562688996f760b5077714c3ab8b62ca56061b6e9ab7906841e43e19f91",
-                "sha256:a8ea72adde0d010f89abece5f024b1be95a5c52472e9a57b3ac7d59aee3c8238",
+                "sha256:7e938ed51a59e29431ea86fab60423ada2757728db0f78952329fa02a789bd31",
-                "sha256:a979d2c7bcc67282b7ec2600db384c63d37d74e250edb99168483605a380bf62",
+                "sha256:87aa70daad6f039e814790a06422a3189311198b674b62f13933a2bdcb6b1bcc",
-                "sha256:b350f9ad09b692aed57e669fc3f8cf918557fae9f0229c6ce9286a6fe8c1b60f",
+                "sha256:99be3a1df2b2b9f731ebe1c264a2c07c465e71cee68e35e1640b645b5213a755",
-                "sha256:be838abc8557a21a60d453c5a4e64c738966b8a0b7d7f8f97eb8bb44041ca452",
+                "sha256:a3f2908666e6f74b8c4893f86dd02e16170f50e4a78ae7f3468b6208d54bc205",
-                "sha256:bfa99692d3c8f994c5850cc8a894cba001abd76d34069a8bfaad173dd46387d6",
+                "sha256:ae3d44a639fd11dbdeca47e35e94febb1ee8bc15daf26673331add37146e0b85",
-                "sha256:c021b66f5b3c4ea0c45422ec3241bfea4a16651e1ee5459a136639d0716ccb3c",
+                "sha256:afb4c2fa3c6f492fd9a8b38d76e13f32d429b8e5e1e00238309391b5591cde0d",
-                "sha256:c7babb64484080057a24c74a82dbf7997904b1710b74caf62e261610f989b437",
+                "sha256:b1515ce3a8a2c3fa537d137c5ca5f8b7a902044d04e07d7c3aa26c3e026120fb",
-                "sha256:c96b7762b601dc8a58d7712235c3c152868116f58a7ffa40dcd1c6f6cd97405e",
+                "sha256:bf391b377413a197000b43ef2b74359974d8927d329a897c9f5ba7b63dca7b9c",
-                "sha256:d67b6e0bae0777a2c6c83275fbd7cbf53cd5f23c2028f908b0f7d996466e5b15",
+                "sha256:c436919117c23355740c669f89720673578b9aa4569bbfe105f6c10101fc1966",
-                "sha256:e15f39fcfb949cfd5536cc9647daba942b1a99b67e4d7211e3bdbcedbc2f823c",
+                "sha256:d2c3c280975638e2a2c2fd9cb36ab111980219757fa163a2755594b9448e4138",
-                "sha256:e380448f1e39736f6230ec284cd6d771956ad802d6ce5bc56947a2481080cac1",
+                "sha256:e585d530764c459cbd5d460aed0288807bb881f376ca9a20e653645217895961",
-                "sha256:e5236f2171b21e704d1854fd809a7228eb22e29c894af31459e41986e6a53f87",
+                "sha256:e76e6638ead4a7d93262a24218f0ff3ff74de6b6c823b7e19dccb31b6a481978",
-                "sha256:ea7b48ce8dbbc86ebadcfe56ebc10d413bdd12c9a5ff0b9147a41993f12b80b3",
+                "sha256:ebfc2f885cafda076c31ae30fa0dd81e7e919ec34059a88d3018ed66e83fcce3",
-                "sha256:f39f5b58d8fe348ed604bb44a89ca93b26130c275db2b249f718f1538cb70500",
+                "sha256:f5797a39933a3d41526da60856735e6684b2b71a8ca99d5f79555ca121be2f4b",
-                "sha256:f545f776e45f74c41329e4020463fdd4d0cd0a7501bdf9e50251dafe7bd959a9",
+                "sha256:f7e5fc5e124200b19a14be33fb0099e956e6ebb5e25d287b0829ef0a78ed76c7",
-                "sha256:f667ac7ae29c19530f199854635f1a97e73d0bfd24163e0db6bdba7dba04eb9f"
+                "sha256:fb350e31e55211fec8ddc89fc0256f3b9bc3b44b68a8bde1cf44b3b4e80c0e42"
             ],
-            "version": "==3.9.6"
+            "version": "==3.9.7"
+        },
+        "pydub": {
+            "hashes": [
+                "sha256:c362fa02da1eebd1d08bd47aa9b0102582dff7ca2269dbe9e043d228a0c1ea93",
+                "sha256:d29901a486fb421c5d7b0f3d5d3a60527179204d8ffb20e74e1ae81c17e81b46"
+            ],
+            "index": "pypi",
+            "version": "==0.23.1"
         },
         "pyjwkest": {
             "hashes": [
@@ -579,6 +615,13 @@
             "index": "pypi",
             "version": "==1.7.1"
         },
+        "pyreadline": {
+            "hashes": [
+                "sha256:4530592fc2e85b25b1a9f79664433da09237c1a270e4d78ea5aa3a2c7229e2d1"
+            ],
+            "markers": "sys_platform == 'win32'",
+            "version": "==2.1"
+        },
         "pyrsistent": {
             "hashes": [
                 "sha256:cdc7b5e3ed77bed61270a47d35434a30617b9becdf2478af76ad2c6ade307280"
@@ -661,6 +704,19 @@
             "index": "pypi",
             "version": "==2.4.3"
         },
+        "simpleaudio": {
+            "hashes": [
+                "sha256:05b63da515f5fc7c6f40e4d9673d22239c5e03e2bda200fc09fd21c185d73713",
+                "sha256:67348e3d3ccbae73bd126beed7f1e242976889620dbc6974c36800cd286430fc",
+                "sha256:691c88649243544db717e7edf6a9831df112104e1aefb5f6038a5d071e8cf41d",
+                "sha256:86f1b0985629852afe67259ac6c24905ca731cb202a6e96b818865c56ced0c27",
+                "sha256:f1a4fe3358429b2ea3181fd782e4c4fff5c123ca86ec7fc29e01ee9acd8a227a",
+                "sha256:f346a4eac9cdbb1b3f3d0995095b7e86c12219964c022f4d920c22f6ca05fb4c",
+                "sha256:f68820297ad51577e3a77369e7e9b23989d30d5ae923bf34c92cf983c04ade04"
+            ],
+            "index": "pypi",
+            "version": "==1.0.4"
+        },
         "six": {
             "hashes": [
                 "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a",

backend/cron/cron_state_checker.py

@@ -9,7 +9,7 @@ from threading import Lock
 from typing import Union, Callable, TypeVar, Generic, Set, List
 
 from backend.models import Recorder
-from backend.tools.simple_state_checker import check_capture_agent_state, ping_capture_agent
+from backend.tools.recorder_state_checker import check_capture_agent_state, ping_capture_agent
 
 logger = logging.getLogger("lrc.cron.recorder_state")
 

backend/models/recorder_model.py

@@ -144,6 +144,9 @@ class Recorder(db.Model, ModelBase):
     _additional_notes_json_string = db.Column(db.UnicodeText, default='')
     additional_camera_connected = db.Column(db.Boolean, default=False)
     firmware_version = db.Column(db.String, nullable=True, default=None)
+    archive_stream1 = db.Column(db.String, nullable=True, default=None)
+    archive_stream2 = db.Column(db.String, nullable=True, default=None)
+    confidence_stream = db.Column(db.String, nullable=True, default=None)
     room_id = db.Column(db.Integer, db.ForeignKey('room.id'))
     room = db.relationship('Room', uselist=False, back_populates='recorder')  # one-to-one relation (uselist=False)
     recorder_model_id = db.Column(db.Integer, db.ForeignKey('recorder_model.id'))

backend/tools/recorder_state_checker.py

@@ -3,6 +3,7 @@ import os
 import logging
 import subprocess
 import threading
+import time
 from io import StringIO
 from logging.handlers import MemoryHandler
 from pprint import pprint
@@ -124,6 +125,23 @@ def get_recorder_adapter(recorder_info: Union[dict, Recorder]) -> RecorderAdapte
     return rec
 
 
+def check_stream_sanity(recorder: Recorder, recorder_adapter: RecorderAdapter):
+    if recorder.archive_stream1 is None and recorder.archive_stream2 is None:  # fall back to default names and rtsp
+        archive_stream_1_url = "rtsp://{}/{}".format(recorder_adapter.address, Config.DEFAULT_ARCHIVE_STREAM_1_NAME)
+        archive_stream_2_url = "rtsp://{}/{}".format(recorder_adapter.address, Config.DEFAULT_ARCHIVE_STREAM_2_NAME)
+    else:
+        archive_stream_1_url = recorder.archive_stream1
+        archive_stream_2_url = recorder.archive_stream2
+
+    for i in range(0, Config.STREAM_SANITY_CHECK_RETRIES):
+        do_checks = False
+        if do_checks:
+            return True
+        else:
+            time.sleep(Config.STREAM_SANITY_CHECK_INTERVAL_SEC)
+    return False  # stream sanity check failed!
+
+
 def check_capture_agent_state(recorder_agent: Union[Recorder, dict]):
     if recorder_agent.get('offline', False):
         logger.info("OK - Recorder {} is in offline / maintenance mode".format(recorder_agent.get('name')))
@@ -142,6 +160,7 @@ def check_capture_agent_state(recorder_agent: Union[Recorder, dict]):
                     logger.info("OK – recorder {} is recording :)".format(recorder_agent.get('name')))
                     with agent_states_lock:
                         agent_states[recorder_agent.get('name')] = 'OK - recorder is recording'
+
                 else:
                     logger.info(rec.get_recording_status())
                     logger.error("FATAL - recorder {} must be recording but is not!!!!".format(recorder_agent.get('name')))

backend/tools/recorder_streams_sanity_checks.py

@@ -0,0 +1,132 @@
+import io
+import sys
+
+import ffmpeg
+import os
+import tempfile
+from PIL import Image
+from pydub import AudioSegment
+from pydub.playback import play
+
+
+def old_test():
+    file_name = tempfile.gettempdir() + os.path.sep + "test.jpg"
+    print(file_name)
+    if os.path.exists(file_name):
+        os.remove(file_name)
+    process = (
+        ffmpeg
+            .input('rtsp://172.22.246.207/extron1')
+            # .input('rtsp://172.22.246.207/extron3')
+            .output(file_name, vframes=1)
+            # .output('-', format='h264')
+            .run(capture_stdout=True)
+    )
+    image = Image.open(file_name)
+    r, g, b = image.split()
+    print(r.histogram())
+    print(g.histogram())
+    print(b.histogram())
+    image.show()
+
+
+# old_test()
+
+def is_single_color_image(image):
+    single_color_image = True
+    color = {}
+    count = 0
+    color_channels = image.split()
+    for c in color_channels:  # r, g, b
+
+        hist = c.histogram()
+        num_of_non_zero_values = len([v for v in hist if v != 0])
+        if num_of_non_zero_values > 1:
+            single_color_image = False
+            break
+        else:
+            color[count] = [i for i in enumerate(hist) if i[1] != 0][0][0]
+        count += 1
+    return single_color_image, color
+
+
+def check_frame_is_valid(stream_url, raise_errors=True):
+    try:
+        frame, _ = (
+            ffmpeg
+                .input(stream_url)
+                .output('pipe:', vframes=1, format='image2', vcodec='mjpeg')
+                .run(capture_stdout=True, capture_stderr=True)
+        )
+        image = Image.open(io.BytesIO(frame))
+        single_color_image, color = is_single_color_image(image)
+        if not single_color_image:
+            image.show()
+            return True, "all good :-)"
+        else:
+            if all(map(lambda x: x == 0, color.values())):
+                return False, "Image is entirely black! (color: {} (RGB))".format(
+                    ':'.join([str(x) for x in color.values()]))
+            return False, "Image has only one single color! (color: {} (RGB))".format(
+                ':'.join([str(x) for x in color.values()]))
+    except ffmpeg.Error as e:
+        msg = "Could not connect to stream URL or other error!"
+        try:
+            msg += " ({})".format(e.stderr.decode('utf-8').strip().split("\n")[-1])
+        except IndexError:
+            pass
+        if raise_errors:
+            raise Exception(msg)
+        else:
+            return False, msg
+
+
+# print(check_frame_is_valid('rtsp://172.22.246.207/extron2'))
+
+def check_if_audio_is_valid(stream_url, sample_length_sec=3, lower_alert_limit_dBFS=-40.0, raise_errors=True):
+    file_name = tempfile.NamedTemporaryFile(suffix='.aac').name
+    if os.path.exists(file_name):
+        os.remove(file_name)
+    try:
+        frame, _ = (
+            ffmpeg
+                .input(stream_url, t=sample_length_sec)
+                .output(file_name)
+                .run(capture_stdout=True, capture_stderr=True)
+        )
+
+        sound = AudioSegment.from_file(file_name, "aac")
+        # print(sound.dBFS)
+        play(sound)
+        if sound.max_dBFS == -float('inf'):
+            return False, "No active audio signal detected!"
+        elif sound.max_dBFS < lower_alert_limit_dBFS:
+            return False, "Very low volume (< {} dBFS) detected! ({})".format(lower_alert_limit_dBFS, sound.max_dBFS)
+
+        return True, "good audio signal detected! ({} max dBFS in {}s sample)".format(sound.max_dBFS, sample_length_sec)
+    except ffmpeg.Error as e:
+        msg = "Could not connect to stream URL or other error!"
+        try:
+            msg += " ({})".format(e.stderr.decode('utf-8').strip().split("\n")[-1])
+        except IndexError:
+            pass
+        if raise_errors:
+            raise Exception(msg)
+        else:
+            return False, msg
+
+print(check_if_audio_is_valid('rtsp://172.22.246.207/extron1'))
+
+
+def check_if_audio_is_valid_stream(stream_url, raise_errors=True):
+    audio, _ = (
+        ffmpeg
+            .input(stream_url, t=2)
+            .output('pipe:', f="adts", acodec='copy')
+            .run(capture_stdout=False, capture_stderr=False)
+    )
+    audio = io.BytesIO(audio)
+    sound = AudioSegment.from_file(audio, "aac")
+    play(sound)
+
+# check_if_audio_is_valid('rtsp://172.22.246.207/extron1')

backend/tools/stream_handling.py

@@ -1,15 +0,0 @@
-import ffmpeg
-
-packet_size = 4096
-
-process = (
-    ffmpeg
-        .input('rtsp://172.22.246.207/extron1')
-        #.input('rtsp://172.22.246.207/extron3')
-        .output('/tmp/test.jpg', vframes=1)
-        #.output('-', format='h264')
-        .run_async(pipe_stdout=True)
-)
-
-while process.poll() is None:
-    packet = process.stdout.read(packet_size)