tobias/lrc-backend
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

further advanced state api and stream checks

Browse Source
This commit is contained in:
Tobias Kurze
2020-02-25 17:00:01 +01:00
parent 1d011af64b
commit f181e4a785
7 changed files with 126 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
backend/tools/recorder_state_checker.py
View File

@@ -23,20 +23,21 @@ from backend.recorder_adapters import RecorderAdapter
from backend.recorder_adapters.epiphan_base import Epiphan
from backend.recorder_adapters.extron_smp import SMP35x
from backend.tools.recorder_streams_sanity_checks import check_frame_is_valid, check_if_audio_is_valid
from backend.tools.send_mail import send_error_mail, get_smtp_error_handler
logger = logging.getLogger("lrc.tools.simple_state_checker")
smtp_error_handler = get_smtp_error_handler(subject="Errors have been detected while checking recorder states!")
#mem_handler = MemoryHandler(capacity=100, flushLevel=logging.FATAL, target=smtp_error_handler)
#mem_handler.setLevel(logging.WARNING)
# mem_handler = MemoryHandler(capacity=100, flushLevel=logging.FATAL, target=smtp_error_handler)
# mem_handler.setLevel(logging.WARNING)
rec_err_state_log_stream = StringIO()
rec_err_state_log_stream_handler = logging.StreamHandler(stream=rec_err_state_log_stream)
rec_err_state_log_stream_handler.setLevel(logging.WARNING)
logger.addHandler(rec_err_state_log_stream_handler)
#logger.addHandler(mem_handler)
# logger.addHandler(mem_handler)
session = requests.session()
session.auth = HTTPBasicAuth(Config.OPENCAST_USER, Config.OPENCAST_PW)
@@ -123,22 +124,60 @@ def get_recorder_adapter(recorder_info: Union[dict, Recorder]) -> RecorderAdapte
return rec
def check_stream_sanity(recorder: Recorder, recorder_adapter: RecorderAdapter):
if recorder.archive_stream1 is None and recorder.archive_stream2 is None: # fall back to default names and rtsp
def check_stream_sanity(recorder_agent: Union[Recorder, dict], recorder_adapter: RecorderAdapter = None):
if recorder_adapter is None:
recorder_info = get_recorder_by_name(recorder_agent.get('name'))
recorder_adapter = get_recorder_adapter(recorder_info)
if not recorder_adapter.is_recording():
return True, "not recording, so there is no stream!", recorder_agent.get('name')
if recorder_agent.get('archive_stream1') is None and recorder_agent.get(
'archive_stream2') is None: # fall back to default names and rtsp
archive_stream_1_url = "rtsp://{}/{}".format(recorder_adapter.address, Config.DEFAULT_ARCHIVE_STREAM_1_NAME)
archive_stream_2_url = "rtsp://{}/{}".format(recorder_adapter.address, Config.DEFAULT_ARCHIVE_STREAM_2_NAME)
else:
archive_stream_1_url = recorder.archive_stream1
archive_stream_2_url = recorder.archive_stream2
archive_stream_1_url = recorder_agent.get('archive_stream1')
archive_stream_2_url = recorder_agent.get('archive_stream2')
frame_msg = frame_msg2 = sound_msg = sound_msg2 = "unknown"
for i in range(0, Config.STREAM_SANITY_CHECK_RETRIES):
do_checks = False
frame_ok, frame_msg = check_frame_is_valid(archive_stream_1_url, raise_errors=False)
if not frame_ok:
logger.warning(
"Archive stream 1 ({}) of {} is not ok (frame): {}".format(archive_stream_1_url,
recorder_agent.get('name'),
frame_msg))
frame_ok, frame_msg2 = check_frame_is_valid(archive_stream_2_url, raise_errors=False)
if not frame_ok:
logger.warning(
"Archive stream 2 {} of ({}) is not ok: {} (frame)".format(archive_stream_2_url,
recorder_agent.get('name'),
frame_msg2))
sound_ok, sound_msg = check_if_audio_is_valid(archive_stream_1_url, raise_errors=False)
if not sound_ok:
logger.warning(
"Archive stream 1 {} of ({}) is not ok (audio): {}".format(archive_stream_1_url,
recorder_agent.get('name'),
sound_msg))
sound_ok, sound_msg2 = check_if_audio_is_valid(archive_stream_2_url, raise_errors=False)
if not sound_ok:
logger.warning(
"Archive stream 2 {} of ({}) is not ok (audio): {}".format(archive_stream_2_url,
recorder_agent.get('name'),
sound_msg2))
if frame_ok and sound_ok:
return True, "At least one archive stream is fine (audio and image)! " \
"(s1: a: {}, v: {}, s2: a: {}, v: {})".format(sound_msg, frame_msg, sound_msg2,
frame_msg2), recorder_agent.get('name')
if do_checks:
return True
else:
time.sleep(Config.STREAM_SANITY_CHECK_INTERVAL_SEC)
return False # stream sanity check failed!
error_msg = "After {} retries, stream checks failed and returned: archive_stream1: audio:{}, frame:{}, " \
"archive_stream2: audio:{}, frame:{}".format(Config.STREAM_SANITY_CHECK_RETRIES, sound_msg,
frame_msg, sound_msg2, frame_msg2)
logger.error(error_msg)
return False, error_msg, recorder_agent.get('name') # stream sanity check failed!
def check_capture_agent_state(recorder_agent: Union[Recorder, dict]):
@@ -150,7 +189,8 @@ def check_capture_agent_state(recorder_agent: Union[Recorder, dict]):
c = get_calender(recorder_agent.get('name'))
is_recording_in_calendar = len(list(c.timeline.now())) >= 1
if is_recording_in_calendar:
logger.info("{} has entry in Calender and should therefore be recording... checking now!".format(recorder_agent.get('name')))
logger.info("{} has entry in Calender and should therefore be recording... checking now!".format(
recorder_agent.get('name')))
if recorder_agent['state'] == "capturing":
recorder_info = get_recorder_by_name(recorder_agent.get('name'))
try:
@@ -159,16 +199,17 @@ def check_capture_agent_state(recorder_agent: Union[Recorder, dict]):
logger.info("OK – recorder {} is recording :)".format(recorder_agent.get('name')))
with agent_states_lock:
agent_states[recorder_agent.get('name')] = 'OK - recorder is recording'
else:
logger.info(rec.get_recording_status())
logger.error("FATAL - recorder {} must be recording but is not!!!!".format(recorder_agent.get('name')))
logger.error(
"FATAL - recorder {} must be recording but is not!!!!".format(recorder_agent.get('name')))
agent_state_error_msg = "FATAL - recorder must be recording but is not!"
with agent_states_lock:
agent_states[recorder_agent['name']] = 'FATAL - recorder is NOT recording, but should!'
except LrcException as e:
logger.fatal("Exception occurred: {}".format(str(e)))
logger.error("Could not check state of recorder {}, Address: {}".format(recorder_agent.get('name'), recorder_info.get('ip')))
logger.error("Could not check state of recorder {}, Address: {}".format(recorder_agent.get('name'),
recorder_info.get('ip')))
else:
logger.error("FATAL: {} is not in capturing state...but should be!!".format(recorder_agent.get('name')))
agent_state_error_msg = "FATAL - is not in capturing state...but should be!"
@@ -187,8 +228,10 @@ def check_capture_agent_state(recorder_agent: Union[Recorder, dict]):
agent_states[recorder_agent.get('name')] = 'OK - recorder is NOT recording'
except LrcException as e:
logger.fatal("Exception occurred: {}".format(str(e)))
logger.error("Could not check state of recorder {}, Address: {}".format(recorder_agent.get('name'), recorder_info.get('ip')))
agent_state_error_msg = "FATAL - Could not check state of recorder! Address: {}".format(recorder_info.get('ip'))
logger.error("Could not check state of recorder {}, Address: {}".format(recorder_agent.get('name'),
recorder_info.get('ip')))
agent_state_error_msg = "FATAL - Could not check state of recorder! Address: {}".format(
recorder_info.get('ip'))
if agent_state_error_msg is None:
return True, agent_states[recorder_agent.get('name')], recorder_agent.get('name')
@@ -213,7 +256,8 @@ def ping_capture_agent(recorder_agent: Union[Recorder, dict]):
return True, "Successfully pinged {}. :-)".format(recorder_agent.get('name')), recorder_agent.get('name')
except subprocess.CalledProcessError:
logger.error("Can not ping {} ({})!!".format(recorder_agent.get('name'), recorder_ip))
return False, "Unable to ping {} ({})".format(recorder_agent.get('name'), recorder_ip), recorder_agent.get('name')
return False, "Unable to ping {} ({})".format(recorder_agent.get('name'), recorder_ip), recorder_agent.get(
'name')
if __name__ == '__main__':
@@ -253,4 +297,4 @@ if __name__ == '__main__':
))
send_error_mail(logged_events, "Errors have been detected while checking recorder states!")
#mem_handler.close()
# mem_handler.close()