exception decorator and mail send for errors and changes to rec state checker

backend/tools/exception_decorator.py

@@ -0,0 +1,13 @@
+from backend import LrcException
+
+
+def exception_decorator(*exceptions):
+    def decorator(func):
+        def new_func(*args, **kwargs):
+            try:
+                ret = func(*args, **kwargs)
+                return ret
+            except exceptions as e:
+                raise LrcException(e)
+        return new_func
+    return decorator

backend/tools/send_mail.py

@@ -0,0 +1,45 @@
+import smtplib
+import traceback
+from email.message import EmailMessage
+from typing import List, Union
+
+from backend import main_logger, Config
+
+
+def send_mail(subject: str, msg_body: str, to_mail: List, from_mail: str = Config.FROM_MAIL):
+    subject = subject.strip().replace("\r", "").split("\n")[0]  # remove breaks resp. cut after line break
+    try:
+        msg = EmailMessage()
+        msg.set_content(msg_body)
+        msg["Subject"] = subject
+        msg["From"] = from_mail
+        msg["To"] = to_mail
+        s = smtplib.SMTP(Config.SMTP_SERVER)
+        s.send_message(msg)
+        s.quit()
+    except Exception as e:
+        main_logger.error(
+            "Could not send E-Mail (Exception: {})".format(str(e)))
+
+
+def send_error_mail(message_or_exception: Union[str, Exception], subject: str = None,
+                    subject_prefix: str = "[LRC] Error: ",
+                    receiver_mail_addresses=Config.ERROR_E_MAIL_TO):
+    if isinstance(message_or_exception, Exception):
+        if subject is None:
+            subject = subject_prefix + str(type(message_or_exception))
+        else:
+            subject = subject_prefix + subject
+        msg_body = str(type(message_or_exception)) + ": " + str(message_or_exception) + "\n\n" + traceback.format_exc()
+    else:
+        if subject is None:
+            subject = str(message_or_exception)
+        else:
+            subject = subject_prefix + subject
+        msg_body = str(message_or_exception)
+
+    send_mail(subject, msg_body, receiver_mail_addresses)
+
+
+def send_warning_mail():
+    pass

backend/tools/simple_state_checker.py

@@ -1,23 +1,31 @@
 import json
 import os
 import logging
+from io import StringIO
+
+import requests
+from requests.auth import HTTPBasicAuth
 from multiprocessing.pool import ThreadPool
 
 from pprint import pprint
 
-import requests
-from requests.auth import HTTPBasicAuth
 from ics import Calendar
 
+from backend import LrcException
 from backend.config import Config
+from backend.recorder_adapters import RecorderAdapter
+
 from backend.recorder_adapters.epiphan_base import EpiphanV1
 from backend.recorder_adapters.extron_smp import SMP
+from backend.tools.send_mail import send_error_mail
 
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-stream_handler = logging.StreamHandler()
-stream_handler.setLevel(logging.DEBUG)
-logger.addHandler(stream_handler)
+logger = logging.getLogger("lrc.tools.simple_state_checker")
+
+rec_err_state_log_stream = StringIO()
+rec_err_state_log_stream_handler = logging.StreamHandler(stream=rec_err_state_log_stream)
+rec_err_state_log_stream_handler.setLevel(logging.ERROR)
+
+logger.addHandler(rec_err_state_log_stream_handler)
 
 base_url = "https://opencast.bibliothek.kit.edu"
 
@@ -46,7 +54,6 @@ def get_service_url(service_type: str):
 def get_calender(rec_id):
     params = {'agentid': rec_id}
     url = get_service_url('org.opencastproject.scheduler') + "/calendars"
-    print(url)
     res = session.get(url, params=params)
     if res.ok:
         return Calendar(res.text)
@@ -80,41 +87,49 @@ def notify_users_of_problem(msg: str):
     pass
 
 
+def get_recorder_adapter(recorder_info:dict) -> RecorderAdapter:
+    if "SMP" in recorder_info["type"]:
+        rec = SMP(recorder_info['ip'], recorder_info['password'])
+    else:
+        rec = EpiphanV1(recorder_info['ip'], recorder_info["username"], recorder_info["password"])
+    return rec
+
+
 def check_capture_agent_state(a: dict):
     logger.debug("Checking Agent {}".format(a['name']))
     c = get_calender(a['name'])
     is_recording_in_calendar = len(list(c.timeline.now())) >= 1
     if is_recording_in_calendar:
+        logger.info("{} has entry in Calender and should therfor be recording... checking now!".format(a['name']))
         if a['state'] == "capturing":
-            print(
+            logger.info(
                 "{} is in capturing state, so there should be an entry in the calendar of the recorder, right? -> {}".format(
                     a['name'], is_recording_in_calendar
                 ))
-            rec = get_recorder_by_name(a['name'])
-            print(rec)
-            if "SMP" in rec["type"]:
-                print("using SMP adapter")
-                rec = SMP(rec['ip'], rec['password'])
-            else:
-                rec = EpiphanV1(rec['ip'], rec["username"], rec["password"])
-            if rec.is_recording():
-                print("OK – recorder is recording :)")
-            else:
-                logger.info(rec.get_recording_status())
-                logger.error("FATAL - recorder {} must be recording!!!!".format(a['name']))
+            recorder_info = get_recorder_by_name(a['name'])
+            try:
+                rec = get_recorder_adapter(recorder_info)
+                if rec.is_recording():
+                    logger.info("OK – recorder {} is recording :)".format(a['name']))
+                else:
+                    logger.info(rec.get_recording_status())
+                    logger.error("FATAL - recorder {} must be recording but is not!!!!".format(a['name']))
+            except LrcException as e:
+                logger.fatal("Exception occurred: {}".format(str(e)))
+                logger.error("Could not check state of recorder {}, Address: {}".format(a['name'], recorder_info['ip']))
         else:
-            print("FATAL: {} is not in capturing state...but should be!!".format(a['name']))
+            logger.error("FATAL: {} is not in capturing state...but should be!!".format(a['name']))
     else:
-        rec = get_recorder_by_name(a['name'])
-        if "SMP" in rec["type"]:
-            logger.debug("using SMP adapter")
-            rec = SMP(rec['ip'], rec['password'])
-        else:
-            rec = EpiphanV1(rec['ip'], rec["username"], rec["password"])
-        if rec.is_recording():
-            logger.error("FATAL - recorder must not be recording!!!!")
-        else:
-            logger.info("OK – recorder is not recording :)")
+        recorder_info = get_recorder_by_name(a['name'])
+        try:
+            rec = get_recorder_adapter(recorder_info)
+            if rec.is_recording():
+                logger.error("FATAL - recorder must not be recording!!!!")
+            else:
+                logger.info("OK – recorder is not recording :)")
+        except LrcException as e:
+            logger.fatal("Exception occurred: {}".format(str(e)))
+            logger.error("Could not check state of recorder {}, Address: {}".format(a['name'], recorder_info['ip']))
 
 
 agents = get_capture_agents()
@@ -123,6 +138,11 @@ logger.info("Got {} capture agents that will be checked...".format(len(agents)))
 pool = ThreadPool(5)
 pool.map(check_capture_agent_state, agents)
 
+pool.close()
+pool.join()
+logged_events = rec_err_state_log_stream.getvalue()
+if len(logged_events) > 0:
+    send_error_mail(logged_events, "Errors have been detected while checking recorder states!")
 exit()
 
 c = get_calender('CS 30.46 Chemie Neuer Hoersaal')