exception decorator and mail send for errors and changes to rec state checker

backend/__init__.py

@@ -3,7 +3,10 @@
 Backend base module
 """
 import logging
+from io import StringIO
 from logging.config import dictConfig
+from typing import Union
+
 import coloredlogs as coloredlogs
 import jwt
 import requests
@@ -25,7 +28,6 @@ __email__ = "it@t-kurze.de"
 # __status__ = "Production"
 __status__ = "Development"
 
-
 dictConfig({
     'version': 1,
     'formatters': {
@@ -66,7 +68,7 @@ dictConfig({
         },
     },
     'loggers': {
-        'mal': {
+        'lrc': {
             'level': Config.LOG_LEVEL,
             'handlers': ['wsgi', 'file', 'errors_file']
         }
@@ -78,18 +80,34 @@ dictConfig({
 })
 
 main_logger = logging.getLogger("lrc")
+
+# following might be dangerous, as buffer might be filled without a mechanism to empty it
+# error_log_stream = StringIO()
+# error_log_stream_handler = logging.StreamHandler(stream=error_log_stream)
+# error_log_stream_handler.setLevel(logging.ERROR)
+# main_logger.addHandler(error_log_stream_handler)
+
 coloredlogs.install(level='DEBUG', logger=main_logger)
 
 
 class LrcException(Exception):
-    def __init__(self, message: str, html_code: int = None):
-        super().__init__(message)
+    def __init__(self, message_or_exception: Union[str, Exception], html_code: int = None):
+        if isinstance(message_or_exception, str):
+            super().__init__(message_or_exception)
+            self.type = None
+        else:
+            super().__init__(str(message_or_exception))
+            self.type = type(message_or_exception)
         self.html_code = html_code
 
     def __repr__(self):
+        if self.type is None:
+            msg = "LRC Exception: \"{}\"".format(', '.join(super().args))
+        else:
+            msg = "LRC Exception: (original Exception: {}) \"{}\"".format(self.type, ', '.join(super().args))
         if self.html_code is not None:
-            return "LRC Exception: \"{}\" (HTML Code: {})".format(', '.join(super().args), self.html_code)
-        return ', '.join(super().args)
+            msg += " (HTML Code: {})".format(self.html_code)
+        return msg
 
     def __str__(self):
         return self.__repr__()

backend/recorder_adapters/__init__.py

@@ -104,6 +104,9 @@ class RecorderAdapter:
     def is_recording(self) -> bool:
         pass
 
+    def get_recording_status(self) -> str:
+        pass
+
 
 def get_defined_recorder_adapters():
     models = []

backend/recorder_adapters/epiphan_base.py

@@ -6,11 +6,14 @@ from pprint import pprint
 
 import requests
 from requests.auth import HTTPBasicAuth
+from requests.exceptions import ConnectionError
 
 from backend import LrcException
 from backend.recorder_adapters import RecorderAdapter
 
 # HOST = "localhost"
+from backend.tools.exception_decorator import exception_decorator
+
 BASE_URL = "http://172.23.8.102"  # Audimax SMP 351
 
 USER = "admin"
@@ -34,12 +37,14 @@ class EpiphanV1(RecorderAdapter):
     def _get_version(self):
         pass
 
-    def get_status(self) -> dict:
+    @exception_decorator(ConnectionError)
+    def get_recording_status(self) -> dict:
         res = self.session.get(self.url + "/admin/ajax/recorder_status.cgi")
         if res.ok:
             return res.json()
         raise LrcException(res.text, res.status_code)
 
+    @exception_decorator(ConnectionError)
     def get_sysinfo(self) -> dict:
         res = self.session.get(self.url + "/ajax/sysinfo.cgi")
         if res.ok:
@@ -47,7 +52,7 @@ class EpiphanV1(RecorderAdapter):
         raise LrcException(res.text, res.status_code)
 
     def is_recording(self) -> bool:
-        state = self.get_status().get('state', None)
+        state = self.get_recording_status().get('state', None)
         return state == "up"
 
     def get_recording_time(self):
@@ -55,7 +60,7 @@ class EpiphanV1(RecorderAdapter):
         Returns recording time in seconds. Also returns 0 if not recording.
         :return:
         """
-        return self.get_status().get('seconds', None)
+        return self.get_recording_status().get('seconds', None)
 
     def start_recording(self):
         res = self.session.get(self.url + "/admin/ajax/start_recorder.cgi")

backend/recorder_adapters/extron_smp.py

@@ -2,8 +2,9 @@ import logging
 
 from backend import LrcException
 from backend.recorder_adapters import telnetlib, TelnetAdapter, RecorderAdapter
+from backend.tools.exception_decorator import exception_decorator
 
-logger = logging.getLogger()
+logger = logging.getLogger("lrc.recorder_adapters.extron_smp")
 
 RECORDER_MODEL_NAME = "SMP 351 / 352"
 VERSION = "0.9.0"
@@ -28,11 +29,13 @@ class SMP(TelnetAdapter, RecorderAdapter):
             self._login()
 
     def _login(self):
-        print("Connecting to {} ...".format(self.address))
+        logger.info("Connecting to {} ...".format(self.address))
         try:
             self.tn = telnetlib.Telnet(self.address)
         except TimeoutError as e:
             raise LrcException(str(e))
+        except ConnectionRefusedError as e:
+            raise LrcException(str(e))
         self.tn.read_until("\r\nPassword:")
         # password = getpass.getpass()
         password = self.admin_pw
@@ -371,6 +374,7 @@ class SMP(TelnetAdapter, RecorderAdapter):
         self.tn.write("{}Y2RCDR\n".format(self.esc_char))
         return TelnetAdapter._get_response_str(self.tn.read_until_non_empty_line())
 
+    @exception_decorator(ConnectionError)
     def get_recording_status(self):
         """
         Status may be one of:

backend/serve_frontend.py

@@ -16,6 +16,7 @@ if not os.path.exists(fe_path) or not os.path.exists(os.path.join(fe_path, "inde
     app.logger.critical(
         "Frontend path and/or index.html does not exist! Please build frontend before continuing! "
         "You might want to go to ../frontend and continue from there.")
+    print("FATAL: Frontend path wrong or index.html missing -> EXITING!")
     exit()
 fe_bp = Blueprint('frontend', __name__, url_prefix='/', template_folder=os.path.join(fe_path, ""))
 

backend/tools/exception_decorator.py

@@ -0,0 +1,13 @@
+from backend import LrcException
+
+
+def exception_decorator(*exceptions):
+    def decorator(func):
+        def new_func(*args, **kwargs):
+            try:
+                ret = func(*args, **kwargs)
+                return ret
+            except exceptions as e:
+                raise LrcException(e)
+        return new_func
+    return decorator

backend/tools/send_mail.py

@@ -0,0 +1,45 @@
+import smtplib
+import traceback
+from email.message import EmailMessage
+from typing import List, Union
+
+from backend import main_logger, Config
+
+
+def send_mail(subject: str, msg_body: str, to_mail: List, from_mail: str = Config.FROM_MAIL):
+    subject = subject.strip().replace("\r", "").split("\n")[0]  # remove breaks resp. cut after line break
+    try:
+        msg = EmailMessage()
+        msg.set_content(msg_body)
+        msg["Subject"] = subject
+        msg["From"] = from_mail
+        msg["To"] = to_mail
+        s = smtplib.SMTP(Config.SMTP_SERVER)
+        s.send_message(msg)
+        s.quit()
+    except Exception as e:
+        main_logger.error(
+            "Could not send E-Mail (Exception: {})".format(str(e)))
+
+
+def send_error_mail(message_or_exception: Union[str, Exception], subject: str = None,
+                    subject_prefix: str = "[LRC] Error: ",
+                    receiver_mail_addresses=Config.ERROR_E_MAIL_TO):
+    if isinstance(message_or_exception, Exception):
+        if subject is None:
+            subject = subject_prefix + str(type(message_or_exception))
+        else:
+            subject = subject_prefix + subject
+        msg_body = str(type(message_or_exception)) + ": " + str(message_or_exception) + "\n\n" + traceback.format_exc()
+    else:
+        if subject is None:
+            subject = str(message_or_exception)
+        else:
+            subject = subject_prefix + subject
+        msg_body = str(message_or_exception)
+
+    send_mail(subject, msg_body, receiver_mail_addresses)
+
+
+def send_warning_mail():
+    pass

backend/tools/simple_state_checker.py

@@ -1,23 +1,31 @@
 import json
 import os
 import logging
+from io import StringIO
+
+import requests
+from requests.auth import HTTPBasicAuth
 from multiprocessing.pool import ThreadPool
 
 from pprint import pprint
 
-import requests
-from requests.auth import HTTPBasicAuth
 from ics import Calendar
 
+from backend import LrcException
 from backend.config import Config
+from backend.recorder_adapters import RecorderAdapter
+
 from backend.recorder_adapters.epiphan_base import EpiphanV1
 from backend.recorder_adapters.extron_smp import SMP
+from backend.tools.send_mail import send_error_mail
 
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-stream_handler = logging.StreamHandler()
-stream_handler.setLevel(logging.DEBUG)
-logger.addHandler(stream_handler)
+logger = logging.getLogger("lrc.tools.simple_state_checker")
+
+rec_err_state_log_stream = StringIO()
+rec_err_state_log_stream_handler = logging.StreamHandler(stream=rec_err_state_log_stream)
+rec_err_state_log_stream_handler.setLevel(logging.ERROR)
+
+logger.addHandler(rec_err_state_log_stream_handler)
 
 base_url = "https://opencast.bibliothek.kit.edu"
 
@@ -46,7 +54,6 @@ def get_service_url(service_type: str):
 def get_calender(rec_id):
     params = {'agentid': rec_id}
     url = get_service_url('org.opencastproject.scheduler') + "/calendars"
-    print(url)
     res = session.get(url, params=params)
     if res.ok:
         return Calendar(res.text)
@@ -80,41 +87,49 @@ def notify_users_of_problem(msg: str):
     pass
 
 
+def get_recorder_adapter(recorder_info:dict) -> RecorderAdapter:
+    if "SMP" in recorder_info["type"]:
+        rec = SMP(recorder_info['ip'], recorder_info['password'])
+    else:
+        rec = EpiphanV1(recorder_info['ip'], recorder_info["username"], recorder_info["password"])
+    return rec
+
+
 def check_capture_agent_state(a: dict):
     logger.debug("Checking Agent {}".format(a['name']))
     c = get_calender(a['name'])
     is_recording_in_calendar = len(list(c.timeline.now())) >= 1
     if is_recording_in_calendar:
+        logger.info("{} has entry in Calender and should therfor be recording... checking now!".format(a['name']))
         if a['state'] == "capturing":
-            print(
+            logger.info(
                 "{} is in capturing state, so there should be an entry in the calendar of the recorder, right? -> {}".format(
                     a['name'], is_recording_in_calendar
                 ))
-            rec = get_recorder_by_name(a['name'])
-            print(rec)
-            if "SMP" in rec["type"]:
-                print("using SMP adapter")
-                rec = SMP(rec['ip'], rec['password'])
-            else:
-                rec = EpiphanV1(rec['ip'], rec["username"], rec["password"])
-            if rec.is_recording():
-                print("OK – recorder is recording :)")
-            else:
-                logger.info(rec.get_recording_status())
-                logger.error("FATAL - recorder {} must be recording!!!!".format(a['name']))
+            recorder_info = get_recorder_by_name(a['name'])
+            try:
+                rec = get_recorder_adapter(recorder_info)
+                if rec.is_recording():
+                    logger.info("OK – recorder {} is recording :)".format(a['name']))
+                else:
+                    logger.info(rec.get_recording_status())
+                    logger.error("FATAL - recorder {} must be recording but is not!!!!".format(a['name']))
+            except LrcException as e:
+                logger.fatal("Exception occurred: {}".format(str(e)))
+                logger.error("Could not check state of recorder {}, Address: {}".format(a['name'], recorder_info['ip']))
         else:
-            print("FATAL: {} is not in capturing state...but should be!!".format(a['name']))
+            logger.error("FATAL: {} is not in capturing state...but should be!!".format(a['name']))
     else:
-        rec = get_recorder_by_name(a['name'])
-        if "SMP" in rec["type"]:
-            logger.debug("using SMP adapter")
-            rec = SMP(rec['ip'], rec['password'])
-        else:
-            rec = EpiphanV1(rec['ip'], rec["username"], rec["password"])
-        if rec.is_recording():
-            logger.error("FATAL - recorder must not be recording!!!!")
-        else:
-            logger.info("OK – recorder is not recording :)")
+        recorder_info = get_recorder_by_name(a['name'])
+        try:
+            rec = get_recorder_adapter(recorder_info)
+            if rec.is_recording():
+                logger.error("FATAL - recorder must not be recording!!!!")
+            else:
+                logger.info("OK – recorder is not recording :)")
+        except LrcException as e:
+            logger.fatal("Exception occurred: {}".format(str(e)))
+            logger.error("Could not check state of recorder {}, Address: {}".format(a['name'], recorder_info['ip']))
 
 
 agents = get_capture_agents()
@@ -123,6 +138,11 @@ logger.info("Got {} capture agents that will be checked...".format(len(agents)))
 pool = ThreadPool(5)
 pool.map(check_capture_agent_state, agents)
 
+pool.close()
+pool.join()
+logged_events = rec_err_state_log_stream.getvalue()
+if len(logged_events) > 0:
+    send_error_mail(logged_events, "Errors have been detected while checking recorder states!")
 exit()
 
 c = get_calender('CS 30.46 Chemie Neuer Hoersaal')