added code to initialize db with models and recodres

backend/tools/simple_state_checker.py

@@ -1,32 +1,38 @@
 import json
 import os
 import logging
+import subprocess
+import threading
 from io import StringIO
+from logging.handlers import MemoryHandler
 
 import requests
 from requests.auth import HTTPBasicAuth
 from multiprocessing.pool import ThreadPool
 from multiprocessing.context import TimeoutError
 
-from pprint import pprint
-
 from ics import Calendar
 
 from backend import LrcException
 from backend.config import Config
 from backend.recorder_adapters import RecorderAdapter
 
-from backend.recorder_adapters.epiphan_base import EpiphanV1
-from backend.recorder_adapters.extron_smp import SMP
-from backend.tools.send_mail import send_error_mail
+from backend.recorder_adapters.epiphan_base import Epiphan
+from backend.recorder_adapters.extron_smp import SMP35x
+from backend.tools.send_mail import send_error_mail, get_smtp_error_handler
 
 logger = logging.getLogger("lrc.tools.simple_state_checker")
 
+smtp_error_handler = get_smtp_error_handler(subject="Errors have been detected while checking recorder states!")
+#mem_handler = MemoryHandler(capacity=100, flushLevel=logging.FATAL, target=smtp_error_handler)
+#mem_handler.setLevel(logging.WARNING)
+
 rec_err_state_log_stream = StringIO()
 rec_err_state_log_stream_handler = logging.StreamHandler(stream=rec_err_state_log_stream)
 rec_err_state_log_stream_handler.setLevel(logging.WARNING)
 
 logger.addHandler(rec_err_state_log_stream_handler)
+#logger.addHandler(mem_handler)
 
 base_url = "https://opencast.bibliothek.kit.edu"
 
@@ -37,6 +43,9 @@ config = {'service_urls': {}}
 
 recorders = None
 
+agent_states_lock = threading.RLock()
+agent_states = {}
+
 
 def get_service_url(service_type: str):
     if service_type in config['service_urls']:
@@ -90,9 +99,9 @@ def notify_users_of_problem(msg: str):
 
 def get_recorder_adapter(recorder_info: dict) -> RecorderAdapter:
     if "SMP" in recorder_info["type"]:
-        rec = SMP(recorder_info['ip'], recorder_info['password'])
+        rec = SMP35x(recorder_info['ip'], recorder_info['password'])
     else:
-        rec = EpiphanV1(recorder_info['ip'], recorder_info["username"], recorder_info["password"])
+        rec = Epiphan(recorder_info['ip'], recorder_info["username"], recorder_info["password"])
     return rec
 
 
@@ -108,9 +117,13 @@ def check_capture_agent_state(a: dict):
                 rec = get_recorder_adapter(recorder_info)
                 if rec.is_recording():
                     logger.info("OK – recorder {} is recording :)".format(a['name']))
+                    with agent_states_lock:
+                        agent_states[a['name']] = 'OK - recorder is recording'
                 else:
                     logger.info(rec.get_recording_status())
                     logger.error("FATAL - recorder {} must be recording but is not!!!!".format(a['name']))
+                    with agent_states_lock:
+                        agent_states[a['name']] = 'FATAL - recorder is NOT recording, but should!'
             except LrcException as e:
                 logger.fatal("Exception occurred: {}".format(str(e)))
                 logger.error("Could not check state of recorder {}, Address: {}".format(a['name'], recorder_info['ip']))
@@ -122,20 +135,51 @@ def check_capture_agent_state(a: dict):
             rec = get_recorder_adapter(recorder_info)
             if rec.is_recording():
                 logger.error("FATAL - recorder must not be recording!!!!")
+                with agent_states_lock:
+                    agent_states[a['name']] = 'FATAL - recorder IS recording, but should NOT!'
             else:
                 logger.info("OK – recorder is not recording :)")
+                with agent_states_lock:
+                    agent_states[a['name']] = 'OK - recorder is NOT recording'
         except LrcException as e:
             logger.fatal("Exception occurred: {}".format(str(e)))
             logger.error("Could not check state of recorder {}, Address: {}".format(a['name'], recorder_info['ip']))
 
 
+def ping_capture_agent(a: dict):
+    recorder_ip = get_recorder_by_name(a['name'])['ip']
+    try:
+        response = subprocess.check_call(
+            ['ping', '-W', '10', '-c', '2', recorder_ip],
+            # stderr=subprocess.STDOUT,  # get all output
+            stdout=subprocess.DEVNULL,  # suppress output
+            stderr=subprocess.DEVNULL,
+            universal_newlines=True  # return string not bytes
+        )
+        logger.info("Successfully pinged {} ({}). :-)".format(a['name'], recorder_ip))
+    except subprocess.CalledProcessError:
+        logger.error("Can not ping {} ({})!!".format(a['name'], recorder_ip))
+
+
 agents = get_capture_agents()
 logger.info("Got {} capture agents that will be checked...".format(len(agents)))
 
+for a in agents:
+    agent_states[a['name']] = 'PROBLEMATIC - unknown'
+
+
 # pool = ThreadPool(5)
 # pool.map(check_capture_agent_state, agents)
 
-NUM_THREADS = 5
+NUM_THREADS = 8
+
+with ThreadPool(NUM_THREADS) as pool:
+    results = [pool.apply_async(ping_capture_agent, (agent,)) for agent in agents]
+    try:
+        [res.get(timeout=12) for res in results]
+    except TimeoutError as e:
+        logger.error("Timeout while pinging capture agent! {}".format(e))
+
 
 with ThreadPool(NUM_THREADS) as pool:
     results = [pool.apply_async(check_capture_agent_state, (agent,)) for agent in agents]
@@ -148,4 +192,9 @@ logger.info("DONE checking capture agents / recorders!")
 
 logged_events = rec_err_state_log_stream.getvalue()
 if len(logged_events) > 0:
+    logged_events += "\n\n=============\nAgent States:\n\n{}".format(''.join(
+        "{:<48}: {}\n".format(a, agent_states[a]) for a in agent_states
+    ))
     send_error_mail(logged_events, "Errors have been detected while checking recorder states!")
+
+#mem_handler.close()