From 7700b4381f3a954d0cbb40b9588f44d6b3fc897c Mon Sep 17 00:00:00 2001 From: Tobias Kurze Date: Wed, 20 Nov 2019 16:19:59 +0100 Subject: [PATCH] now rooms, recorders and recorder models get initialized in DB --- backend/__main__.py | 5 +- backend/models/recorder_model.py | 125 +++++++++++++--------- backend/models/room_model.py | 9 ++ backend/recorder_adapters/epiphan_base.py | 3 +- backend/tools/model_updater.py | 117 ++++++++++++++++---- backend/tools/scrape_rooms.py | 2 +- 6 files changed, 191 insertions(+), 70 deletions(-) diff --git a/backend/__main__.py b/backend/__main__.py index 14135bc..413d465 100644 --- a/backend/__main__.py +++ b/backend/__main__.py @@ -16,6 +16,9 @@ from backend.tools.model_updater import update_recorder_models_database def main(): logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') + db.drop_all() + db.create_all() + print(app.config.get("SERVER_NAME", None)) server_name = app.config.get("SERVER_NAME", None) if server_name is not None and "ubkaps154.ubka.uni-karlsruhe.de" in server_name: @@ -31,8 +34,8 @@ def main(): except Exception as e: logging.critical(e) + room_model.pre_fill_table() - recorder_model.pre_fill_table() update_recorder_models_database() app.run(debug=True, host="0.0.0.0") diff --git a/backend/models/recorder_model.py b/backend/models/recorder_model.py index 69a71b8..7c8cfdd 100644 --- a/backend/models/recorder_model.py +++ b/backend/models/recorder_model.py @@ -3,30 +3,25 @@ Models for lecture recorder """ import importlib +import ipaddress import json import pkgutil import os +import re from sqlalchemy import MetaData from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import validates -from backend import db, app, login_manager -import re +from backend import db, app, login_manager, LrcException from sqlalchemy import or_ from datetime import datetime, timedelta from backend.models.virtual_command_model import virtual_command_recorder_command_table, virtual_command_recorder_table -from backend.recorder_adapters import get_defined_recorder_adapters -from backend.tools.helpers import file_md5 metadata = MetaData() -KNOWN_RECORDERS = {re.compile(r'(SMP)[\s]*([\d]+)[\s]*.?[\s]*([\S]*)'): 'SMP', - re.compile( - r'(LectureRecorder X2|LectureRecorder|VGADVI Recorder|DVI Broadcaster DL|DVIRecorderDL)'): 'Epiphan'} - class RecorderModel(db.Model): __table_args__ = {'extend_existing': True} @@ -40,6 +35,7 @@ class RecorderModel(db.Model): recorders = db.relationship('Recorder', back_populates='recorder_model') checksum = db.Column(db.String(63), unique=True, nullable=False) # checksum of the recorder commands! (see: model_updater.py) + last_checksum_change = db.Column(db.DateTime, nullable=True, default=None) _requires_user = db.Column(db.Integer, default=False, name='requires_user') _requires_password = db.Column(db.Integer, default=True, name='requires_password') @@ -55,6 +51,10 @@ class RecorderModel(db.Model): def get_by_adapter_id(name): return RecorderModel.query.filter(RecorderModel.record_adapter_id == name).first() + @staticmethod + def get_where_adapter_id_contains(adapter_id): + return RecorderModel.query.filter(RecorderModel.record_adapter_id.contains(adapter_id)).first() + @staticmethod def get_by_checksum(md5_sum): return RecorderModel.query.filter(RecorderModel.checksum == md5_sum).first() @@ -75,6 +75,9 @@ class RecorderModel(db.Model): def requires_password(self, val: bool): self._requires_password = 1 if val else 0 + def __str__(self): + return self.model_name + " (record adapter: {})".format(self.record_adapter_id) + class Recorder(db.Model): __table_args__ = {'extend_existing': True} @@ -82,21 +85,27 @@ class Recorder(db.Model): created_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow()) last_time_modified = db.Column(db.DateTime, nullable=False, default=datetime.utcnow()) name = db.Column(db.Unicode(63), unique=True, nullable=False) + model_name = db.Column(db.Unicode(63), unique=False, nullable=False) + serial_number = db.Column(db.Unicode(63), unique=True, nullable=True) locked = db.Column(db.Boolean, default=False) lock_message = db.Column(db.String, nullable=True, default=None) + in_maintenance = db.Column(db.Boolean, default=False) offline = db.Column(db.Boolean, default=False) description = db.Column(db.Unicode(255), unique=False, nullable=True, default="") - room_id = db.Column(db.Integer, db.ForeignKey('room.id')) - room = db.relationship('Room', uselist=False, back_populates='recorder') # one-to-one relation (uselist=False) - ip = db.Column(db.String(15), unique=True, nullable=True, default=None) - configured_options_json_string = db.Column(db.UnicodeText, default='') - firmware_version = db.Column(db.String, nullable=True, default=None) - ip6 = db.Column(db.String(46), unique=True, nullable=True, default=None) + _mac = db.Column(db.String(17), unique=True, nullable=True) + _ip = db.Column(db.String(15), unique=True, nullable=True, default=None) + _ip6 = db.Column(db.String(46), unique=True, nullable=True, default=None) network_name = db.Column(db.String(127), unique=True, nullable=True, default=None) telnet_port = db.Column(db.Integer, unique=False, nullable=False, default=23) ssh_port = db.Column(db.Integer, unique=False, nullable=False, default=22) username = db.Column(db.String, nullable=True, default=None) password = db.Column(db.String, nullable=True, default=None) + _configured_options_json_string = db.Column(db.UnicodeText, default='') + _additional_notes_json_string = db.Column(db.UnicodeText, default='') + additional_camera_connected = db.Column(db.Boolean, default=False) + firmware_version = db.Column(db.String, nullable=True, default=None) + room_id = db.Column(db.Integer, db.ForeignKey('room.id')) + room = db.relationship('Room', uselist=False, back_populates='recorder') # one-to-one relation (uselist=False) recorder_model_id = db.Column(db.Integer, db.ForeignKey('recorder_model.id')) recorder_model = db.relationship('RecorderModel', back_populates='recorders') virtual_commands = db.relationship('VirtualCommand', secondary=virtual_command_recorder_table, @@ -124,14 +133,64 @@ class Recorder(db.Model): @hybrid_property def configured_options(self) -> list: - return json.loads(self.configured_options_json_string) + return json.loads(self._configured_options_json_string) @configured_options.setter def configured_options(self, value: list): - self.configured_options_json_string = json.dumps(value) + self._configured_options_json_string = json.dumps(value) def add_configured_option(self, value: str): - self.configured_options_json_string = json.dumps(self.configured_options.append(value)) + self._configured_options_json_string = json.dumps(self.configured_options.append(value)) + + @hybrid_property + def additional_notes(self) -> list: + return json.loads(self._additional_notes_json_string) + + @additional_notes.setter + def additional_notes(self, value: list): + self._additional_notes_json_string = json.dumps(value) + + def add_additional_notes(self, value: str): + self._additional_notes_json_string = json.dumps(self._additional_notes_json_string.append(value)) + + @hybrid_property + def mac(self) -> str: + return self._mac.upper() + + @mac.setter + def mac(self, value: str): + if value is None or value == '': + return + if re.match("[0-9a-f]{2}([-:]?)[0-9a-f]{2}(\\1[0-9a-f]{2}){4}$", value.lower()): + self._mac = value.replace('-', ':').lower() + else: + raise LrcException("'{}' is not a valid MAC Address!".format(value)) + + @hybrid_property + def ip(self) -> str: + return self._ip + + @ip.setter + def ip(self, value: str): + try: + ipaddress.IPv4Interface(value) + self._ip = value + except (ipaddress.AddressValueError, ipaddress.NetmaskValueError): + raise LrcException("'{}' is not a valid IPv4 Address!".format(value)) + + @hybrid_property + def ip6(self) -> str: + return self._ip6 + + @ip6.setter + def ip6(self, value: str): + try: + ipaddress.IPv6Interface(value) + self._ip6 = value + except (ipaddress.AddressValueError, ipaddress.NetmaskValueError): + raise LrcException("'{}' is not a valid IPv6 Address!".format(value)) + + # handle bad ip def __str__(self): return self.name @@ -173,35 +232,3 @@ class RecorderCommand(db.Model): @parameters.setter def parameters(self, parameters_dict: dict): self.parameters_string = json.dumps(parameters_dict) - - -def pre_create_recorders(): - models = set() - f = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, 'models', 'initial_recorders.json')) - with open(f, 'r') as json_file: - recorders = json.load(json_file)['recorders'] - for r in recorders: - type = r.get('type') - firmware_version = r.get('firmware_version', None) - for k_r in KNOWN_RECORDERS: - if match := k_r.search(type): - print(KNOWN_RECORDERS[k_r]) - print(match) - print(match.groups()) - - models.add(r.get('type', '')) - models.discard('') - print(models) - - -def pre_fill_table(): - try: - pre_create_recorders() - print(db.session.commit()) - except IntegrityError as e: - db.session.rollback() - - -if __name__ == '__main__': - pass - # pre_create_recorder_models() diff --git a/backend/models/room_model.py b/backend/models/room_model.py index 0469eeb..5d5864f 100644 --- a/backend/models/room_model.py +++ b/backend/models/room_model.py @@ -44,6 +44,15 @@ class Room(db.Model): """ return Room.query.filter(Room.name == name).first() + @staticmethod + def get_by_building_number(building_number): + """ + Find group by name + :param name: + :return: + """ + return Room.query.filter(Room.building_number == building_number) + @staticmethod def get_all(): """ diff --git a/backend/recorder_adapters/epiphan_base.py b/backend/recorder_adapters/epiphan_base.py index f3341d9..7db096b 100644 --- a/backend/recorder_adapters/epiphan_base.py +++ b/backend/recorder_adapters/epiphan_base.py @@ -22,7 +22,8 @@ from backend.recorder_adapters import RecorderAdapter # HOST = "localhost" from backend.tools.exception_decorator import exception_decorator -RECORDER_MODEL_NAME = "Epiphan Recorder Adapter (for: )" +RECORDER_MODEL_NAME = "Epiphan Recorder Adapter (for: LectureRecorder X2, LectureRecorder, VGADVI Recorder, " \ + "DVI Broadcaster DL and DVIRecorderDL)" BASE_URL = "http://172.23.8.102" # Audimax SMP 351 diff --git a/backend/tools/model_updater.py b/backend/tools/model_updater.py index 758da0b..e4d5b60 100644 --- a/backend/tools/model_updater.py +++ b/backend/tools/model_updater.py @@ -4,18 +4,27 @@ # Copyright (c) 2019. Tobias Kurze import hashlib +import json import logging +import os +import re from datetime import datetime from json import dumps from pprint import pprint from sqlalchemy import and_ +from sqlalchemy.exc import IntegrityError from backend import db -from backend.models.recorder_model import RecorderModel, RecorderCommand +from backend.models import Room +from backend.models.recorder_model import RecorderModel, RecorderCommand, Recorder from backend.recorder_adapters import get_defined_recorder_adapters from backend.tools.helpers import calculate_md5_checksum +KNOWN_RECORDERS = {re.compile(r'(?PSMP)[\s]*(?P[\d]+)[\s]*.?[\s]*(?P[\S]*)'): 'SMP', + re.compile( + r'(?PLectureRecorder X2|LectureRecorder|VGADVI Recorder|DVI Broadcaster DL|DVIRecorderDL)'): 'Epiphan'} + def create_recorder_commands_for_recorder_adapter(command_definitions: dict, recorder_model: RecorderModel): existing_recorder_commands = RecorderCommand.query.filter( @@ -46,24 +55,96 @@ def create_recorder_commands_for_recorder_adapter(command_definitions: dict, rec def update_recorder_models_database(): r_as = get_defined_recorder_adapters() for r_a in r_as: - r_m = RecorderModel.get_by_adapter_id(r_a["id"]) - model_checksum = calculate_md5_checksum(dumps(r_a["commands"])) - if r_m is None: - r_m = RecorderModel(record_adapter_id=r_a["id"], model_name=r_a["name"], checksum=model_checksum, - requires_user=r_a.get('requires_user', None), - requires_password=r_a.get('requires_password', None)) - db.session.add(r_m) - db.session.flush() - db.session.refresh(r_m) - else: - if not model_checksum == r_m.checksum: - r_m.model_name = r_a["name"] - r_m.last_time_modified = datetime.utcnow() - r_m.checksum = model_checksum - create_recorder_commands_for_recorder_adapter(r_a["commands"], r_m) + try: + r_m = RecorderModel.get_by_adapter_id(r_a["id"]) + model_checksum = calculate_md5_checksum(dumps(r_a["commands"])) + if r_m is None: + r_m = RecorderModel(record_adapter_id=r_a["id"], model_name=r_a["name"], checksum=model_checksum, + **r_a.get('class', {}).get_recorder_params()) + db.session.add(r_m) + db.session.flush() + db.session.refresh(r_m) + else: + if not r_m.model_name == r_a["name"]: + r_m.model_name = r_a["name"] + r_m.last_time_modified = datetime.utcnow() + if not model_checksum == r_m.checksum: + r_m.last_time_modified = datetime.utcnow() + r_m.checksum = model_checksum + r_m.last_checksum_change = datetime.utcnow() + create_recorder_commands_for_recorder_adapter(r_a["commands"], r_m) + except IntegrityError as e: + db.session.rollback() db.session.commit() +def get_recorder_room(rec: dict) -> Room(): + rooms = Room.get_by_building_number(rec.get('building', None)) + if rooms.count() <= 1: + return rooms.first() + room_name = rec.get('room') + for room in rooms: + if all([r_n in room.name for r_n in room_name.split()]): + return room + + +def create_default_recorders(): + models = RecorderModel.get_all() + for m in models: + print(m) + + f = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, 'models', 'initial_recorders.json')) + with open(f, 'r') as json_file: + recorders = json.load(json_file)['recorders'] + for r in recorders: + type = r.get('type') + firmware_version = r.get('firmware_version', None) + room_rec_name = r.get('name') + username = r.get('username', None) + password = r.get('password', None) + mac = r.get('mac', None) + ip4 = r.get('ip', None) + additional_camera = r.get('additional_camera', False) + description = r.get('description', '') + for k_r in KNOWN_RECORDERS: + if match := k_r.search(type): + name = match.groupdict().get('name') + model_number = match.groupdict().get('number', None) + + options = match.groupdict().get('options', None) + + if model_number is not None: + model_name = name + model_number[:-1] # just get prefix (remove last digit) + else: + model_name = KNOWN_RECORDERS[k_r] + rec_model = RecorderModel.get_where_adapter_id_contains(model_name) + + rec = Recorder(name=room_rec_name + " Recorder", model_name=model_name, recorder_model=rec_model, + username=username, password=password, firmware_version=firmware_version) + rec.mac = mac + rec.ip = ip4 + rec.additional_camera_connected = additional_camera + rec.additional_note = description + + rec.room = get_recorder_room(r) + print(rec) + + db.session.add(rec) + db.session.flush() + db.session.refresh(rec) + + print(recorders) + + if __name__ == '__main__': - db.create_all() - update_recorder_models_database() + recorders = Recorder.get_all() + for r in recorders: + print("{}: {}".format(r, r.room)) + # db.drop_all() + # db.create_all() + # update_recorder_models_database() + # create_default_recorders() + # db.session.commit() + + # print(get_recorder_room({"room": "Grosser Hörsaal Bauingenieure", "building": "10.50"})) + # print(get_recorder_room({"room": "Grosser Hörsaal Bauingenieure", "building": "30.95"})) diff --git a/backend/tools/scrape_rooms.py b/backend/tools/scrape_rooms.py index 322dcc3..06a5296 100644 --- a/backend/tools/scrape_rooms.py +++ b/backend/tools/scrape_rooms.py @@ -57,4 +57,4 @@ def scrape_rooms(): if __name__ == '__main__': - scrape_rooms() + pprint(scrape_rooms())