diff --git a/Pipfile b/Pipfile index f771728..5381d84 100644 --- a/Pipfile +++ b/Pipfile @@ -31,6 +31,10 @@ ics = "*" coloredlogs = "*" pythonping = "*" scapy = "*" +python-socketioclient = "*" +python-socketio = {version = "*",extras = ["client"]} +socketio-client = "*" +websocket-client = "*" [dev-packages] diff --git a/backend/__main__.py b/backend/__main__.py index 95e5a1e..761b776 100644 --- a/backend/__main__.py +++ b/backend/__main__.py @@ -11,6 +11,7 @@ from backend import app, db from backend.models import room_model, recorder_model, RecorderCommand from backend.recorder_adapters import get_defined_recorder_adapters from backend.tools.model_updater import update_recorder_models_database, create_default_recorders +from backend.websocket.base import WebSocketBase def main(): @@ -37,7 +38,11 @@ def main(): except Exception as e: logging.critical(e) - app.run(debug=True, host="0.0.0.0") + wsb = WebSocketBase() + print("running websocket...(replaces normal app.run()") + wsb.start_websocket(debug=True) + # print("running web app...") + #app.run(debug=True, host="0.0.0.0", threaded=True) if __name__ == '__main__': diff --git a/backend/api/__init__.py b/backend/api/__init__.py index 4c441e1..fe4a2d1 100644 --- a/backend/api/__init__.py +++ b/backend/api/__init__.py @@ -27,6 +27,7 @@ api_v1 = Api(api_bp, prefix="/v1", version='0.1', title='Vue Test API', api_user = Namespace('user', description="User management namespace", authorizations=api_authorizations) api_group = Namespace('group', description="Group management namespace", authorizations=api_authorizations) +api_permissions = Namespace('permissions', description="Permissions management namespace", authorizations=api_authorizations) api_room = Namespace('room', description="Room management namespace", authorizations=api_authorizations) api_recorder = Namespace('recorder', description="Recorder management namespace", authorizations=api_authorizations) api_virtual_command = Namespace('virtual_command', description="Virtual command namespace", @@ -38,6 +39,7 @@ api_control = Namespace('control', description="Control namespace", api_v1.add_namespace(api_user) api_v1.add_namespace(api_group) +api_v1.add_namespace(api_permissions) api_v1.add_namespace(api_room) api_v1.add_namespace(api_recorder) api_v1.add_namespace(api_virtual_command) @@ -58,6 +60,7 @@ auth_api_v1.add_namespace(auth_api_register_ns) from .example_api import * from .auth_api import * from .user_api import * +from .permission_api import * from .group_api import * from .room_api import * from .recorder_api import * diff --git a/backend/api/models.py b/backend/api/models.py index 2e89c12..0c9fb97 100644 --- a/backend/api/models.py +++ b/backend/api/models.py @@ -20,7 +20,9 @@ user_model = api_user.model('User', { fields.Nested(api_user.model('user_group', {'id': fields.Integer(), 'name': fields.String()})), required=False, description='Group memberships.'), 'favorite_recorders': fields.List( - fields.Nested(api_user.model('favorite_recorder', {'id': fields.Integer(), 'name': fields.String()})), + fields.Nested(api_user.model('favorite_recorder', + {'id': fields.Integer(), 'name': fields.String(), 'offline': fields.Boolean(), + 'created_at': fields.DateTime(), 'last_time_modified': fields.DateTime()})), required=False, description='Favorite recorders.'), }) diff --git a/backend/api/permission_api.py b/backend/api/permission_api.py new file mode 100644 index 0000000..941a399 --- /dev/null +++ b/backend/api/permission_api.py @@ -0,0 +1,91 @@ +# Copyright (c) 2019. Tobias Kurze +""" +This module provides functions related to authentication through the API. +For example: listing of available auth providers or registration of users. + +Login through API does not start a new session, but instead returns JWT. +""" +from flask_jwt_extended import jwt_required +from flask_restplus import fields, Resource + +from backend import db +from backend.api import api_permissions +from backend.models.user_model import Permission + +permission_model = api_permissions.model('Permission', { + 'id': fields.String(required=False, description='The permission\'s identifier'), + 'name': fields.String(required=True, description='The permission\'s name'), + 'description': fields.String(required=False, description='The permission\'s description'), + 'groups': fields.List(fields.Nested(api_permissions.model('group_member', + {'id': fields.Integer(), + 'name': fields.String(), + 'description': fields.String()})), + required=False, description='Groups having the permission.'), + 'access_control_entry': fields.Nested(api_permissions.model('group_member', + {'id': fields.Integer(), + 'name': fields.String(), + 'url': fields.String()}), + required=False, description="Access Control Entry"), +}) + + +@api_permissions.route('/') +@api_permissions.response(404, 'permission not found') +@api_permissions.param('id', 'The permission identifier') +class PermissionResource(Resource): + @jwt_required + @api_permissions.doc('get_permission') + @api_permissions.marshal_with(permission_model) + def get(self, id): + """Fetch a user given its identifier""" + permission = Permission.get_by_id(id) + if permission is not None: + return permission + api_permissions.abort(404) + + @jwt_required + @api_permissions.doc('delete_permission') + @api_permissions.response(204, 'permission deleted') + def delete(self, id): + """Delete a permission given its identifier""" + permission = Permission.get_by_id(id) + if permission is not None: + permission.delete() + return '', 204 + api_permissions.abort(404) + + @jwt_required + @api_permissions.doc('update_permission') + @api_permissions.expect(permission_model) + @api_permissions.marshal_with(permission_model) + def put(self, id): + """Update a task given its identifier""" + permission = Permission.get_by_id(id) + if permission is not None: + permission.name = api_permissions["name"] + db.session.commit() + return permission + api_permissions.abort(404) + + +@api_permissions.route('') +class PermissionList(Resource): + @jwt_required + @api_permissions.doc('permissions') + @api_permissions.marshal_list_with(permission_model) + def get(self): + """ + List all permissions + :return: permissions + """ + return Permission.get_all() + + @jwt_required + @api_permissions.doc('create_permission') + @api_permissions.expect(permission_model) + @api_permissions.marshal_with(permission_model, code=201) + def post(self): + permission = Permission(**api_permissions.payload) + db.session.add(permission) + db.session.commit() + return permission diff --git a/backend/api/user_api.py b/backend/api/user_api.py index af2834b..ec2a868 100644 --- a/backend/api/user_api.py +++ b/backend/api/user_api.py @@ -70,7 +70,7 @@ class UserFavoriteRecorders(Resource): args = generic_id_parser.parse_args() current_user_id = get_jwt_identity() user = User.get_by_identifier(current_user_id) - print(user) + print(args) recorder = Recorder.get_by_identifier(args["id"]) print(recorder) if recorder is None: diff --git a/backend/config.py b/backend/config.py index 1286c24..e642601 100644 Binary files a/backend/config.py and b/backend/config.py differ diff --git a/backend/cron/__init__.py b/backend/cron/__init__.py index 2609fec..6baa80e 100644 --- a/backend/cron/__init__.py +++ b/backend/cron/__init__.py @@ -1,7 +1,76 @@ import logging +from multiprocessing.pool import ThreadPool +from threading import Lock +from typing import Union -cron_log_handler = logging.FileHandler(CRON_LOG_FILE) +from backend import app, LrcException +from backend.models import Recorder +from backend.tools.simple_state_checker import check_capture_agent_state, ping_capture_agent + +cron_log_handler = logging.FileHandler(app.config.get('CRON_LOG_FILE')) cron_logger = logging.getLogger("mal.cron") cron_logger.addHandler(cron_log_handler) logging.getLogger("apscheduler.scheduler").addHandler(cron_log_handler) -logging.getLogger("apscheduler.executors.default").addHandler(cron_log_handler) \ No newline at end of file +logging.getLogger("apscheduler.executors.default").addHandler(cron_log_handler) + +recorder_jobs_lock = Lock() +recorder_jobs = set() + +NUM_THREADS = 8 + + +def add_recorder_to_state_check(recorder: Union[int, Recorder]): + if isinstance(recorder, int): + recorder = Recorder.get_by_identifier(recorder) + if recorder is None: + cron_logger.warning( + "Could not add recorder to state check, as specified id could not be found / recorder is None") + raise LrcException("Recorder is None / could not be found!") + recorder_jobs_lock.acquire() + recorder_jobs.add(recorder) + recorder_jobs_lock.release() + + +def remove_recorder_from_state_check(recorder: Union[int, Recorder]): + if isinstance(recorder, int): + recorder = Recorder.get_by_identifier(recorder) + if recorder is None: + cron_logger.warning( + "Could not remove recorder from state check, as specified id could not be found / recorder is None") + raise LrcException("Recorder is None / could not be found (and therefor not removed)!") + recorder_jobs_lock.acquire() + recorder_jobs.remove(recorder) + recorder_jobs_lock.release() + + +def check_recorder_state(): + recorder_jobs_lock.acquire() + recorders = list(recorder_jobs) + recorder_jobs_lock.release() + + recorder_states = {r['name']: {'state_ok': False, 'msg': 'unknown state!'} for r in recorders} + + with ThreadPool(NUM_THREADS) as pool: + results = [pool.apply_async(check_capture_agent_state, (recorder,)) for recorder in recorders] + try: + state_results = [res.get(timeout=12) for res in results] + except TimeoutError as e: + cron_logger.error("Timeout while getting capture agent state! {}".format(e)) + + for r in state_results: + if r[0]: # ok :) + recorder_states[r[2]] = {'state_ok': True} + else: + recorder_states[r[2]]['msg'] = r[1] + + with ThreadPool(NUM_THREADS) as pool: + results = [pool.apply_async(ping_capture_agent, (recorder,)) for recorder in recorders] + try: + ping_results = [res.get(timeout=12) for res in results] + except TimeoutError as e: + cron_logger.error("Timeout while pinging capture agent! {}".format(e)) + + for r in ping_results: + if not r[0]: # ok :) + recorder_states[r[2]]['msg'] = r[1] + diff --git a/backend/models/user_model.py b/backend/models/user_model.py index 0027c9b..f6b0dc7 100644 --- a/backend/models/user_model.py +++ b/backend/models/user_model.py @@ -486,6 +486,22 @@ class Permission(db.Model): back_populates='permissions') access_control_entry = db.relationship('AccessControlEntry', back_populates='required_permission') + @staticmethod + def get_by_name(name): + """ + Find permission by name + :param name: + :return: + """ + return Permission.query.filter(Permission.name == name).first() + + @staticmethod + def get_all(): + """ + Return all permissions + :return: + """ + return Permission.query.all() @event.listens_for(User.__table__, 'after_create') def insert_initial_users(*args, **kwargs): diff --git a/backend/tools/simple_state_checker.py b/backend/tools/simple_state_checker.py index 3dd9e00..c126d99 100644 --- a/backend/tools/simple_state_checker.py +++ b/backend/tools/simple_state_checker.py @@ -106,6 +106,7 @@ def get_recorder_adapter(recorder_info: dict) -> RecorderAdapter: def check_capture_agent_state(a: dict): + agent_state_error_msg = None logger.debug("Checking Agent {}".format(a['name'])) c = get_calender(a['name']) is_recording_in_calendar = len(list(c.timeline.now())) >= 1 @@ -122,6 +123,7 @@ def check_capture_agent_state(a: dict): else: logger.info(rec.get_recording_status()) logger.error("FATAL - recorder {} must be recording but is not!!!!".format(a['name'])) + agent_state_error_msg = "FATAL - recorder must be recording but is not!" with agent_states_lock: agent_states[a['name']] = 'FATAL - recorder is NOT recording, but should!' except LrcException as e: @@ -129,12 +131,14 @@ def check_capture_agent_state(a: dict): logger.error("Could not check state of recorder {}, Address: {}".format(a['name'], recorder_info['ip'])) else: logger.error("FATAL: {} is not in capturing state...but should be!!".format(a['name'])) + agent_state_error_msg = "FATAL - is not in capturing state...but should be!" else: recorder_info = get_recorder_by_name(a['name']) try: rec = get_recorder_adapter(recorder_info) if rec.is_recording(): logger.error("FATAL - recorder must not be recording!!!!") + agent_state_error_msg = "FATAL - is not in capturing state...but should be!" with agent_states_lock: agent_states[a['name']] = 'FATAL - recorder IS recording, but should NOT!' else: @@ -144,6 +148,11 @@ def check_capture_agent_state(a: dict): except LrcException as e: logger.fatal("Exception occurred: {}".format(str(e))) logger.error("Could not check state of recorder {}, Address: {}".format(a['name'], recorder_info['ip'])) + agent_state_error_msg = "FATAL - Could not check state of recorder! Address: {}".format(recorder_info['ip']) + + if agent_state_error_msg is None: + return True, "", a['name'] + return False, agent_state_error_msg, a['name'] def ping_capture_agent(a: dict): @@ -157,8 +166,10 @@ def ping_capture_agent(a: dict): universal_newlines=True # return string not bytes ) logger.info("Successfully pinged {} ({}). :-)".format(a['name'], recorder_ip)) + return True, "", a['name'] except subprocess.CalledProcessError: logger.error("Can not ping {} ({})!!".format(a['name'], recorder_ip)) + return False, "Unable to ping", a['name'] agents = get_capture_agents() diff --git a/backend/websocket/base.py b/backend/websocket/base.py index 53cdd5e..51d2671 100644 --- a/backend/websocket/base.py +++ b/backend/websocket/base.py @@ -1,6 +1,7 @@ import logging import threading +from flask_jwt_extended import verify_jwt_in_request, get_current_user, jwt_required, get_jwt_claims, get_jwt_identity from flask_login import current_user from flask_socketio import SocketIO, emit @@ -35,17 +36,16 @@ class WebSocketBase: @socketio.on('connect') def connect_handler(): logger.debug("new connection...") - print(current_user) - if current_user.is_authenticated: - logger.debug("user is authenticated") - print("allowed!") - emit('my response', - {'message': '{0} has joined'.format(current_user.name)}, - broadcast=True) - else: + try: + print(verify_jwt_in_request()) + print(get_jwt_identity()) + except: logger.info("user is not authenticated!") print("not allowed!!") return False # not allowed here + logger.debug("user is authenticated") + print("allowed!") + return True @socketio.on_error() def handle_error(self, error): @@ -55,5 +55,5 @@ class WebSocketBase: if __name__ == '__main__': wsb = WebSocketBase() - #wsb.start_websocket_in_thread(debug=True) + # wsb.start_websocket_in_thread(debug=True) wsb.start_websocket(debug=True) diff --git a/backend/websocket/handlers.py b/backend/websocket/handlers.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/websocket/websocket_base.py b/backend/websocket/websocket_base_testing.py similarity index 100% rename from backend/websocket/websocket_base.py rename to backend/websocket/websocket_base_testing.py diff --git a/backend/websocket/websocket_test_client.py b/backend/websocket/websocket_test_client.py index ea62a9c..ece025b 100644 --- a/backend/websocket/websocket_test_client.py +++ b/backend/websocket/websocket_test_client.py @@ -3,19 +3,23 @@ import logging import threading import time - -from flask import Flask -from flask_socketio import SocketIO, emit - -from backend import app - -app = Flask(__name__) -app.config['SECRET_KEY'] = 'secret!' +from socketIO_client import SocketIO, LoggingNamespace logging.basicConfig() + +token = "# replace with: JWT Access Token" +#print(token) + +print("params") +#socketIO = SocketIO('127.0.0.1', 5443, params={'jwt': '{}'.format(token)}) +print("headers") +#socketIO = SocketIO('127.0.0.1', 5443, headers={'Authorization': 'Bearer {}'.format(token)}) +print("cookies") +socketIO = SocketIO('127.0.0.1', 5443, cookies={'access_token_cookie': '{}'.format(token)}) + #socketio = SocketIO(message_queue="redis://") -socketio = SocketIO(app, port=5443, debug=True) +socketio = SocketIO('127.0.0.1', 5443) #socketio.run(app, host="localhost", port=5000) #socketio.init_app(app, host="localhost", port=5000, cors_allowed_origins="*", ) @@ -24,4 +28,4 @@ socketio = SocketIO(app, port=5443, debug=True) socketio.emit("server_event", {'data': 42, 'msg': 'toll'}) print("sent message!") socketio.emit({'data': 42, 'msg': 'toll'}) -print("sent message 2!") \ No newline at end of file +print("sent message 2!")