added permissions api and websocket stuff

Pipfile

@@ -31,6 +31,10 @@ ics = "*"
 coloredlogs = "*"
 pythonping = "*"
 scapy = "*"
+python-socketioclient = "*"
+python-socketio = {version = "*",extras = ["client"]}
+socketio-client = "*"
+websocket-client = "*"
 
 [dev-packages]
 

backend/__main__.py

@@ -11,6 +11,7 @@ from backend import app, db
 from backend.models import room_model, recorder_model, RecorderCommand
 from backend.recorder_adapters import get_defined_recorder_adapters
 from backend.tools.model_updater import update_recorder_models_database, create_default_recorders
+from backend.websocket.base import WebSocketBase
 
 
 def main():
@@ -37,7 +38,11 @@ def main():
     except Exception as e:
         logging.critical(e)
 
-    app.run(debug=True, host="0.0.0.0")
+    wsb = WebSocketBase()
+    print("running websocket...(replaces normal app.run()")
+    wsb.start_websocket(debug=True)
+    # print("running web app...")
+    #app.run(debug=True, host="0.0.0.0", threaded=True)
 
 
 if __name__ == '__main__':

backend/api/__init__.py

@@ -27,6 +27,7 @@ api_v1 = Api(api_bp, prefix="/v1", version='0.1', title='Vue Test API',
 
 api_user = Namespace('user', description="User management namespace", authorizations=api_authorizations)
 api_group = Namespace('group', description="Group management namespace", authorizations=api_authorizations)
+api_permissions = Namespace('permissions', description="Permissions management namespace", authorizations=api_authorizations)
 api_room = Namespace('room', description="Room management namespace", authorizations=api_authorizations)
 api_recorder = Namespace('recorder', description="Recorder management namespace", authorizations=api_authorizations)
 api_virtual_command = Namespace('virtual_command', description="Virtual command namespace",
@@ -38,6 +39,7 @@ api_control = Namespace('control', description="Control namespace",
 
 api_v1.add_namespace(api_user)
 api_v1.add_namespace(api_group)
+api_v1.add_namespace(api_permissions)
 api_v1.add_namespace(api_room)
 api_v1.add_namespace(api_recorder)
 api_v1.add_namespace(api_virtual_command)
@@ -58,6 +60,7 @@ auth_api_v1.add_namespace(auth_api_register_ns)
 from .example_api import *
 from .auth_api import *
 from .user_api import *
+from .permission_api import *
 from .group_api import *
 from .room_api import *
 from .recorder_api import *

backend/api/models.py

@@ -20,7 +20,9 @@ user_model = api_user.model('User', {
         fields.Nested(api_user.model('user_group', {'id': fields.Integer(), 'name': fields.String()})),
         required=False, description='Group memberships.'),
     'favorite_recorders': fields.List(
-        fields.Nested(api_user.model('favorite_recorder', {'id': fields.Integer(), 'name': fields.String()})),
+        fields.Nested(api_user.model('favorite_recorder',
+                                     {'id': fields.Integer(), 'name': fields.String(), 'offline': fields.Boolean(),
+                                      'created_at': fields.DateTime(), 'last_time_modified': fields.DateTime()})),
         required=False, description='Favorite recorders.'),
 })
 

backend/api/permission_api.py

@@ -0,0 +1,91 @@
+#  Copyright (c) 2019. Tobias Kurze
+"""
+This module provides functions related to authentication through the API.
+For example: listing of available auth providers or registration of users.
+
+Login through API does not start a new session, but instead returns JWT.
+"""
+from flask_jwt_extended import jwt_required
+from flask_restplus import fields, Resource
+
+from backend import db
+from backend.api import api_permissions
+from backend.models.user_model import Permission
+
+permission_model = api_permissions.model('Permission', {
+    'id': fields.String(required=False, description='The permission\'s identifier'),
+    'name': fields.String(required=True, description='The permission\'s name'),
+    'description': fields.String(required=False, description='The permission\'s description'),
+    'groups': fields.List(fields.Nested(api_permissions.model('group_member',
+                                                              {'id': fields.Integer(),
+                                                               'name': fields.String(),
+                                                               'description': fields.String()})),
+                          required=False, description='Groups having the permission.'),
+    'access_control_entry': fields.Nested(api_permissions.model('group_member',
+                                                                {'id': fields.Integer(),
+                                                                 'name': fields.String(),
+                                                                 'url': fields.String()}),
+                                          required=False, description="Access Control Entry"),
+})
+
+
+@api_permissions.route('/<int:id>')
+@api_permissions.response(404, 'permission not found')
+@api_permissions.param('id', 'The permission identifier')
+class PermissionResource(Resource):
+    @jwt_required
+    @api_permissions.doc('get_permission')
+    @api_permissions.marshal_with(permission_model)
+    def get(self, id):
+        """Fetch a user given its identifier"""
+        permission = Permission.get_by_id(id)
+        if permission is not None:
+            return permission
+        api_permissions.abort(404)
+
+    @jwt_required
+    @api_permissions.doc('delete_permission')
+    @api_permissions.response(204, 'permission deleted')
+    def delete(self, id):
+        """Delete a permission given its identifier"""
+        permission = Permission.get_by_id(id)
+        if permission is not None:
+            permission.delete()
+            return '', 204
+        api_permissions.abort(404)
+
+    @jwt_required
+    @api_permissions.doc('update_permission')
+    @api_permissions.expect(permission_model)
+    @api_permissions.marshal_with(permission_model)
+    def put(self, id):
+        """Update a task given its identifier"""
+        permission = Permission.get_by_id(id)
+        if permission is not None:
+            permission.name = api_permissions["name"]
+            db.session.commit()
+            return permission
+        api_permissions.abort(404)
+
+
+@api_permissions.route('')
+class PermissionList(Resource):
+    @jwt_required
+    @api_permissions.doc('permissions')
+    @api_permissions.marshal_list_with(permission_model)
+    def get(self):
+        """
+        List all permissions
+        :return: permissions
+        """
+        return Permission.get_all()
+
+    @jwt_required
+    @api_permissions.doc('create_permission')
+    @api_permissions.expect(permission_model)
+    @api_permissions.marshal_with(permission_model, code=201)
+    def post(self):
+        permission = Permission(**api_permissions.payload)
+        db.session.add(permission)
+        db.session.commit()
+        return permission

backend/api/user_api.py

@@ -70,7 +70,7 @@ class UserFavoriteRecorders(Resource):
             args = generic_id_parser.parse_args()
             current_user_id = get_jwt_identity()
             user = User.get_by_identifier(current_user_id)
-            print(user)
+            print(args)
             recorder = Recorder.get_by_identifier(args["id"])
             print(recorder)
             if recorder is None:

backend/cron/__init__.py

@@ -1,7 +1,76 @@
 import logging
+from multiprocessing.pool import ThreadPool
+from threading import Lock
+from typing import Union
 
-cron_log_handler = logging.FileHandler(CRON_LOG_FILE)
+from backend import app, LrcException
+from backend.models import Recorder
+from backend.tools.simple_state_checker import check_capture_agent_state, ping_capture_agent
+
+cron_log_handler = logging.FileHandler(app.config.get('CRON_LOG_FILE'))
 cron_logger = logging.getLogger("mal.cron")
 cron_logger.addHandler(cron_log_handler)
 logging.getLogger("apscheduler.scheduler").addHandler(cron_log_handler)
 logging.getLogger("apscheduler.executors.default").addHandler(cron_log_handler)
+
+recorder_jobs_lock = Lock()
+recorder_jobs = set()
+
+NUM_THREADS = 8
+
+
+def add_recorder_to_state_check(recorder: Union[int, Recorder]):
+    if isinstance(recorder, int):
+        recorder = Recorder.get_by_identifier(recorder)
+    if recorder is None:
+        cron_logger.warning(
+            "Could not add recorder to state check, as specified id could not be found / recorder is None")
+        raise LrcException("Recorder is None / could not be found!")
+    recorder_jobs_lock.acquire()
+    recorder_jobs.add(recorder)
+    recorder_jobs_lock.release()
+
+
+def remove_recorder_from_state_check(recorder: Union[int, Recorder]):
+    if isinstance(recorder, int):
+        recorder = Recorder.get_by_identifier(recorder)
+    if recorder is None:
+        cron_logger.warning(
+            "Could not remove recorder from state check, as specified id could not be found / recorder is None")
+        raise LrcException("Recorder is None / could not be found (and therefor not removed)!")
+    recorder_jobs_lock.acquire()
+    recorder_jobs.remove(recorder)
+    recorder_jobs_lock.release()
+
+
+def check_recorder_state():
+    recorder_jobs_lock.acquire()
+    recorders = list(recorder_jobs)
+    recorder_jobs_lock.release()
+
+    recorder_states = {r['name']: {'state_ok': False, 'msg': 'unknown state!'} for r in recorders}
+
+    with ThreadPool(NUM_THREADS) as pool:
+        results = [pool.apply_async(check_capture_agent_state, (recorder,)) for recorder in recorders]
+        try:
+            state_results = [res.get(timeout=12) for res in results]
+        except TimeoutError as e:
+            cron_logger.error("Timeout while getting capture agent state! {}".format(e))
+
+        for r in state_results:
+            if r[0]:  # ok :)
+                recorder_states[r[2]] = {'state_ok': True}
+            else:
+                recorder_states[r[2]]['msg'] = r[1]
+
+    with ThreadPool(NUM_THREADS) as pool:
+        results = [pool.apply_async(ping_capture_agent, (recorder,)) for recorder in recorders]
+        try:
+            ping_results = [res.get(timeout=12) for res in results]
+        except TimeoutError as e:
+            cron_logger.error("Timeout while pinging capture agent! {}".format(e))
+
+        for r in ping_results:
+            if not r[0]:  # ok :)
+                recorder_states[r[2]]['msg'] = r[1]
+

backend/models/user_model.py

@@ -486,6 +486,22 @@ class Permission(db.Model):
                              back_populates='permissions')
     access_control_entry = db.relationship('AccessControlEntry', back_populates='required_permission')
 
+    @staticmethod
+    def get_by_name(name):
+        """
+        Find permission by name
+        :param name:
+        :return:
+        """
+        return Permission.query.filter(Permission.name == name).first()
+
+    @staticmethod
+    def get_all():
+        """
+        Return all permissions
+        :return:
+        """
+        return Permission.query.all()
 
 @event.listens_for(User.__table__, 'after_create')
 def insert_initial_users(*args, **kwargs):

backend/tools/simple_state_checker.py

@@ -106,6 +106,7 @@ def get_recorder_adapter(recorder_info: dict) -> RecorderAdapter:
 
 
 def check_capture_agent_state(a: dict):
+    agent_state_error_msg = None
     logger.debug("Checking Agent {}".format(a['name']))
     c = get_calender(a['name'])
     is_recording_in_calendar = len(list(c.timeline.now())) >= 1
@@ -122,6 +123,7 @@ def check_capture_agent_state(a: dict):
                 else:
                     logger.info(rec.get_recording_status())
                     logger.error("FATAL - recorder {} must be recording but is not!!!!".format(a['name']))
+                    agent_state_error_msg = "FATAL - recorder must be recording but is not!"
                     with agent_states_lock:
                         agent_states[a['name']] = 'FATAL - recorder is NOT recording, but should!'
             except LrcException as e:
@@ -129,12 +131,14 @@ def check_capture_agent_state(a: dict):
                 logger.error("Could not check state of recorder {}, Address: {}".format(a['name'], recorder_info['ip']))
         else:
             logger.error("FATAL: {} is not in capturing state...but should be!!".format(a['name']))
+            agent_state_error_msg = "FATAL - is not in capturing state...but should be!"
     else:
         recorder_info = get_recorder_by_name(a['name'])
         try:
             rec = get_recorder_adapter(recorder_info)
             if rec.is_recording():
                 logger.error("FATAL - recorder must not be recording!!!!")
+                agent_state_error_msg = "FATAL - is not in capturing state...but should be!"
                 with agent_states_lock:
                     agent_states[a['name']] = 'FATAL - recorder IS recording, but should NOT!'
             else:
@@ -144,6 +148,11 @@ def check_capture_agent_state(a: dict):
         except LrcException as e:
             logger.fatal("Exception occurred: {}".format(str(e)))
             logger.error("Could not check state of recorder {}, Address: {}".format(a['name'], recorder_info['ip']))
+            agent_state_error_msg = "FATAL - Could not check state of recorder! Address: {}".format(recorder_info['ip'])
+
+    if agent_state_error_msg is None:
+        return True, "", a['name']
+    return False, agent_state_error_msg, a['name']
 
 
 def ping_capture_agent(a: dict):
@@ -157,8 +166,10 @@ def ping_capture_agent(a: dict):
             universal_newlines=True  # return string not bytes
         )
         logger.info("Successfully pinged {} ({}). :-)".format(a['name'], recorder_ip))
+        return True, "", a['name']
     except subprocess.CalledProcessError:
         logger.error("Can not ping {} ({})!!".format(a['name'], recorder_ip))
+        return False, "Unable to ping", a['name']
 
 
 agents = get_capture_agents()

backend/websocket/base.py

@@ -1,6 +1,7 @@
 import logging
 import threading
 
+from flask_jwt_extended import verify_jwt_in_request, get_current_user, jwt_required, get_jwt_claims, get_jwt_identity
 from flask_login import current_user
 from flask_socketio import SocketIO, emit
 
@@ -35,17 +36,16 @@ class WebSocketBase:
     @socketio.on('connect')
     def connect_handler():
         logger.debug("new connection...")
-        print(current_user)
+        try:
-        if current_user.is_authenticated:
+            print(verify_jwt_in_request())
-            logger.debug("user is authenticated")
+            print(get_jwt_identity())
-            print("allowed!")
+        except:
-            emit('my response',
-                 {'message': '{0} has joined'.format(current_user.name)},
-                 broadcast=True)
-        else:
             logger.info("user is not authenticated!")
             print("not allowed!!")
             return False  # not allowed here
+        logger.debug("user is authenticated")
+        print("allowed!")
+        return True
 
     @socketio.on_error()
     def handle_error(self, error):

backend/websocket/websocket_test_client.py

@@ -3,19 +3,23 @@
 import logging
 import threading
 import time
-
+from socketIO_client import SocketIO, LoggingNamespace
-from flask import Flask
-from flask_socketio import SocketIO, emit
-
-from backend import app
-
-app = Flask(__name__)
-app.config['SECRET_KEY'] = 'secret!'
 
 logging.basicConfig()
 
+
+token = "# replace with: JWT Access Token"
+#print(token)
+
+print("params")
+#socketIO = SocketIO('127.0.0.1', 5443, params={'jwt': '{}'.format(token)})
+print("headers")
+#socketIO = SocketIO('127.0.0.1', 5443, headers={'Authorization': 'Bearer {}'.format(token)})
+print("cookies")
+socketIO = SocketIO('127.0.0.1', 5443, cookies={'access_token_cookie': '{}'.format(token)})
+
 #socketio = SocketIO(message_queue="redis://")
-socketio = SocketIO(app, port=5443, debug=True)
+socketio = SocketIO('127.0.0.1', 5443)
 
 #socketio.run(app, host="localhost", port=5000)
 #socketio.init_app(app, host="localhost", port=5000, cors_allowed_origins="*", )