Merge pull request #928 from dfir-iris/tidy_imports

Tidy imports

Author: whikernel

.github/workflows/ci.yml

@@ -31,6 +31,12 @@ jobs:
         uses: astral-sh/ruff-action@v3
         with:
           args: check --output-format=github
+      - name: Check dependencies with import-linter
+        run: |
+          python -m venv venv
+          source venv/bin/activate
+          pip install import-linter
+          PYTHONPATH=source lint-imports
 
   build-docker-db:
     name: Build docker db

pyproject.toml

@@ -1,5 +1,30 @@
 [tool.ruff.lint]
 preview = true
-select = ["E101", "E225", "E23", "E24", "E3", "E4", "E7", "E9", "F", "RET506", "UP032", "W29"]
+select = ["E101", "E225", "E23", "E24", "E3", "E4", "E7", "E9", "F", "PLR0402", "RET506", "UP032", "W29"]
 ignore = ["E402", "E711", "E712", "E721", "E722"]
 
+[tool.importlinter]
+root_package = "app"
+include_external_packages = true
+
+[[tool.importlinter.contracts]]
+name = "Do not import the persistence layer from the API layer"
+type = "forbidden"
+source_modules = "app.blueprints.rest.v2.alerts"
+forbidden_modules = "app.datamgmt.alerts"
+allow_indirect_imports = true
+
+[[tool.importlinter.contracts]]
+name = "Do not import sqlalchemy from the API layer"
+type = "forbidden"
+source_modules = ["app.blueprints.rest.search_routes", "app.blueprints.rest.dim_tasks_routes", "app.blueprints.rest.case.case_notes_routes", "app.blueprints.rest.case.case_routes"]
+forbidden_modules = "sqlalchemy"
+allow_indirect_imports = true
+
+[[tool.importlinter.contracts]]
+name = "Do not import sqlalchemy from the business layer"
+type = "forbidden"
+source_modules = "app.business"
+forbidden_modules = "sqlalchemy"
+allow_indirect_imports = true
+

source/app/blueprints/pages/dim_tasks/dim_tasks.py

@@ -26,7 +26,7 @@
 from app.models.authorization import Permissions
 from app.blueprints.access_controls import ac_case_requires, ac_requires
 from app.blueprints.responses import response_error
-from app.business.dim_tasks import dim_tasks_get
+from app.business.asynchronous_tasks import dim_tasks_get
 
 dim_tasks_blueprint = Blueprint(
     'dim_tasks',

source/app/blueprints/rest/case/case_notes_routes.py

@@ -20,8 +20,6 @@
 from datetime import datetime
 from flask import Blueprint
 from flask import request
-from sqlalchemy import or_
-from sqlalchemy import and_
 
 from app import db
 from app import app
@@ -48,7 +46,7 @@
 from app.datamgmt.states import get_notes_state
 from app.iris_engine.module_handler.module_handler import call_modules_hook
 from app.iris_engine.utils.tracker import track_activity
-from app.models.models import Notes
+from app.business.notes import notes_search
 from app.models.authorization import CaseAccessLevel
 from app.schema.marshables import CaseNoteDirectorySchema
 from app.schema.marshables import CaseNoteRevisionSchema
@@ -335,11 +333,7 @@ def case_notes_state(caseid):
 def case_search_notes(caseid):
     search_input = request.args.get('search_input')
 
-    notes = Notes.query.filter(
-        and_(Notes.note_case_id == caseid,
-             or_(Notes.note_title.ilike(f'%{search_input}%'),
-                 Notes.note_content.ilike(f'%{search_input}%')))
-    ).all()
+    notes = notes_search(caseid, search_input)
 
     note_schema = CaseNoteSchema(many=True)
     serialized_notes = note_schema.dump(notes)

source/app/blueprints/rest/case/case_routes.py

@@ -22,8 +22,6 @@
 import traceback
 from flask import Blueprint
 from flask import request
-from sqlalchemy import and_
-from sqlalchemy import desc
 
 from app import app
 from app import db
@@ -39,14 +37,13 @@
 from app.datamgmt.manage.manage_users_db import get_user
 from app.datamgmt.manage.manage_users_db import get_users_list_restricted_from_case
 from app.business.access_controls import set_user_case_access, ac_fast_check_user_has_case_access
+from app.business.activity import activity_search_in_case
 from app.business.cases import cases_export_to_json
 from app.iris_engine.access_control.utils import ac_set_case_access_for_users
 from app.iris_engine.utils.tracker import track_activity
 from app.models.models import CaseStatus
 from app.models.models import ReviewStatusList
-from app.models.models import UserActivity
 from app.models.authorization import CaseAccessLevel
-from app.models.authorization import User
 from app.schema.marshables import TaskLogSchema
 from app.schema.marshables import CaseSchema
 from app.schema.marshables import CaseDetailsSchema
@@ -111,21 +108,7 @@ def summary_fetch(caseid):
 @ac_requires_case_identifier(CaseAccessLevel.read_only, CaseAccessLevel.full_access)
 @ac_api_requires()
 def activity_fetch(caseid):
-    ua = UserActivity.query.with_entities(
-        UserActivity.activity_date,
-        User.name,
-        UserActivity.activity_desc,
-        UserActivity.is_from_api
-    ).filter(and_(
-        UserActivity.case_id == caseid,
-        UserActivity.display_in_ui == True
-    )).join(
-        UserActivity.user
-    ).order_by(
-        desc(UserActivity.activity_date)
-    ).limit(40).all()
-
-    output = [a._asdict() for a in ua]
+    output = activity_search_in_case(caseid)
 
     return response_success('', data=output)
 

source/app/blueprints/rest/dim_tasks_routes.py

@@ -18,12 +18,8 @@
 
 from flask import Blueprint
 from flask import request
-import json
-import pickle
-from sqlalchemy import desc
 
 from app.iris_engine.module_handler.module_handler import call_modules_hook
-from app.models.models import CeleryTaskMeta
 from app.models.models import IrisHook
 from app.models.models import IrisModule
 from app.models.models import IrisModuleHook
@@ -41,7 +37,7 @@
 from app.blueprints.access_controls import ac_api_requires
 from app.blueprints.responses import response_error
 from app.blueprints.responses import response_success
-from iris_interface.IrisInterfaceStatus import IIStatus
+from app.business.asynchronous_tasks import asynchronous_tasks_search
 
 dim_tasks_rest_blueprint = Blueprint('dim_tasks_rest', __name__)
 
@@ -176,55 +172,6 @@ def list_dim_hook_options_ioc(hook_type):
 @dim_tasks_rest_blueprint.route('/dim/tasks/list/<int:count>', methods=['GET'])
 @ac_api_requires()
 def list_dim_tasks(count):
-    tasks = CeleryTaskMeta.query.filter(
-        ~ CeleryTaskMeta.name.like('app.iris_engine.updater.updater.%')
-    ).order_by(desc(CeleryTaskMeta.date_done)).limit(count).all()
+    data = asynchronous_tasks_search(count)
 
-    data = []
-
-    for row in tasks:
-
-        tkp = {'state': row.status, 'case': "Unknown", 'module': row.name, 'task_id': row.task_id, 'date_done': row.date_done, 'user': "Unknown"}
-
-        try:
-            _ = row.result
-        except AttributeError:
-            # Legacy task
-            data.append(tkp)
-            continue
-
-        if row.name is not None and 'task_hook_wrapper' in row.name:
-            task_name = f"{row.kwargs}::{row.kwargs}"
-        else:
-            task_name = row.name
-
-        user = None
-        case_name = None
-        if row.kwargs and row.kwargs != b'{}':
-            kwargs = json.loads(row.kwargs.decode('utf-8'))
-            if kwargs:
-                user = kwargs.get('init_user')
-                case_name = f"Case #{kwargs.get('caseid')}"
-                task_name = f"{kwargs.get('module_name')}::{kwargs.get('hook_name')}"
-
-        try:
-            result = pickle.loads(row.result)
-        except:
-            result = None
-
-        if isinstance(result, IIStatus):
-            try:
-                success = result.is_success()
-            except:
-                success = None
-        else:
-            success = None
-
-        tkp['state'] = "success" if success else str(row.result)
-        tkp['user'] = user if user else "Shadow Iris"
-        tkp['module'] = task_name
-        tkp['case'] = case_name if case_name else ""
-
-        data.append(tkp)
-
-    return response_success("", data=data)
+    return response_success('', data=data)

source/app/blueprints/rest/search_routes.py

@@ -18,19 +18,11 @@
 
 from flask import Blueprint
 from flask import request
-from sqlalchemy import and_
 
-from app.iris_engine.utils.tracker import track_activity
-from app.models.comments import Comments
 from app.models.authorization import Permissions
-from app.models.cases import Cases
-from app.models.models import Client
-from app.models.iocs import Ioc
-from app.models.models import IocType
-from app.models.models import Notes
-from app.models.iocs import Tlp
 from app.blueprints.access_controls import ac_api_requires
 from app.blueprints.responses import response_success
+from app.business.search import search
 
 search_rest_blueprint = Blueprint('search_rest', __name__)
 
@@ -42,79 +34,7 @@ def search_file_post():
     jsdata = request.get_json()
     search_value = jsdata.get('search_value')
     search_type = jsdata.get('search_type')
-    files = []
-    search_condition = and_()
 
-    track_activity(f'started a global search for {search_value} on {search_type}')
+    files = search(search_type, search_value)
 
-    if search_type == "ioc":
-        res = Ioc.query.with_entities(
-                            Ioc.ioc_value.label('ioc_name'),
-                            Ioc.ioc_description.label('ioc_description'),
-                            Ioc.ioc_misp,
-                            IocType.type_name,
-                            Tlp.tlp_name,
-                            Tlp.tlp_bscolor,
-                            Cases.name.label('case_name'),
-                            Cases.case_id,
-                            Client.name.label('customer_name')
-                    ).filter(
-                        and_(
-                            Ioc.ioc_value.like(search_value),
-                            Ioc.case_id == Cases.case_id,
-                            Client.client_id == Cases.client_id,
-                            Ioc.ioc_tlp_id == Tlp.tlp_id,
-                            search_condition
-                        )
-                    ).join(Ioc.ioc_type).all()
-
-        files = [row._asdict() for row in res]
-
-    if search_type == "notes":
-
-        ns = []
-        if search_value:
-            search_value = f'%{search_value}%'
-            ns = Notes.query.filter(
-                Notes.note_content.like(search_value),
-                Cases.client_id == Client.client_id,
-                search_condition
-            ).with_entities(
-                Notes.note_id,
-                Notes.note_title,
-                Cases.name.label('case_name'),
-                Client.name.label('client_name'),
-                Cases.case_id
-            ).join(
-                Notes.case
-            ).order_by(
-                Client.name
-            ).all()
-
-            ns = [row._asdict() for row in ns]
-
-        files = ns
-
-    if search_type == "comments":
-        search_value = f'%{search_value}%'
-        comments = Comments.query.filter(
-            Comments.comment_text.like(search_value),
-            Cases.client_id == Client.client_id,
-            search_condition
-        ).with_entities(
-            Comments.comment_id,
-            Comments.comment_text,
-            Cases.name.label('case_name'),
-            Client.name.label('customer_name'),
-            Cases.case_id
-        ).join(
-            Comments.case
-        ).join(
-            Cases.client
-        ).order_by(
-            Client.name
-        ).all()
-
-        files = [row._asdict() for row in comments]
-
-    return response_success("Results fetched", files)
+    return response_success('Results fetched', files)

source/app/blueprints/rest/v2/alerts.py

@@ -31,7 +31,7 @@
 from app.blueprints.rest.parsing import parse_comma_separated_identifiers
 from app.blueprints.rest.v2.alerts_routes.comments import alerts_comments_blueprint
 from app.iris_engine.access_control.iris_user import iris_current_user
-from app.datamgmt.alerts.alerts_db import get_filtered_alerts
+from app.business.alerts import alerts_search
 from app.models.authorization import Permissions
 from app.schema.marshables import AlertSchema
 from app.schema.marshables import IocSchema
@@ -50,7 +50,7 @@ class AlertsOperations:
     def __init__(self):
         self._schema = AlertSchema()
 
-    def list(self):
+    def search(self):
         page = request.args.get('page', 1, type=int)
         per_page = request.args.get('per_page', 10, type=int)
 
@@ -91,31 +91,31 @@ def list(self):
         else:
             fields = None
 
-        filtered_alerts = get_filtered_alerts(
-            start_date=request.args.get('creation_start_date'),
-            end_date=request.args.get('creation_end_date'),
-            source_start_date=request.args.get('source_start_date'),
-            source_end_date=request.args.get('source_end_date'),
-            source_reference=request.args.get('source_reference'),
-            title=request.args.get('alert_title'),
-            description=request.args.get('alert_description'),
-            status=request.args.get('alert_status_id', type=int),
-            severity=request.args.get('alert_severity_id', type=int),
-            owner=request.args.get('alert_owner_id', type=int),
-            source=request.args.get('alert_source'),
-            tags=request.args.get('alert_tags'),
-            classification=request.args.get('alert_classification_id', type=int),
-            client=request.args.get('alert_customer_id'),
-            case_id=request.args.get('case_id', type=int),
-            alert_ids=alert_ids,
-            page=page,
-            per_page=per_page,
-            sort=request.args.get('sort'),
-            custom_conditions=request.args.get('custom_conditions'),
-            assets=alert_assets,
-            iocs=alert_iocs,
-            resolution_status=request.args.get('alert_resolution_id', type=int),
-            current_user_id=iris_current_user.id
+        filtered_alerts = alerts_search(
+            iris_current_user.id,
+            request.args.get('creation_start_date'),
+            request.args.get('creation_end_date'),
+            request.args.get('source_start_date'),
+            request.args.get('source_end_date'),
+            request.args.get('alert_title'),
+            request.args.get('alert_description'),
+            request.args.get('alert_status_id', type=int),
+            request.args.get('alert_severity_id', type=int),
+            request.args.get('alert_owner_id', type=int),
+            request.args.get('alert_source'),
+            request.args.get('alert_tags'),
+            request.args.get('case_id', type=int),
+            request.args.get('alert_customer_id'),
+            request.args.get('alert_classification_id', type=int),
+            alert_ids,
+            alert_assets,
+            alert_iocs,
+            request.args.get('alert_resolution_id', type=int),
+            request.args.get('source_reference'),
+            request.args.get('custom_conditions'),
+            page,
+            per_page,
+            request.args.get('sort')
         )
 
         if filtered_alerts is None:
@@ -218,7 +218,7 @@ def delete(self, identifier):
 @alerts_blueprint.get('')
 @ac_api_requires(Permissions.alerts_read)
 def alerts_list_route() -> Response:
-    return alerts_operations.list()
+    return alerts_operations.search()
 
 
 @alerts_blueprint.post('')