Api v2 get paginated customers

.github/workflows/ci.yml

@@ -31,12 +31,14 @@ jobs:
         uses: astral-sh/ruff-action@v3
         with:
           args: check --output-format=github
-      - name: Check dependencies with import-linter
+      - name: Check import dependencies with import-linter
         run: |
-          python -m venv venv
-          source venv/bin/activate
           pip install import-linter
           PYTHONPATH=source lint-imports
+      - name: Looking for dead code with vulture
+        run: |
+          pip install vulture
+          vulture
 
   build-docker-db:
     name: Build docker db
@@ -131,10 +133,8 @@ jobs:
         uses: actions/checkout@v4
       - name: Start development server
         run: |
-          # Even though, we use --env-file option when running docker compose, this is still necessary, because the compose has a env_file attribute :(
-          # TODO should move basic.env file, which is in directory tests, up. It's used in several places. Maybe, rename it into dev.env
-          cp tests/data/basic.env .env
-          docker compose --file docker-compose.dev.yml --env-file tests/data/basic.env up --detach --wait
+          cp .env.tests.model .env
+          docker compose --file docker-compose.dev.yml up --detach --wait
       - name: Generate GraphQL documentation
         run: |
           npx spectaql@^3.0.2 source/spectaql/config.yml
@@ -170,9 +170,7 @@ jobs:
         uses: actions/checkout@v4
       - name: Start development server
         run: |
-          # Even though, we use --env-file option when running docker compose, this is still necessary, because the compose has a env_file attribute :(
-          # TODO should move basic.env file, which is in directory tests, up. It's used in several places. Maybe, rename it into dev.env
-          cp tests/data/basic.env .env
+          cp .env.tests.model .env
           docker compose --file docker-compose.dev.yml up --detach --wait
       - name: Inspect development server start failure
         if: ${{ failure() || cancelled() }}
@@ -226,8 +224,7 @@ jobs:
       - name: Check out iris
         uses: actions/checkout@v4
       - name: Set up .env file
-        # TODO should move basic.env file, which is in directory tests, up. It's used in several places. Maybe, rename it into dev.env?
-        run: cp tests/data/basic.env .env
+        run: cp .env.tests.model .env
       - name: Run tests
         working-directory: tests_database_migration
         run: |
@@ -277,8 +274,7 @@ jobs:
         run: npx playwright install chromium firefox
       - name: Start development server
         run: |
-          # TODO should move basic.env file, which is in directory tests, up. It's used in several places. Maybe, rename it into dev.env
-          cp tests/data/basic.env .env
+          cp .env.tests.model .env
           docker compose --file docker-compose.dev.yml up --detach --wait
       - name: Run end to end tests
         working-directory: e2e

pyproject.toml

@@ -1,8 +1,12 @@
 [tool.ruff.lint]
 preview = true
-select = ["E101", "E225", "E23", "E24", "E3", "E4", "E7", "E9", "F", "PLR0402", "RET506", "TID252", "UP032", "W29"]
+select = ["ARG003", "ARG005", "B00", "E101", "E20", "E225", "E23", "E24", "E3", "E4", "E7", "E9", "F", "FURB142", "FURB145", "FURB148", "PLR0402", "RET506", "RUF029", "RUF100", "TID252", "UP032", "W29", "W391"]
 ignore = ["E402", "E711", "E712", "E721", "E722"]
 
+[tool.vulture]
+paths = ["source/app", ".vulture.ignore"]
+ignore_decorators = ["@*.route", "@app.*", "@*.post", "@*.get", "@*.put", "@*.delete", "@pre_load", "@post_load"]
+
 [tool.importlinter]
 root_package = "app"
 include_external_packages = true
@@ -24,7 +28,7 @@ allow_indirect_imports = true
 [[tool.importlinter.contracts]]
 name = "Do not import API layer from the business layer"
 type = "forbidden"
-source_modules = ["app.business.access_controls", "app.business.assets"]
+source_modules = ["app.business.access_controls", "app.business.assets", "app.business.cases"]
 forbidden_modules = "app.blueprints.iris_user"
 allow_indirect_imports = true
 
@@ -39,6 +43,34 @@ allow_indirect_imports = true
 name = "Do not import API layer from the persistence layer"
 type = "forbidden"
 source_modules = "app.datamgmt.dashboard"
-forbidden_modules = "app.blueprints.iris_user"
+forbidden_modules = "app.blueprints"
+allow_indirect_imports = true
+
+[[tool.importlinter.contracts]]
+name = "Do not import API layer from the persistence layer (access_controls)"
+type = "forbidden"
+source_modules = "app.datamgmt.alerts.alerts_db"
+forbidden_modules = "app.blueprints.access_controls"
+allow_indirect_imports = true
+
+[[tool.importlinter.contracts]]
+name = "Do not import marshables from the persistence layer"
+type = "forbidden"
+source_modules = ["app.datamgmt.manage.manage_case_state_db", "app.datamgmt.manage.manage_groups_db"]
+forbidden_modules = "app.schema.marshables"
+allow_indirect_imports = true
+
+[[tool.importlinter.contracts]]
+name = "Do not import marshmallow from the persistence layer"
+type = "forbidden"
+source_modules = "app.datamgmt.client"
+forbidden_modules = "marshmallow"
+allow_indirect_imports = true
+
+[[tool.importlinter.contracts]]
+name = "Do not import the engine from the persistence layer"
+type = "forbidden"
+source_modules = "app.datamgmt.case"
+forbidden_modules = "app.iris_engine"
 allow_indirect_imports = true
 

scripts/gunicorn-cfg.py

@@ -27,4 +27,3 @@
 
 def worker_exit(server, worker):
     sys.exit(4)
-

source/app/__init__.py

@@ -16,20 +16,18 @@
 #  along with this program; if not, write to the Free Software Foundation,
 #  Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
-import collections
-import json
 import os
-from flask import Flask, g
+from flask import Flask
+from flask import g
 from flask import session
 from flask_bcrypt import Bcrypt
 from flask_caching import Cache
 from flask_cors import CORS
 
 from flask_login import LoginManager
 from flask_marshmallow import Marshmallow
-from flask_socketio import SocketIO, Namespace
-from flask_sqlalchemy import SQLAlchemy
-from functools import partial
+from flask_socketio import SocketIO
+from flask_socketio import Namespace
 
 from werkzeug.middleware.proxy_fix import ProxyFix
 
@@ -39,6 +37,8 @@
 from app.iris_engine.tasker.celery import set_celery_flask_context
 from app.iris_engine.access_control.oidc_handler import get_oidc_client
 from app.jinja_filters import register_jinja_filters
+from app.models.authorization import ac_flag_match_mask
+from app.db import db
 
 
 class ReverseProxied(object):
@@ -59,12 +59,6 @@ class AlertsNamespace(Namespace):
 APP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 TEMPLATE_PATH = os.path.join(APP_PATH, 'templates/')
 
-SQLALCHEMY_ENGINE_OPTIONS = {
-    "json_deserializer": partial(json.loads, object_pairs_hook=collections.OrderedDict),
-    "pool_pre_ping": True
-}
-
-db = SQLAlchemy(engine_options=SQLALCHEMY_ENGINE_OPTIONS)  # flask-sqlalchemy
 bc = Bcrypt()  # flask-bcrypt
 ma = Marshmallow()
 celery = make_celery(__name__)
@@ -75,10 +69,13 @@ def ac_current_user_has_permission(*permissions):
     """
     Return True if current user has permission
     """
+    if 'permissions' not in session:
+        return False
+
+    current_user_permissions = session['permissions']
     for permission in permissions:
 
-        if ('permissions' in session and
-                session['permissions'] & permission.value == permission.value):
+        if ac_flag_match_mask(current_user_permissions, permission.value):
             return True
 
     return False

source/app/alembic/versions/10a7616f3cc7_add_module_types.py

@@ -42,4 +42,3 @@ def upgrade():
 
 def downgrade():
     pass
-

source/app/alembic/versions/ad4e0cd17597_add_ioctype_validation.py

@@ -93,4 +93,3 @@ def upgrade():
 def downgrade():
     op.drop_column('ioc_type', 'type_validation_regex')
     op.drop_column('ioc_type', 'type_validation_expect')
-

source/app/alembic/versions/cd519d2d24df_password_policy_edition.py

@@ -57,4 +57,3 @@ def upgrade():
 
 def downgrade():
     pass
-

source/app/alembic/versions/d5a720d1b99b_add_alerts_indexes.py

@@ -61,4 +61,3 @@ def downgrade():
 
     # Drop AlertSimilarity table
     op.drop_table('alert_similarity')
-

source/app/blueprints/access_controls.py

@@ -45,13 +45,14 @@
 from app.business.auth import validate_auth_token
 from app.business.auth import update_session_current_case
 from app.datamgmt.case.case_db import get_case
-from app.datamgmt.manage.manage_access_control_db import user_has_client_access
+from app.business.access_controls import access_controls_user_has_customer_access
 from app.datamgmt.manage.manage_users_db import get_user
 from app.blueprints.iris_user import iris_current_user
 from app.business.access_controls import ac_fast_check_user_has_case_access
 from app.iris_engine.access_control.utils import ac_get_effective_permissions_of_user
 from app.iris_engine.utils.tracker import track_activity
 from app.models.authorization import Permissions
+from app.models.authorization import ac_flag_match_mask
 from app.models.authorization import CaseAccessLevel
 
 
@@ -388,7 +389,7 @@ def inner_wrap(f):
         @wraps(f)
         def wrap(*args, **kwargs):
             client_id = kwargs.get('client_id')
-            if not user_has_client_access(iris_current_user.id, client_id):
+            if not ac_current_user_has_customer_access(client_id):
                 return _ac_return_access_denied()
 
             return f(*args, **kwargs)
@@ -438,7 +439,7 @@ def inner_wrap(f):
         @wraps(f)
         def wrap(*args, **kwargs):
             client_id = kwargs.get('client_id')
-            if not user_has_client_access(iris_current_user.id, client_id):
+            if not ac_current_user_has_customer_access(client_id):
                 return response_error("Permission denied", status=403)
 
             return f(*args, **kwargs)
@@ -582,3 +583,17 @@ def is_authentication_ldap():
 
 def ac_fast_check_current_user_has_case_access(cid, access_level):
     return ac_fast_check_user_has_case_access(iris_current_user.id, cid, access_level)
+
+
+def ac_current_user_has_permission(permission):
+    """
+    Return True if current user has permission
+    """
+    return ac_flag_match_mask(session['permissions'], permission.value)
+
+
+def ac_current_user_has_customer_access(customer_identifier):
+    if ac_current_user_has_permission(Permissions.server_administrator):
+        return True
+
+    return access_controls_user_has_customer_access(iris_current_user, customer_identifier)

source/app/blueprints/graphql/cases.py

@@ -27,6 +27,7 @@
 from graphene import Float
 from graphene import String
 
+from app.blueprints.access_controls import ac_current_user_has_customer_access
 from app.models.cases import Cases
 from app.models.authorization import Permissions
 from app.models.authorization import CaseAccessLevel
@@ -36,14 +37,12 @@
 from app.business.cases import cases_delete
 from app.business.cases import cases_update
 from app.business.cases import cases_get_by_identifier
-from app.business.errors import BusinessProcessingError
+from app.models.errors import BusinessProcessingError
 from app.blueprints.graphql.permissions import permissions_check_current_user_has_some_permission
 from app.blueprints.graphql.permissions import permissions_check_current_user_has_some_case_access
 from app.iris_engine.module_handler.module_handler import call_deprecated_on_preload_modules_hook
 from app.schema.marshables import CaseSchema
 from app.blueprints.iris_user import iris_current_user
-from app.datamgmt.manage.manage_access_control_db import user_has_client_access
-
 from app.blueprints.graphql.iocs import IOCConnection
 
 
@@ -115,7 +114,7 @@ def mutate(root, info, name, description, client_id, soc_id=None, classification
         schema = CaseSchema()
         case = schema.load(request_data)
         case_template_id = request_data.pop('case_template_id', None)
-        result = cases_create(case, case_template_id)
+        result = cases_create(iris_current_user, case, case_template_id)
         return CaseCreate(case=result)
 
 
@@ -185,7 +184,7 @@ def mutate(root, info, case_id, name=None, soc_id=None, classification_id=None,
 
         # If user tries to update the customer, check if the user has access to the new customer
         if request.get('case_customer') and request.get('case_customer') != case.client_id:
-            if not user_has_client_access(iris_current_user.id, request.get('case_customer')):
+            if not ac_current_user_has_customer_access(request.get('case_customer')):
                 raise BusinessProcessingError('Invalid customer ID. Permission denied.')
 
         if 'case_name' in request:

source/app/blueprints/graphql/sliced_result.py

@@ -29,4 +29,3 @@ def __getitem__(self, index: slice) -> any:
 
     def __len__(self) -> int:
         return self._total
-

source/app/blueprints/pages/alerts/alerts_routes.py

@@ -25,11 +25,9 @@
 from werkzeug import Response
 
 from app.datamgmt.alerts.alerts_db import get_alert_by_id
-from app.datamgmt.manage.manage_access_control_db import user_has_client_access
 from app.models.authorization import Permissions
 from app.blueprints.responses import response_error
-from app.blueprints.access_controls import ac_requires
-from app.blueprints.iris_user import iris_current_user
+from app.blueprints.access_controls import ac_requires, ac_current_user_has_customer_access
 
 alerts_blueprint = Blueprint(
     'alerts',
@@ -78,7 +76,7 @@ def alert_comment_modal(cur_id, caseid, url_redir):
     if not alert:
         return response_error('Invalid alert ID')
 
-    if not user_has_client_access(iris_current_user.id, alert.alert_customer_id):
+    if not ac_current_user_has_customer_access(alert.alert_customer_id):
         return response_error('User not entitled to update alerts for the client', status=403)
 
     return render_template("modal_conversation.html", element_id=cur_id, element_type='alerts',

source/app/blueprints/pages/case/case_ioc_routes.py

@@ -22,7 +22,7 @@
 from flask import url_for
 
 from app.business.iocs import iocs_get
-from app.business.errors import ObjectNotFoundError
+from app.models.errors import ObjectNotFoundError
 from app.datamgmt.case.assets_type import get_assets_types
 from app.datamgmt.case.case_db import get_case
 from app.datamgmt.case.case_iocs_db import get_case_iocs_comments_count

source/app/blueprints/pages/login/login_routes.py

@@ -34,7 +34,7 @@
 
 from app import app
 from app import bc
-from app import db
+from app.db import db
 from app import oidc_client
 from app.blueprints.access_controls import is_authentication_oidc
 from app.blueprints.access_controls import is_authentication_ldap

source/app/blueprints/pages/manage/manage_cases_routes.py

@@ -34,13 +34,15 @@
 from app.datamgmt.manage.manage_cases_db import get_case_protagonists
 from app.datamgmt.manage.manage_common import get_severities_list
 from app.forms import AddCaseForm
-from app.iris_engine.access_control.utils import ac_current_user_has_permission
 from app.models.authorization import CaseAccessLevel
 from app.models.authorization import Permissions
 from app.schema.marshables import CaseDetailsSchema
-from app.blueprints.access_controls import ac_api_return_access_denied, ac_fast_check_current_user_has_case_access
+from app.blueprints.access_controls import ac_api_return_access_denied
+from app.blueprints.access_controls import ac_fast_check_current_user_has_case_access
+from app.blueprints.access_controls import ac_current_user_has_permission
 from app.blueprints.access_controls import ac_requires
 from app.blueprints.responses import response_error
+from app.schema.marshables import CaseStateSchema
 
 manage_cases_blueprint = Blueprint('manage_case',
                                    __name__,
@@ -81,18 +83,18 @@ def _details_case(cur_id: int, caseid: int, url_redir: bool) -> Union[str, Respo
 
     case_classifications = get_case_classifications_list()
     case_states = get_case_states_list()
+    dumped_case_states = CaseStateSchema(many=True).dump(case_states)
     user_is_server_administrator = ac_current_user_has_permission(Permissions.server_administrator)
 
-    customers = get_client_list(current_user_id=iris_current_user.id,
-                                is_server_administrator=user_is_server_administrator)
+    customers = get_client_list(iris_current_user.id, user_is_server_administrator)
 
     severities = get_severities_list()
     protagonists = [r._asdict() for r in get_case_protagonists(cur_id)]
 
     form = FlaskForm()
 
-    return render_template("modal_case_info_from_case.html", data=res, form=form, protagonists=protagonists,
-                           case_classifications=case_classifications, case_states=case_states, customers=customers,
+    return render_template('modal_case_info_from_case.html', data=res, form=form, protagonists=protagonists,
+                           case_classifications=case_classifications, case_states=dumped_case_states, customers=customers,
                            severities=severities)
 
 
@@ -117,9 +119,7 @@ def add_case_modal(caseid: int, url_redir: bool):
     form = AddCaseForm()
 
     # Show only clients that the user has access to
-    client_list = get_client_list(current_user_id=iris_current_user.id,
-                                  is_server_administrator=ac_current_user_has_permission(
-                                      Permissions.server_administrator))
+    client_list = get_client_list(iris_current_user.id, ac_current_user_has_permission(Permissions.server_administrator))
 
     form.case_customer_id.choices = [(c['customer_id'], c['customer_name']) for c in client_list]