feat(database): Add persistent data management using sqlite & sqlalchemy

.gitignore

@@ -10,4 +10,5 @@ __pycache__
 uploads
 node_modules
 schemas.json
-.mypy_cache
+.mypy_cache
+*.db

app.py

@@ -10,6 +10,7 @@
 from src.opengeodeweb_back.routes.models import blueprint_models
 from src.opengeodeweb_back.utils_functions import handle_exception
 from src.opengeodeweb_back import app_config
+from src.opengeodeweb_back.database import initialize_database
 
 
 """ Global config """
@@ -57,5 +58,6 @@ def return_error():
 
 # ''' Main '''
 if __name__ == "__main__":
+    initialize_database(app)
     print(f"Python is running in {FLASK_DEBUG} mode")
     app.run(debug=FLASK_DEBUG, host=DEFAULT_HOST, port=PORT, ssl_context=SSL)

requirements.in

@@ -8,3 +8,4 @@ fastjsonschema==2.16.2
 Flask[async]==3.0.3
 Flask-Cors==6.0.1
 werkzeug==3.0.3
+Flask-SQLAlchemy==3.1.1

requirements.txt

@@ -17,12 +17,17 @@ flask[async]==3.0.3
     #   -r requirements.in
     #   flask
     #   flask-cors
+    #   flask-sqlalchemy
 flask-cors==6.0.1
     # via -r requirements.in
+flask-sqlalchemy==3.1.1
+    # via -r requirements.in
 geode-common==33.9.0
     # via geode-viewables
 geode-viewables==3.2.0
     # via -r requirements.in
+greenlet==3.2.4
+    # via sqlalchemy
 itsdangerous==2.2.0
     # via flask
 jinja2==3.1.6
@@ -54,6 +59,10 @@ opengeode-io==7.3.2
     #   -r requirements.in
     #   geode-viewables
     #   opengeode-geosciencesio
+sqlalchemy==2.0.43
+    # via flask-sqlalchemy
+typing-extensions==4.15.0
+    # via sqlalchemy
 werkzeug==3.0.3
     # via
     #   -r requirements.in

src/opengeodeweb_back/app_config.py

@@ -4,6 +4,7 @@
 
 # Third party imports
 # Local application imports
+from .database import DATABASE_FILENAME
 
 
 class Config(object):
@@ -15,19 +16,27 @@ class Config(object):
     REQUEST_COUNTER = 0
     LAST_REQUEST_TIME = time.time()
     LAST_PING_TIME = time.time()
+    SQLALCHEMY_TRACK_MODIFICATIONS = False
 
 
 class ProdConfig(Config):
     SSL = None
     ORIGINS = ""
     MINUTES_BEFORE_TIMEOUT = "1"
     SECONDS_BETWEEN_SHUTDOWNS = "10"
-    DATA_FOLDER_PATH = "/data/"
+    DATA_FOLDER_PATH = "/data"
+    SQLALCHEMY_DATABASE_URI = f"sqlite:///{os.path.abspath(
+        os.path.join(DATA_FOLDER_PATH, DATABASE_FILENAME)
+        )}"
 
 
 class DevConfig(Config):
     SSL = None
     ORIGINS = "*"
     MINUTES_BEFORE_TIMEOUT = "1"
     SECONDS_BETWEEN_SHUTDOWNS = "10"
-    DATA_FOLDER_PATH = "./data/"
+    BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+    DATA_FOLDER_PATH = os.path.join(BASE_DIR, "data")
+    SQLALCHEMY_DATABASE_URI = f"sqlite:///{os.path.join(
+        BASE_DIR, DATA_FOLDER_PATH, DATABASE_FILENAME
+        )}"

src/opengeodeweb_back/data.py

@@ -0,0 +1,44 @@
+from sqlalchemy import String, JSON
+from sqlalchemy.orm import Mapped, mapped_column
+from .database import database, Base
+import uuid
+
+
+class Data(Base):
+    __tablename__ = "datas"
+
+    id: Mapped[str] = mapped_column(
+        String, primary_key=True, default=lambda: str(uuid.uuid4()).replace("-", "")
+    )
+    name: Mapped[str] = mapped_column(String, nullable=False)
+    native_file_name: Mapped[str] = mapped_column(String, nullable=False)
+    viewable_file_name: Mapped[str] = mapped_column(String, nullable=False)
+    geode_object: Mapped[str] = mapped_column(String, nullable=False)
+
+    light_viewable: Mapped[str | None] = mapped_column(String, nullable=True)
+    input_file: Mapped[str | None] = mapped_column(String, nullable=True)
+    additional_files: Mapped[list[str] | None] = mapped_column(JSON, nullable=True)
+
+    @staticmethod
+    def create(
+        name: str,
+        geode_object: str,
+        input_file: str | None = None,
+        additional_files: list[str] | None = None,
+    ) -> "Data":
+        input_file = input_file if input_file is not None else ""
+        additional_files = additional_files if additional_files is not None else []
+
+        data_entry = Data(
+            name=name,
+            geode_object=geode_object,
+            input_file=input_file,
+            additional_files=additional_files,
+            native_file_name="",
+            viewable_file_name="",
+            light_viewable=None,
+        )
+
+        database.session.add(data_entry)
+        database.session.flush()
+        return data_entry

src/opengeodeweb_back/database.py

@@ -0,0 +1,19 @@
+from flask import Flask
+from flask_sqlalchemy import SQLAlchemy
+from sqlalchemy.orm import DeclarativeBase
+
+DATABASE_FILENAME = "project.db"
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+database = SQLAlchemy(model_class=Base)
+
+
+def initialize_database(app: Flask) -> SQLAlchemy:
+    database.init_app(app)
+    with app.app_context():
+        database.create_all()
+    return database

src/opengeodeweb_back/utils_functions.py

@@ -2,7 +2,6 @@
 import os
 import threading
 import time
-import uuid
 import zipfile
 from collections.abc import Callable
 from typing import Any
@@ -17,6 +16,8 @@
 
 # Local application imports
 from . import geode_functions
+from .data import Data
+from .database import database
 
 
 def increment_request_counter(current_app: flask.Flask) -> None:
@@ -152,17 +153,29 @@ def handle_exception(exception: HTTPException) -> flask.Response:
     return response
 
 
-def create_unique_data_folder() -> tuple[str, str]:
+def create_data_folder_from_id(data_id: str) -> str:
     base_data_folder = flask.current_app.config["DATA_FOLDER_PATH"]
-    generated_id = str(uuid.uuid4()).replace("-", "")
-    data_path = os.path.join(base_data_folder, generated_id)
+    data_path = os.path.join(base_data_folder, data_id)
     os.makedirs(data_path, exist_ok=True)
-    return generated_id, data_path
+    return data_path
 
 
 def save_all_viewables_and_return_info(
-    geode_object, data, generated_id, data_path, additional_files=None
-):
+    geode_object: str,
+    data: Any,
+    input_file: str,
+    additional_files: list[str] | None = None,
+) -> dict[str, Any]:
+    if additional_files is None:
+        additional_files = []
+
+    data_entry = Data.create(
+        name=data.name(),
+        geode_object=geode_object,
+        input_file=input_file,
+        additional_files=additional_files,
+    )
+    data_path = create_data_folder_from_id(data_entry.id)
     saved_native_file_path = geode_functions.save(
         geode_object,
         data,
@@ -177,36 +190,50 @@ def save_all_viewables_and_return_info(
     )
     with open(saved_light_viewable_file_path, "rb") as f:
         binary_light_viewable = f.read()
+    data_entry.native_file_name = os.path.basename(saved_native_file_path[0])
+    data_entry.viewable_file_name = os.path.basename(saved_viewable_file_path)
+    data_entry.light_viewable = os.path.basename(saved_light_viewable_file_path)
+
+    database.session.commit()
 
     return {
-        "name": data.name(),
-        "native_file_name": os.path.basename(saved_native_file_path[0]),
-        "viewable_file_name": os.path.basename(saved_viewable_file_path),
-        "id": generated_id,
+        "name": data_entry.name,
+        "native_file_name": data_entry.native_file_name,
+        "viewable_file_name": data_entry.viewable_file_name,
+        "id": data_entry.id,
         "object_type": geode_functions.get_object_type(geode_object),
         "binary_light_viewable": binary_light_viewable.decode("utf-8"),
-        "geode_object": geode_object,
-        "input_files": additional_files or [],
+        "geode_object": data_entry.geode_object,
+        "input_files": data_entry.input_file,
+        "additional_files": data_entry.additional_files,
     }
 
 
-def generate_native_viewable_and_light_viewable_from_object(geode_object, data):
-    generated_id, data_path = create_unique_data_folder()
-    return save_all_viewables_and_return_info(
-        geode_object, data, generated_id, data_path
-    )
+def generate_native_viewable_and_light_viewable_from_object(
+    geode_object: str, data: Any
+) -> dict[str, Any]:
+    return save_all_viewables_and_return_info(geode_object, data, input_file="")
 
 
-def generate_native_viewable_and_light_viewable_from_file(geode_object, input_filename):
-    generated_id, data_path = create_unique_data_folder()
+def generate_native_viewable_and_light_viewable_from_file(
+    geode_object: str, input_filename: str
+) -> dict[str, Any]:
+    temp_data_entry = Data.create(
+        name="temp",
+        geode_object=geode_object,
+        input_file=input_filename,
+        additional_files=[],
+    )
+
+    data_path = create_data_folder_from_id(temp_data_entry.id)
 
     full_input_filename = geode_functions.upload_file_path(input_filename)
     copied_full_path = os.path.join(
         data_path, werkzeug.utils.secure_filename(input_filename)
     )
     shutil.copy2(full_input_filename, copied_full_path)
 
-    additional_files_copied = []
+    additional_files_copied: list[str] = []
     additional = geode_functions.additional_files(geode_object, full_input_filename)
     for additional_file in additional.mandatory_files + additional.optional_files:
         if additional_file.is_missing:
@@ -221,12 +248,14 @@ def generate_native_viewable_and_light_viewable_from_file(geode_object, input_fi
         shutil.copy2(source_path, dest_path)
         additional_files_copied.append(additional_file.filename)
 
-    data = geode_functions.load_data(geode_object, generated_id, input_filename)
+    data = geode_functions.load_data(geode_object, temp_data_entry.id, input_filename)
+
+    database.session.delete(temp_data_entry)
+    database.session.flush()
 
     return save_all_viewables_and_return_info(
         geode_object,
         data,
-        generated_id,
-        data_path,
+        input_file=input_filename,
         additional_files=additional_files_copied,
     )

tests/conftest.py

@@ -3,10 +3,12 @@
 import shutil
 
 # Third party imports
+import os
 import pytest
 
 # Local application imports
 from app import app
+from src.opengeodeweb_back.database import initialize_database
 
 TEST_ID = "1"
 
@@ -15,14 +17,22 @@
 def copy_data():
     shutil.rmtree("./data", ignore_errors=True)
     shutil.copytree("./tests/data/", f"./data/{TEST_ID}/", dirs_exist_ok=True)
-
-
-@pytest.fixture
-def client():
     app.config["TESTING"] = True
     app.config["SERVER_NAME"] = "TEST"
     app.config["DATA_FOLDER_PATH"] = "./data/"
     app.config["UPLOAD_FOLDER"] = "./tests/data/"
+    BASE_DIR = os.path.abspath(os.path.dirname(__file__))
+    db_path = os.path.join(BASE_DIR, "data", "project.db")
+    app.config["SQLALCHEMY_DATABASE_URI"] = f"sqlite:///{db_path}"
+
+    print("Current working directory:", os.getcwd())
+    print("Directory contents:", os.listdir("."))
+
+    initialize_database(app)
+
+
+@pytest.fixture
+def client():
     app.config["REQUEST_COUNTER"] = 0
     app.config["LAST_REQUEST_TIME"] = time.time()
     client = app.test_client()

tests/test_routes.py

@@ -321,3 +321,15 @@ def test_create_point(client):
 
     # Test all params
     test_utils.test_route_wrong_params(client, route, get_full_data)
+
+
+def test_database_uri_path(client):
+    app = client.application
+    with app.app_context():
+        base_dir = os.path.abspath(os.path.dirname(__file__))
+        expected_db_path = os.path.join(base_dir, "data", "project.db")
+        expected_uri = f"sqlite:///{expected_db_path}"
+
+        assert app.config["SQLALCHEMY_DATABASE_URI"] == expected_uri
+
+        assert os.path.exists(expected_db_path)