diff --git a/.gitignore b/.gitignore index 01b64dd9..d3ead969 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ __pycache__ uploads node_modules schemas.json -.mypy_cache \ No newline at end of file +.mypy_cache +*.db diff --git a/app.py b/app.py index 23dd0e1f..bfa77814 100644 --- a/app.py +++ b/app.py @@ -10,6 +10,7 @@ from src.opengeodeweb_back.routes.models import blueprint_models from src.opengeodeweb_back.utils_functions import handle_exception from src.opengeodeweb_back import app_config +from src.opengeodeweb_back.database import initialize_database """ Global config """ @@ -57,5 +58,6 @@ def return_error(): # ''' Main ''' if __name__ == "__main__": + initialize_database(app) print(f"Python is running in {FLASK_DEBUG} mode") app.run(debug=FLASK_DEBUG, host=DEFAULT_HOST, port=PORT, ssl_context=SSL) diff --git a/requirements.in b/requirements.in index 10ebe922..5ad963ec 100644 --- a/requirements.in +++ b/requirements.in @@ -8,3 +8,4 @@ fastjsonschema==2.16.2 Flask[async]==3.0.3 Flask-Cors==6.0.1 werkzeug==3.0.3 +Flask-SQLAlchemy==3.1.1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 1e0a5aa2..f6b829e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,12 +17,17 @@ flask[async]==3.0.3 # -r requirements.in # flask # flask-cors + # flask-sqlalchemy flask-cors==6.0.1 # via -r requirements.in +flask-sqlalchemy==3.1.1 + # via -r requirements.in geode-common==33.9.0 # via geode-viewables geode-viewables==3.2.0 # via -r requirements.in +greenlet==3.2.4 + # via sqlalchemy itsdangerous==2.2.0 # via flask jinja2==3.1.6 @@ -54,6 +59,10 @@ opengeode-io==7.3.2 # -r requirements.in # geode-viewables # opengeode-geosciencesio +sqlalchemy==2.0.43 + # via flask-sqlalchemy +typing-extensions==4.15.0 + # via sqlalchemy werkzeug==3.0.3 # via # -r requirements.in diff --git a/src/opengeodeweb_back/app_config.py b/src/opengeodeweb_back/app_config.py index a4e87fc2..9f60b76d 100644 --- a/src/opengeodeweb_back/app_config.py +++ b/src/opengeodeweb_back/app_config.py @@ -4,6 +4,7 @@ # Third party imports # Local application imports +from .database import DATABASE_FILENAME class Config(object): @@ -15,6 +16,7 @@ class Config(object): REQUEST_COUNTER = 0 LAST_REQUEST_TIME = time.time() LAST_PING_TIME = time.time() + SQLALCHEMY_TRACK_MODIFICATIONS = False class ProdConfig(Config): @@ -22,7 +24,10 @@ class ProdConfig(Config): ORIGINS = "" MINUTES_BEFORE_TIMEOUT = "1" SECONDS_BETWEEN_SHUTDOWNS = "10" - DATA_FOLDER_PATH = "/data/" + DATA_FOLDER_PATH = "/data" + SQLALCHEMY_DATABASE_URI = f"sqlite:///{os.path.abspath( + os.path.join(DATA_FOLDER_PATH, DATABASE_FILENAME) + )}" class DevConfig(Config): @@ -30,4 +35,8 @@ class DevConfig(Config): ORIGINS = "*" MINUTES_BEFORE_TIMEOUT = "1" SECONDS_BETWEEN_SHUTDOWNS = "10" - DATA_FOLDER_PATH = "./data/" + BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + DATA_FOLDER_PATH = os.path.join(BASE_DIR, "data") + SQLALCHEMY_DATABASE_URI = f"sqlite:///{os.path.join( + BASE_DIR, DATA_FOLDER_PATH, DATABASE_FILENAME + )}" diff --git a/src/opengeodeweb_back/data.py b/src/opengeodeweb_back/data.py new file mode 100644 index 00000000..ce3b6a13 --- /dev/null +++ b/src/opengeodeweb_back/data.py @@ -0,0 +1,44 @@ +from sqlalchemy import String, JSON +from sqlalchemy.orm import Mapped, mapped_column +from .database import database, Base +import uuid + + +class Data(Base): + __tablename__ = "datas" + + id: Mapped[str] = mapped_column( + String, primary_key=True, default=lambda: str(uuid.uuid4()).replace("-", "") + ) + name: Mapped[str] = mapped_column(String, nullable=False) + native_file_name: Mapped[str] = mapped_column(String, nullable=False) + viewable_file_name: Mapped[str] = mapped_column(String, nullable=False) + geode_object: Mapped[str] = mapped_column(String, nullable=False) + + light_viewable: Mapped[str | None] = mapped_column(String, nullable=True) + input_file: Mapped[str | None] = mapped_column(String, nullable=True) + additional_files: Mapped[list[str] | None] = mapped_column(JSON, nullable=True) + + @staticmethod + def create( + name: str, + geode_object: str, + input_file: str | None = None, + additional_files: list[str] | None = None, + ) -> "Data": + input_file = input_file if input_file is not None else "" + additional_files = additional_files if additional_files is not None else [] + + data_entry = Data( + name=name, + geode_object=geode_object, + input_file=input_file, + additional_files=additional_files, + native_file_name="", + viewable_file_name="", + light_viewable=None, + ) + + database.session.add(data_entry) + database.session.flush() + return data_entry diff --git a/src/opengeodeweb_back/database.py b/src/opengeodeweb_back/database.py new file mode 100644 index 00000000..d5e2f0f7 --- /dev/null +++ b/src/opengeodeweb_back/database.py @@ -0,0 +1,19 @@ +from flask import Flask +from flask_sqlalchemy import SQLAlchemy +from sqlalchemy.orm import DeclarativeBase + +DATABASE_FILENAME = "project.db" + + +class Base(DeclarativeBase): + pass + + +database = SQLAlchemy(model_class=Base) + + +def initialize_database(app: Flask) -> SQLAlchemy: + database.init_app(app) + with app.app_context(): + database.create_all() + return database diff --git a/src/opengeodeweb_back/utils_functions.py b/src/opengeodeweb_back/utils_functions.py index 3efd0af9..1fcd84aa 100644 --- a/src/opengeodeweb_back/utils_functions.py +++ b/src/opengeodeweb_back/utils_functions.py @@ -2,7 +2,6 @@ import os import threading import time -import uuid import zipfile from collections.abc import Callable from typing import Any @@ -17,6 +16,8 @@ # Local application imports from . import geode_functions +from .data import Data +from .database import database def increment_request_counter(current_app: flask.Flask) -> None: @@ -152,17 +153,29 @@ def handle_exception(exception: HTTPException) -> flask.Response: return response -def create_unique_data_folder() -> tuple[str, str]: +def create_data_folder_from_id(data_id: str) -> str: base_data_folder = flask.current_app.config["DATA_FOLDER_PATH"] - generated_id = str(uuid.uuid4()).replace("-", "") - data_path = os.path.join(base_data_folder, generated_id) + data_path = os.path.join(base_data_folder, data_id) os.makedirs(data_path, exist_ok=True) - return generated_id, data_path + return data_path def save_all_viewables_and_return_info( - geode_object, data, generated_id, data_path, additional_files=None -): + geode_object: str, + data: Any, + input_file: str, + additional_files: list[str] | None = None, +) -> dict[str, Any]: + if additional_files is None: + additional_files = [] + + data_entry = Data.create( + name=data.name(), + geode_object=geode_object, + input_file=input_file, + additional_files=additional_files, + ) + data_path = create_data_folder_from_id(data_entry.id) saved_native_file_path = geode_functions.save( geode_object, data, @@ -177,28 +190,42 @@ def save_all_viewables_and_return_info( ) with open(saved_light_viewable_file_path, "rb") as f: binary_light_viewable = f.read() + data_entry.native_file_name = os.path.basename(saved_native_file_path[0]) + data_entry.viewable_file_name = os.path.basename(saved_viewable_file_path) + data_entry.light_viewable = os.path.basename(saved_light_viewable_file_path) + + database.session.commit() return { - "name": data.name(), - "native_file_name": os.path.basename(saved_native_file_path[0]), - "viewable_file_name": os.path.basename(saved_viewable_file_path), - "id": generated_id, + "name": data_entry.name, + "native_file_name": data_entry.native_file_name, + "viewable_file_name": data_entry.viewable_file_name, + "id": data_entry.id, "object_type": geode_functions.get_object_type(geode_object), "binary_light_viewable": binary_light_viewable.decode("utf-8"), - "geode_object": geode_object, - "input_files": additional_files or [], + "geode_object": data_entry.geode_object, + "input_files": data_entry.input_file, + "additional_files": data_entry.additional_files, } -def generate_native_viewable_and_light_viewable_from_object(geode_object, data): - generated_id, data_path = create_unique_data_folder() - return save_all_viewables_and_return_info( - geode_object, data, generated_id, data_path - ) +def generate_native_viewable_and_light_viewable_from_object( + geode_object: str, data: Any +) -> dict[str, Any]: + return save_all_viewables_and_return_info(geode_object, data, input_file="") -def generate_native_viewable_and_light_viewable_from_file(geode_object, input_filename): - generated_id, data_path = create_unique_data_folder() +def generate_native_viewable_and_light_viewable_from_file( + geode_object: str, input_filename: str +) -> dict[str, Any]: + temp_data_entry = Data.create( + name="temp", + geode_object=geode_object, + input_file=input_filename, + additional_files=[], + ) + + data_path = create_data_folder_from_id(temp_data_entry.id) full_input_filename = geode_functions.upload_file_path(input_filename) copied_full_path = os.path.join( @@ -206,7 +233,7 @@ def generate_native_viewable_and_light_viewable_from_file(geode_object, input_fi ) shutil.copy2(full_input_filename, copied_full_path) - additional_files_copied = [] + additional_files_copied: list[str] = [] additional = geode_functions.additional_files(geode_object, full_input_filename) for additional_file in additional.mandatory_files + additional.optional_files: if additional_file.is_missing: @@ -221,12 +248,14 @@ def generate_native_viewable_and_light_viewable_from_file(geode_object, input_fi shutil.copy2(source_path, dest_path) additional_files_copied.append(additional_file.filename) - data = geode_functions.load_data(geode_object, generated_id, input_filename) + data = geode_functions.load_data(geode_object, temp_data_entry.id, input_filename) + + database.session.delete(temp_data_entry) + database.session.flush() return save_all_viewables_and_return_info( geode_object, data, - generated_id, - data_path, + input_file=input_filename, additional_files=additional_files_copied, ) diff --git a/tests/conftest.py b/tests/conftest.py index f9891df3..22203f27 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,10 +3,12 @@ import shutil # Third party imports +import os import pytest # Local application imports from app import app +from src.opengeodeweb_back.database import initialize_database TEST_ID = "1" @@ -15,14 +17,22 @@ def copy_data(): shutil.rmtree("./data", ignore_errors=True) shutil.copytree("./tests/data/", f"./data/{TEST_ID}/", dirs_exist_ok=True) - - -@pytest.fixture -def client(): app.config["TESTING"] = True app.config["SERVER_NAME"] = "TEST" app.config["DATA_FOLDER_PATH"] = "./data/" app.config["UPLOAD_FOLDER"] = "./tests/data/" + BASE_DIR = os.path.abspath(os.path.dirname(__file__)) + db_path = os.path.join(BASE_DIR, "data", "project.db") + app.config["SQLALCHEMY_DATABASE_URI"] = f"sqlite:///{db_path}" + + print("Current working directory:", os.getcwd()) + print("Directory contents:", os.listdir(".")) + + initialize_database(app) + + +@pytest.fixture +def client(): app.config["REQUEST_COUNTER"] = 0 app.config["LAST_REQUEST_TIME"] = time.time() client = app.test_client() diff --git a/tests/test_routes.py b/tests/test_routes.py index 83a0f375..3b9d5cf6 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -321,3 +321,15 @@ def test_create_point(client): # Test all params test_utils.test_route_wrong_params(client, route, get_full_data) + + +def test_database_uri_path(client): + app = client.application + with app.app_context(): + base_dir = os.path.abspath(os.path.dirname(__file__)) + expected_db_path = os.path.join(base_dir, "data", "project.db") + expected_uri = f"sqlite:///{expected_db_path}" + + assert app.config["SQLALCHEMY_DATABASE_URI"] == expected_uri + + assert os.path.exists(expected_db_path) diff --git a/tests/test_utils_functions.py b/tests/test_utils_functions.py index 52282717..cea87147 100644 --- a/tests/test_utils_functions.py +++ b/tests/test_utils_functions.py @@ -5,8 +5,11 @@ # Third party imports import flask import shutil +import uuid # Local application imports +from src.opengeodeweb_back.database import database +from src.opengeodeweb_back.data import Data from src.opengeodeweb_back import geode_functions, utils_functions @@ -72,15 +75,15 @@ def test_handle_exception(client): assert type(data["code"]) is int -def test_create_unique_data_folder(client): +def test_create_data_folder_from_id(client): app = client.application with app.app_context(): - generated_id, data_path = utils_functions.create_unique_data_folder() - assert isinstance(generated_id, str) - assert re.fullmatch(r"[0-9a-f]{32}", generated_id) + test_id = str(uuid.uuid4()).replace("-", "") + data_path = utils_functions.create_data_folder_from_id(test_id) + assert isinstance(data_path, str) assert os.path.exists(data_path) assert data_path.startswith(flask.current_app.config["DATA_FOLDER_PATH"]) - assert generated_id in data_path + assert test_id in data_path shutil.rmtree(data_path, ignore_errors=True) assert not os.path.exists(data_path) @@ -88,24 +91,66 @@ def test_create_unique_data_folder(client): def test_save_all_viewables_and_return_info(client): app = client.application with app.app_context(): + base_dir = os.path.abspath(os.path.dirname(__file__)) + expected_db_path = os.path.join(base_dir, "data", "project.db") + expected_uri = f"sqlite:///{expected_db_path}" + + assert app.config["SQLALCHEMY_DATABASE_URI"] == expected_uri + assert os.path.exists(expected_db_path) + geode_object = "BRep" data = geode_functions.load(geode_object, "./tests/data/test.og_brep") - generated_id, data_path = utils_functions.create_unique_data_folder() + input_file = "test.og_brep" additional_files = ["additional_file.txt"] result = utils_functions.save_all_viewables_and_return_info( - geode_object, data, generated_id, data_path, additional_files + geode_object, data, input_file, additional_files ) - assert isinstance(result, dict) - assert result["name"] == data.name() - assert result["native_file_name"].startswith("native.") - assert result["viewable_file_name"].endswith(".vtm") - assert re.match(r"[0-9a-f]{32}", result["id"]) - assert isinstance(result["object_type"], str) - assert isinstance(result["binary_light_viewable"], str) - assert result["geode_object"] == geode_object - assert result["input_files"] == additional_files + assert isinstance(result, dict) + assert result["name"] == data.name() + assert result["native_file_name"].startswith("native.") + assert result["viewable_file_name"].endswith(".vtm") + assert isinstance(result["id"], str) + assert len(result["id"]) == 32 + assert re.match(r"[0-9a-f]{32}", result["id"]) + assert isinstance(result["object_type"], str) + assert isinstance(result["binary_light_viewable"], str) + assert result["geode_object"] == geode_object + assert result["input_files"] == input_file + + db_entry = database.session.get(Data, result["id"]) + assert db_entry is not None + assert db_entry.name == data.name() + assert db_entry.native_file_name == result["native_file_name"] + assert db_entry.viewable_file_name == result["viewable_file_name"] + assert db_entry.geode_object == geode_object + assert db_entry.input_file == input_file + assert db_entry.additional_files == additional_files + + expected_data_path = os.path.join(app.config["DATA_FOLDER_PATH"], result["id"]) + assert os.path.exists(expected_data_path) + + +def test_save_all_viewables_commits_to_db_properly(client): + app = client.application + with app.app_context(): + geode_object = "BRep" + data = geode_functions.load(geode_object, "./tests/data/test.og_brep") + input_file = "test.og_brep" + result = utils_functions.save_all_viewables_and_return_info( + geode_object, data, input_file + ) + data_id = result["id"] + db_entry_before = database.session.get(Data, data_id) + assert db_entry_before is not None + assert db_entry_before.native_file_name == result["native_file_name"] + database.session.rollback() + db_entry_after = database.session.get(Data, data_id) + assert ( + db_entry_after is not None + ), "database.session.commit() was not called - entry missing after rollback" + assert db_entry_after.native_file_name == result["native_file_name"] def test_generate_native_viewable_and_light_viewable_from_object(client): @@ -130,7 +175,7 @@ def test_generate_native_viewable_and_light_viewable_from_object(client): assert re.match(r"[0-9a-f]{32}", result["id"]) assert isinstance(result["object_type"], str) assert isinstance(result["binary_light_viewable"], str) - assert result["input_files"] == [] + assert result["input_files"] == "" def test_generate_native_viewable_and_light_viewable_from_file(client): @@ -153,4 +198,4 @@ def test_generate_native_viewable_and_light_viewable_from_file(client): assert re.match(r"[0-9a-f]{32}", result["id"]) assert isinstance(result["object_type"], str) assert isinstance(result["binary_light_viewable"], str) - assert isinstance(result["input_files"], list) + assert isinstance(result["input_files"], str)