From 2d03d7d4e1ac8ee61c88438ca719936c3e019bb8 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Mon, 4 Nov 2024 13:22:43 +0100 Subject: [PATCH 1/4] Implementing testing api for storage change to limits tests - add patch to mergin client --- mergin/client.py | 8 +++++++ mergin/test/test_client.py | 49 +++++++++++++++++++++++++------------- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/mergin/client.py b/mergin/client.py index ebd63d74..1df74ccc 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -236,6 +236,13 @@ def post(self, path, data=None, headers={}): data = json.dumps(data, cls=DateTimeEncoder).encode("utf-8") request = urllib.request.Request(url, data, headers, method="POST") return self._do_request(request) + + def patch(self, path, data=None, headers={}): + url = urllib.parse.urljoin(self.url, urllib.parse.quote(path)) + if headers.get("Content-Type", None) == "application/json": + data = json.dumps(data, cls=DateTimeEncoder).encode("utf-8") + request = urllib.request.Request(url, data, headers, method="PATCH") + return self._do_request(request) def is_server_compatible(self): """ @@ -661,6 +668,7 @@ def projects_list( order_params=order_params, only_public=only_public, ) + print(resp["projects"]) fetched_projects += len(resp["projects"]) count = resp["count"] projects += resp["projects"] diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index b5d238fc..925d24d5 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -32,7 +32,6 @@ from ..utils import ( generate_checksum, get_versions_with_file_changes, - is_version_acceptable, unique_path_name, conflicted_copy_file_name, edit_conflict_file_name, @@ -745,12 +744,12 @@ def test_set_editor_access(mc): assert API_USER2 not in access["writersnames"] -def test_available_storage_validation(mcStorage): +def test_available_workspace_storage(mcStorage): """ Testing of storage limit - applies to user pushing changes into own project (namespace matching username). This test also tests giving read and write access to another user. Additionally tests also uploading of big file. """ - test_project = "test_available_storage_validation" + test_project = "test_available_workspace_storage" test_project_fullname = STORAGE_WORKSPACE + "/" + test_project # cleanups @@ -766,15 +765,28 @@ def test_available_storage_validation(mcStorage): # get info about storage capacity storage_remaining = 0 + client_workspace = None + for workspace in mcStorage.workspaces_list(): + if workspace["name"] == STORAGE_WORKSPACE: + client_workspace = workspace + break + assert client_workspace is not None + current_storage = client_workspace["storage"] + client_workspace_id = client_workspace["id"] + # 5 MB + testing_storage = 5242880 + # add storage limit, to prevent creating too big files + mcStorage.patch( + f"/v1/tests/workspaces/{client_workspace_id}", + {"limits_override": {"storage": testing_storage, "projects": 1, "api_allowed": True}}, + {"Content-Type": "application/json"}, + ) if mcStorage.server_type() == ServerType.OLD: user_info = mcStorage.user_info() - storage_remaining = user_info["storage"] - user_info["disk_usage"] + storage_remaining = testing_storage - user_info["disk_usage"] else: - for workspace in mcStorage.workspaces_list(): - if workspace["name"] == STORAGE_WORKSPACE: - storage_remaining = workspace["storage"] - workspace["disk_usage"] - break + storage_remaining = testing_storage - client_workspace["disk_usage"] # generate dummy data (remaining storage + extra 1024b) dummy_data_path = project_dir + "/data" @@ -786,18 +798,23 @@ def test_available_storage_validation(mcStorage): try: mcStorage.push_project(project_dir) except ClientError as e: + print("Pushe fail") # Expecting "You have reached a data limit" 400 server error msg. assert "You have reached a data limit" in str(e) got_right_err = True - assert got_right_err + finally: + assert got_right_err - # Expecting empty project - project_info = get_project_info(mcStorage, STORAGE_WORKSPACE, test_project) - assert project_info["version"] == "v0" - assert project_info["disk_usage"] == 0 + # Expecting empty project + project_info = get_project_info(mcStorage, STORAGE_WORKSPACE, test_project) + assert project_info["version"] == "v0" + assert project_info["disk_usage"] == 0 - # remove dummy big file from a disk - remove_folders([project_dir]) + # remove dummy big file from a disk + remove_folders([project_dir]) + + # revert storage limit to default value + mcStorage.patch(f"/v1/tests/workspaces/{client_workspace_id}", {"limits_override": { "storage": current_storage, "projects": 1, "api_allowed": True }}, {"Content-Type": "application/json"}) def test_available_storage_validation2(mc, mc2): @@ -866,7 +883,7 @@ def get_project_info(mc, namespace, project_name): :param project_name: project's name :return: dict with project info """ - projects = mc.projects_list(flag="created") + projects = mc.projects_list(flag="created", namespace=namespace) test_project_list = [p for p in projects if p["name"] == project_name and p["namespace"] == namespace] assert len(test_project_list) == 1 return test_project_list[0] From 0e424354e7393fcf7cc209d5c3c6dd42b2dd335f Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Mon, 4 Nov 2024 13:39:19 +0100 Subject: [PATCH 2/4] Cleanup and black - use limits api in project limit hit test to allow run this test separately --- mergin/client.py | 1 - mergin/test/test_client.py | 26 +++++++++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/mergin/client.py b/mergin/client.py index 1df74ccc..637e704d 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -668,7 +668,6 @@ def projects_list( order_params=order_params, only_public=only_public, ) - print(resp["projects"]) fetched_projects += len(resp["projects"]) count = resp["count"] projects += resp["projects"] diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index 925d24d5..e165ab3a 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -814,7 +814,11 @@ def test_available_workspace_storage(mcStorage): remove_folders([project_dir]) # revert storage limit to default value - mcStorage.patch(f"/v1/tests/workspaces/{client_workspace_id}", {"limits_override": { "storage": current_storage, "projects": 1, "api_allowed": True }}, {"Content-Type": "application/json"}) + mcStorage.patch( + f"/v1/tests/workspaces/{client_workspace_id}", + {"limits_override": {"storage": current_storage, "projects": 1, "api_allowed": True}}, + {"Content-Type": "application/json"}, + ) def test_available_storage_validation2(mc, mc2): @@ -2670,6 +2674,19 @@ def test_error_projects_limit_hit(mcStorage: MerginClient): test_project = "test_another_project_above_projects_limit" test_project_fullname = STORAGE_WORKSPACE + "/" + test_project + client_workspace = None + for workspace in mcStorage.workspaces_list(): + if workspace["name"] == STORAGE_WORKSPACE: + client_workspace = workspace + break + client_workspace_id = client_workspace["id"] + client_workspace_storage = client_workspace["storage"] + mcStorage.patch( + f"/v1/tests/workspaces/{client_workspace_id}", + {"limits_override": {"storage": client_workspace_storage, "projects": 0, "api_allowed": True}}, + {"Content-Type": "application/json"}, + ) + project_dir = os.path.join(TMP_DIR, test_project, API_USER) with pytest.raises(ClientError) as e: @@ -2682,3 +2699,10 @@ def test_error_projects_limit_hit(mcStorage: MerginClient): assert e.value.http_error == 422 assert e.value.http_method == "POST" assert e.value.url == f"{mcStorage.url}v1/project/testpluginstorage" + + # back to original values... (1 project, api allowed ...) + mcStorage.patch( + f"/v1/tests/workspaces/{client_workspace_id}", + {"limits_override": {"storage": client_workspace_storage, "projects": 1, "api_allowed": True}}, + {"Content-Type": "application/json"}, + ) From 23fa36671aa3a50ec13614d655ecd9a4e0424bf9 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Mon, 4 Nov 2024 13:43:36 +0100 Subject: [PATCH 3/4] Black --- mergin/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mergin/client.py b/mergin/client.py index 637e704d..4cd7ef39 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -236,7 +236,7 @@ def post(self, path, data=None, headers={}): data = json.dumps(data, cls=DateTimeEncoder).encode("utf-8") request = urllib.request.Request(url, data, headers, method="POST") return self._do_request(request) - + def patch(self, path, data=None, headers={}): url = urllib.parse.urljoin(self.url, urllib.parse.quote(path)) if headers.get("Content-Type", None) == "application/json": From 007cf909d923118c899a99d385713be332d4eed0 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Thu, 7 Nov 2024 10:04:58 +0100 Subject: [PATCH 4/4] Added teardown for limit overrides --- mergin/test/test_client.py | 45 +++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index e165ab3a..458eef63 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -67,9 +67,27 @@ def mc2(): @pytest.fixture(scope="function") -def mcStorage(): +def mcStorage(request): client = create_client(API_USER, USER_PWD) - create_workspace_for_client(client, STORAGE_WORKSPACE) + workspace_name = create_workspace_for_client(client, STORAGE_WORKSPACE) + print(workspace_name) + client_workspace = None + for workspace in client.workspaces_list(): + if workspace["name"] == workspace_name: + client_workspace = workspace + break + client_workspace_id = client_workspace["id"] + client_workspace_storage = client_workspace["storage"] + + def teardown(): + # back to original values... (1 project, api allowed ...) + client.patch( + f"/v1/tests/workspaces/{client_workspace_id}", + {"limits_override": {"storage": client_workspace_storage, "projects": 1, "api_allowed": True}}, + {"Content-Type": "application/json"}, + ) + + request.addfinalizer(teardown) return client @@ -78,11 +96,13 @@ def create_client(user, pwd): return MerginClient(SERVER_URL, login=user, password=pwd) -def create_workspace_for_client(mc: MerginClient, workspace_name=None): +def create_workspace_for_client(mc: MerginClient, workspace_name=None) -> str: + workspace_name = workspace_name or mc.username() try: - mc.create_workspace(workspace_name or mc.username()) + mc.create_workspace(workspace_name) except ClientError: - return + pass + return workspace_name def cleanup(mc, project, dirs): @@ -798,7 +818,6 @@ def test_available_workspace_storage(mcStorage): try: mcStorage.push_project(project_dir) except ClientError as e: - print("Pushe fail") # Expecting "You have reached a data limit" 400 server error msg. assert "You have reached a data limit" in str(e) got_right_err = True @@ -813,13 +832,6 @@ def test_available_workspace_storage(mcStorage): # remove dummy big file from a disk remove_folders([project_dir]) - # revert storage limit to default value - mcStorage.patch( - f"/v1/tests/workspaces/{client_workspace_id}", - {"limits_override": {"storage": current_storage, "projects": 1, "api_allowed": True}}, - {"Content-Type": "application/json"}, - ) - def test_available_storage_validation2(mc, mc2): """ @@ -2699,10 +2711,3 @@ def test_error_projects_limit_hit(mcStorage: MerginClient): assert e.value.http_error == 422 assert e.value.http_method == "POST" assert e.value.url == f"{mcStorage.url}v1/project/testpluginstorage" - - # back to original values... (1 project, api allowed ...) - mcStorage.patch( - f"/v1/tests/workspaces/{client_workspace_id}", - {"limits_override": {"storage": client_workspace_storage, "projects": 1, "api_allowed": True}}, - {"Content-Type": "application/json"}, - )