From cca41f4ad65cbd6beecfb17a087f7931785982c1 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 12 Jun 2025 12:50:54 +0200 Subject: [PATCH 1/8] allow sending logs from client --- mergin/client.py | 71 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/mergin/client.py b/mergin/client.py index 192f7e31..e27846b1 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -40,6 +40,8 @@ this_dir = os.path.dirname(os.path.realpath(__file__)) json_headers = {"Content-Type": "application/json"} +MERGIN_DEFAULT_LOGS_URL = "https://g4pfq226j0.execute-api.eu-west-1.amazonaws.com/mergin_client_log_submit" + class TokenError(Exception): pass @@ -1360,3 +1362,72 @@ def remove_project_collaborator(self, project_id: str, user_id: int): Remove a user from project collaborators """ self.delete(f"v2/projects/{project_id}/collaborators/{user_id}") + + def server_config(self) -> dict: + """Get server configuration as dictionary.""" + response = self.get("/config") + return json.load(response) + + def server_version_newer_or_equal_than(self, version: str) -> bool: + """Check if the server version is newer or equal to the specified version.""" + required_major, required_minor, required_fix = version.split(".") + server_version = self.server_version() + if server_version: + server_major, server_minor, server_fix = server_version.split(".") + return ( + int(server_major) > int(required_major) + or (int(server_major) == int(required_major) and int(server_minor) > int(required_minor)) + or ( + int(server_major) == int(required_major) + and int(server_minor) == int(required_minor) + and int(server_fix) >= int(required_fix) + ) + ) + return False + + def send_logs( + self, + logfile: str, + global_log_file: typing.Optional[str] = None, + application: typing.Optional[str] = None, + meta: typing.Optional[str] = None, + ): + """Send logs to configured server or the default Mergin server.""" + + if application is None: + application = "mergin-client-{}".format(__version__) + + params = {"app": application, "username": self.username()} + + config = self.server_config() + diagnostic_logs_url = config.get("diagnostic_logs_url", None) + + if self.server_version_newer_or_equal_than("2023.4.1") and ( + diagnostic_logs_url is None or diagnostic_logs_url == "" + ): + url = self.url() + "?" + urllib.parse.urlencode(params) + else: + url = MERGIN_DEFAULT_LOGS_URL + "?" + urllib.parse.urlencode(params) + + if meta is None: + meta = "Python API Client\nSystem: {} \nMergin Maps URL: {} \nMergin Maps user: {} \n--------------------------------\n\n".format( + platform.system(), self.url, self.username() + ) + + global_logs = b"" + if global_log_file and os.path.exists(global_log_file): + with open(global_log_file, "rb") as f: + if os.path.getsize(global_log_file) > 100 * 1024: + f.seek(-100 * 1024, os.SEEK_END) + global_logs = f.read() + b"\n--------------------------------\n\n" + + with open(logfile, "rb") as f: + if os.path.getsize(logfile) > 512 * 1024: + f.seek(-512 * 1024, os.SEEK_END) + logs = f.read() + + payload = meta.encode() + global_logs + logs + header = {"content-type": "text/plain"} + + request = urllib.request.Request(url, data=payload, headers=header) + return self._do_request(request) From 0100286bd13dd864aaa879e088f87c2b091809fb Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 12 Jun 2025 12:54:57 +0200 Subject: [PATCH 2/8] rename variable --- mergin/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mergin/client.py b/mergin/client.py index e27846b1..78b11087 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -1368,9 +1368,9 @@ def server_config(self) -> dict: response = self.get("/config") return json.load(response) - def server_version_newer_or_equal_than(self, version: str) -> bool: + def server_version_newer_or_equal_than(self, required_version: str) -> bool: """Check if the server version is newer or equal to the specified version.""" - required_major, required_minor, required_fix = version.split(".") + required_major, required_minor, required_fix = required_version.split(".") server_version = self.server_version() if server_version: server_major, server_minor, server_fix = server_version.split(".") From b2f0c7ae9bc10eb7b25804fb96639d0d3d7adf16 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 12 Jun 2025 13:10:33 +0200 Subject: [PATCH 3/8] use proper function to check version --- mergin/client.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/mergin/client.py b/mergin/client.py index 78b11087..80f2cd16 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -1368,23 +1368,6 @@ def server_config(self) -> dict: response = self.get("/config") return json.load(response) - def server_version_newer_or_equal_than(self, required_version: str) -> bool: - """Check if the server version is newer or equal to the specified version.""" - required_major, required_minor, required_fix = required_version.split(".") - server_version = self.server_version() - if server_version: - server_major, server_minor, server_fix = server_version.split(".") - return ( - int(server_major) > int(required_major) - or (int(server_major) == int(required_major) and int(server_minor) > int(required_minor)) - or ( - int(server_major) == int(required_major) - and int(server_minor) == int(required_minor) - and int(server_fix) >= int(required_fix) - ) - ) - return False - def send_logs( self, logfile: str, @@ -1402,7 +1385,7 @@ def send_logs( config = self.server_config() diagnostic_logs_url = config.get("diagnostic_logs_url", None) - if self.server_version_newer_or_equal_than("2023.4.1") and ( + if is_version_acceptable(self.server_version(), "2025.4.1") and ( diagnostic_logs_url is None or diagnostic_logs_url == "" ): url = self.url() + "?" + urllib.parse.urlencode(params) From d706fe1f4ee6581917ea18410c9dec8834db3188 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 12 Jun 2025 14:13:15 +0200 Subject: [PATCH 4/8] use self.post() if sending on server --- mergin/client.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/mergin/client.py b/mergin/client.py index 80f2cd16..bbe58f94 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -1385,10 +1385,12 @@ def send_logs( config = self.server_config() diagnostic_logs_url = config.get("diagnostic_logs_url", None) + use_server_api = False if is_version_acceptable(self.server_version(), "2025.4.1") and ( diagnostic_logs_url is None or diagnostic_logs_url == "" ): - url = self.url() + "?" + urllib.parse.urlencode(params) + url = "v2/diagnostic-logs" + "?" + urllib.parse.urlencode(params) + use_server_api = True else: url = MERGIN_DEFAULT_LOGS_URL + "?" + urllib.parse.urlencode(params) @@ -1412,5 +1414,8 @@ def send_logs( payload = meta.encode() + global_logs + logs header = {"content-type": "text/plain"} - request = urllib.request.Request(url, data=payload, headers=header) - return self._do_request(request) + if use_server_api: + return self.post(url, data=payload, headers=header) + else: + request = urllib.request.Request(url, data=payload, headers=header) + return self._do_request(request) From 3106f3e81c050d7efa5a8fe50de9e252d8a8f370 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 12 Jun 2025 14:18:01 +0200 Subject: [PATCH 5/8] stored predefined values in common --- mergin/client.py | 8 +++----- mergin/common.py | 5 +++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/mergin/client.py b/mergin/client.py index bbe58f94..599a7a40 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -19,7 +19,7 @@ from typing import List -from .common import ClientError, LoginError, WorkspaceRole, ProjectRole +from .common import ClientError, LoginError, WorkspaceRole, ProjectRole, LOG_FILE_SIZE_TO_SEND, MERGIN_DEFAULT_LOGS_URL from .merginproject import MerginProject from .client_pull import ( download_file_finalize, @@ -40,8 +40,6 @@ this_dir = os.path.dirname(os.path.realpath(__file__)) json_headers = {"Content-Type": "application/json"} -MERGIN_DEFAULT_LOGS_URL = "https://g4pfq226j0.execute-api.eu-west-1.amazonaws.com/mergin_client_log_submit" - class TokenError(Exception): pass @@ -1402,8 +1400,8 @@ def send_logs( global_logs = b"" if global_log_file and os.path.exists(global_log_file): with open(global_log_file, "rb") as f: - if os.path.getsize(global_log_file) > 100 * 1024: - f.seek(-100 * 1024, os.SEEK_END) + if os.path.getsize(global_log_file) > LOG_FILE_SIZE_TO_SEND: + f.seek(-LOG_FILE_SIZE_TO_SEND, os.SEEK_END) global_logs = f.read() + b"\n--------------------------------\n\n" with open(logfile, "rb") as f: diff --git a/mergin/common.py b/mergin/common.py index 191c1753..9e65a6ce 100644 --- a/mergin/common.py +++ b/mergin/common.py @@ -6,6 +6,11 @@ # there is an upper limit for chunk size on server, ideally should be requested from there once implemented UPLOAD_CHUNK_SIZE = 10 * 1024 * 1024 +# size of the log file part to send (if file is larger only this size from end will be sent) +LOG_FILE_SIZE_TO_SEND = 100 * 1024 + +# default URL for submitting logs +MERGIN_DEFAULT_LOGS_URL = "https://g4pfq226j0.execute-api.eu-west-1.amazonaws.com/mergin_client_log_submit" this_dir = os.path.dirname(os.path.realpath(__file__)) From 46ec1041e929b0e5b93b28b018b608f60c02b467 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 12 Jun 2025 14:22:01 +0200 Subject: [PATCH 6/8] add test --- mergin/test/test_client.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index d636dd85..69b51da8 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -2833,3 +2833,15 @@ def test_access_management(mc: MerginClient, mc2: MerginClient): with pytest.raises(ClientError) as exc_info: mc.remove_workspace_member(workspace_id, new_user["id"]) assert exc_info.value.http_error == 404 + + +def test_server_config(mc: MerginClient): + """Test retrieving server configuration and some keys.""" + config = mc.server_config() + + assert config + assert isinstance(config, dict) + + assert "server_type" in config + assert "version" in config + assert "server_configured" in config From c4e4b5b24528aac9fdc755e4d7c460b9670913fe Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 12 Jun 2025 16:09:41 +0200 Subject: [PATCH 7/8] use server provided url --- mergin/client.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mergin/client.py b/mergin/client.py index 599a7a40..f7ff4de9 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -1390,7 +1390,11 @@ def send_logs( url = "v2/diagnostic-logs" + "?" + urllib.parse.urlencode(params) use_server_api = True else: - url = MERGIN_DEFAULT_LOGS_URL + "?" + urllib.parse.urlencode(params) + if diagnostic_logs_url: + url = diagnostic_logs_url + "?" + urllib.parse.urlencode(params) + else: + # fallback to default logs URL + url = MERGIN_DEFAULT_LOGS_URL + "?" + urllib.parse.urlencode(params) if meta is None: meta = "Python API Client\nSystem: {} \nMergin Maps URL: {} \nMergin Maps user: {} \n--------------------------------\n\n".format( From 7d5cc2cf9ae1189e8f4eef06516758b37d56aea8 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 13 Jun 2025 11:17:57 +0200 Subject: [PATCH 8/8] test sending logs --- mergin/test/test_client.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index 69b51da8..31de795f 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -2845,3 +2845,29 @@ def test_server_config(mc: MerginClient): assert "server_type" in config assert "version" in config assert "server_configured" in config + + +def test_send_logs(mc: MerginClient, monkeypatch): + """Test that logs can be send to the server.""" + test_project = "test_logs_send" + project = API_USER + "/" + test_project + project_dir = os.path.join(TMP_DIR, test_project) + + cleanup(mc, project, [project_dir]) + # prepare local project + shutil.copytree(TEST_DATA_DIR, project_dir) + + # create remote project + mc.create_project_and_push(project, directory=project_dir) + + # patch mc.server_config() to return empty config which means that logs will be send to the server + # but it is not configured to accept them so client error with message will be raised + def server_config(self): + return {} + + monkeypatch.setattr(mc, "server_config", server_config.__get__(mc)) + + logs_path = os.path.join(project_dir, ".mergin", "client-log.txt") + + with pytest.raises(ClientError, match="The requested URL was not found on the server"): + mc.send_logs(logs_path)