From a0450953d56009251c72ed8a05d8690def117a20 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Thu, 11 Sep 2025 18:06:27 +0200 Subject: [PATCH 01/10] add max of 100 files max 10GB of non versioned files max 5GB of versioned files --- mergin/client_push.py | 45 +++++++++++++++-------- mergin/common.py | 10 +++++- mergin/editor.py | 2 +- mergin/local_changes.py | 39 +++++++++++++++++++- mergin/merginproject.py | 14 ++++---- mergin/test/test_local_changes.py | 59 +++++++++++++++++++++++++++++++ 6 files changed, 145 insertions(+), 24 deletions(-) diff --git a/mergin/client_push.py b/mergin/client_push.py index 2400c959..188167db 100644 --- a/mergin/client_push.py +++ b/mergin/client_push.py @@ -24,7 +24,14 @@ from .local_changes import LocalChange, LocalChanges -from .common import UPLOAD_CHUNK_ATTEMPT_WAIT, UPLOAD_CHUNK_ATTEMPTS, UPLOAD_CHUNK_SIZE, ClientError, ErrorCode +from .common import ( + MAX_UPLOAD_VERSIONED_SIZE, + UPLOAD_CHUNK_ATTEMPT_WAIT, + UPLOAD_CHUNK_ATTEMPTS, + UPLOAD_CHUNK_SIZE, + MAX_UPLOAD_MEDIA_SIZE, + ClientError, +) from .merginproject import MerginProject from .editor import filter_changes from .utils import get_data_checksum @@ -296,7 +303,7 @@ def push_project_async(mc, directory) -> Optional[UploadJob]: mp.log.info(f"--- push {project_path} - nothing to do") return - mp.log.debug("push changes:\n" + pprint.pformat(changes)) + mp.log.debug("push changes:\n" + pprint.pformat(asdict(changes))) tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-") # If there are any versioned files (aka .gpkg) that are not updated through a diff, @@ -304,20 +311,15 @@ def push_project_async(mc, directory) -> Optional[UploadJob]: # That's because if there are pending transactions, checkpointing or switching from WAL mode # won't work, and we would end up with some changes left in -wal file which do not get # uploaded. The temporary copy using geodiff uses sqlite backup API and should copy everything. - for f in changes["updated"]: - if mp.is_versioned_file(f["path"]) and "diff" not in f: + for f in changes.updated: + if mp.is_versioned_file(f.path) and not f.diff: mp.copy_versioned_file_for_upload(f, tmp_dir.name) - for f in changes["added"]: - if mp.is_versioned_file(f["path"]): + for f in changes.added: + if mp.is_versioned_file(f.path): mp.copy_versioned_file_for_upload(f, tmp_dir.name) - local_changes = LocalChanges( - added=[LocalChange(**change) for change in changes["added"]], - updated=[LocalChange(**change) for change in changes["updated"]], - removed=[LocalChange(**change) for change in changes["removed"]], - ) - job = create_upload_job(mc, mp, local_changes, tmp_dir) + job = create_upload_job(mc, mp, changes, tmp_dir) return job @@ -471,7 +473,7 @@ def remove_diff_files(job: UploadJob) -> None: os.remove(diff_file) -def get_push_changes_batch(mc, mp: MerginProject) -> Tuple[dict, int]: +def get_push_changes_batch(mc, mp: MerginProject) -> Tuple[LocalChanges, int]: """ Get changes that need to be pushed to the server. """ @@ -479,4 +481,19 @@ def get_push_changes_batch(mc, mp: MerginProject) -> Tuple[dict, int]: project_role = mp.project_role() changes = filter_changes(mc, project_role, changes) - return changes, sum(len(v) for v in changes.values()) + local_changes = LocalChanges( + added=[LocalChange(**change) for change in changes["added"]], + updated=[LocalChange(**change) for change in changes["updated"]], + removed=[LocalChange(**change) for change in changes["removed"]], + ) + if local_changes.get_media_upload_size() > MAX_UPLOAD_MEDIA_SIZE: + raise ClientError( + f"Total size of media files to upload exceeds the maximum allowed size of {MAX_UPLOAD_MEDIA_SIZE / (1024**3)} GiB." + ) + + if local_changes.get_gpgk_upload_size() > MAX_UPLOAD_VERSIONED_SIZE: + raise ClientError( + f"Total size of GPKG files to upload exceeds the maximum allowed size of {MAX_UPLOAD_VERSIONED_SIZE / (1024**3)} GiB." + ) + + return local_changes, sum(len(v) for v in changes.values()) diff --git a/mergin/common.py b/mergin/common.py index 25b58c48..bc4c60ce 100644 --- a/mergin/common.py +++ b/mergin/common.py @@ -24,6 +24,12 @@ # seconds to wait between sync callback calls SYNC_CALLBACK_WAIT = 0.01 +# maximum size of media files able to upload in one push (in bytes) +MAX_UPLOAD_MEDIA_SIZE = 10 * (1024**3) + +# maximum size of GPKG files able to upload in one push (in bytes) +MAX_UPLOAD_VERSIONED_SIZE = 5 * (1024**3) + # default URL for submitting logs MERGIN_DEFAULT_LOGS_URL = "https://g4pfq226j0.execute-api.eu-west-1.amazonaws.com/mergin_client_log_submit" @@ -39,7 +45,9 @@ class ErrorCode(Enum): class ClientError(Exception): - def __init__(self, detail: str, url=None, server_code=None, server_response=None, http_error=None, http_method=None): + def __init__( + self, detail: str, url=None, server_code=None, server_response=None, http_error=None, http_method=None + ): self.detail = detail self.url = url self.http_error = http_error diff --git a/mergin/editor.py b/mergin/editor.py index b1dac863..bb8f1d18 100644 --- a/mergin/editor.py +++ b/mergin/editor.py @@ -1,7 +1,7 @@ from itertools import filterfalse from typing import Callable, Dict, List -from .utils import is_mergin_config, is_qgis_file, is_versioned_file +from .utils import is_qgis_file EDITOR_ROLE_NAME = "editor" diff --git a/mergin/local_changes.py b/mergin/local_changes.py index 511960cd..a73be299 100644 --- a/mergin/local_changes.py +++ b/mergin/local_changes.py @@ -1,6 +1,10 @@ from dataclasses import dataclass, field from datetime import datetime -from typing import Dict, Optional, List, Tuple +from typing import Optional, List, Tuple + +from .utils import is_versioned_file + +MAX_UPLOAD_CHANGES = 100 @dataclass @@ -55,6 +59,18 @@ class LocalChanges: updated: List[LocalChange] = field(default_factory=list) removed: List[LocalChange] = field(default_factory=list) + def __post_init__(self): + """ + Enforce a limit of changes combined from `added` and `updated`. + """ + total_changes = len(self.get_upload_changes()) + if total_changes > MAX_UPLOAD_CHANGES: + # Calculate how many changes to keep from `added` and `updated` + added_limit = min(len(self.added), MAX_UPLOAD_CHANGES) + updated_limit = MAX_UPLOAD_CHANGES - added_limit + self.added = self.added[:added_limit] + self.updated = self.updated[:updated_limit] + def to_server_payload(self) -> dict: return { "added": [change.to_server_data() for change in self.added], @@ -96,3 +112,24 @@ def update_chunks(self, server_chunks: List[Tuple[str, str]]) -> None: for change in self.updated: change.chunks = self._map_unique_chunks(change.chunks, server_chunks) + + def get_media_upload_size(self) -> int: + """ + Calculate the total size of media files in added and updated changes. + """ + total_size = 0 + for change in self.get_upload_changes(): + if not is_versioned_file(change.path): + total_size += change.size + return total_size + + def get_gpgk_upload_size(self) -> int: + """ + Calculate the total size of gpgk files in added and updated changes. + Do not calculate diffs (only new or overwriten files). + """ + total_size = 0 + for change in self.get_upload_changes(): + if is_versioned_file(change.path) and not change.diff: + total_size += change.size + return total_size diff --git a/mergin/merginproject.py b/mergin/merginproject.py index 72b1449c..61b417e5 100644 --- a/mergin/merginproject.py +++ b/mergin/merginproject.py @@ -21,7 +21,7 @@ conflicted_copy_file_name, edit_conflict_file_name, ) - +from .local_changes import LocalChange this_dir = os.path.dirname(os.path.realpath(__file__)) @@ -470,20 +470,20 @@ def get_push_changes(self): changes["updated"] = [f for f in changes["updated"] if f not in not_updated] return changes - def copy_versioned_file_for_upload(self, f, tmp_dir): + def copy_versioned_file_for_upload(self, f: LocalChange, tmp_dir: str) -> str: """ Make a temporary copy of the versioned file using geodiff, to make sure that we have full content in a single file (nothing left in WAL journal) """ - path = f["path"] + path = f.path self.log.info("Making a temporary copy (full upload): " + path) tmp_file = os.path.join(tmp_dir, path) os.makedirs(os.path.dirname(tmp_file), exist_ok=True) self.geodiff.make_copy_sqlite(self.fpath(path), tmp_file) - f["size"] = os.path.getsize(tmp_file) - f["checksum"] = generate_checksum(tmp_file) - f["chunks"] = [str(uuid.uuid4()) for i in range(math.ceil(f["size"] / UPLOAD_CHUNK_SIZE))] - f["upload_file"] = tmp_file + f.size = os.path.getsize(tmp_file) + f.checksum = generate_checksum(tmp_file) + f.chunks = [str(uuid.uuid4()) for i in range(math.ceil(f.size / UPLOAD_CHUNK_SIZE))] + f.upload_file = tmp_file return tmp_file def get_list_of_push_changes(self, push_changes): diff --git a/mergin/test/test_local_changes.py b/mergin/test/test_local_changes.py index 6fe18388..8c263c7d 100644 --- a/mergin/test/test_local_changes.py +++ b/mergin/test/test_local_changes.py @@ -118,3 +118,62 @@ def test_local_changes_get_upload_changes(): assert len(upload_changes) == 2 # Only added and updated should be included assert upload_changes[0].path == "file1.txt" # First change is from added assert upload_changes[1].path == "file2.txt" # Second change is from updated + + +def test_local_changes_get_media_upload_size(): + """Test the get_media_upload_size method of LocalChanges.""" + # Create sample LocalChange instances + added = [ + LocalChange(path="file1.txt", checksum="abc123", size=1024, mtime=datetime.now()), + LocalChange(path="file2.jpg", checksum="xyz789", size=2048, mtime=datetime.now()), + ] + updated = [ + LocalChange(path="file3.mp4", checksum="lmn456", size=5120, mtime=datetime.now()), + LocalChange(path="file4.gpkg", checksum="opq123", size=1024, mtime=datetime.now()), + ] + + # Initialize LocalChanges + local_changes = LocalChanges(added=added, updated=updated) + + # Call get_media_upload_size + media_size = local_changes.get_media_upload_size() + + # Assertions + assert media_size == 8192 # Only non-versioned files (txt, jpg, mp4) are included + + +def test_local_changes_get_gpgk_upload_size(): + """Test the get_gpgk_upload_size method of LocalChanges.""" + # Create sample LocalChange instances + added = [ + LocalChange(path="file1.gpkg", checksum="abc123", size=1024, mtime=datetime.now()), + LocalChange(path="file2.gpkg", checksum="xyz789", size=2048, mtime=datetime.now(), diff={"path": "diff1"}), + ] + updated = [ + LocalChange(path="file3.gpkg", checksum="lmn456", size=5120, mtime=datetime.now()), + LocalChange(path="file4.txt", checksum="opq123", size=1024, mtime=datetime.now()), + ] + + # Initialize LocalChanges + local_changes = LocalChanges(added=added, updated=updated) + + # Call get_gpgk_upload_size + gpkg_size = local_changes.get_gpgk_upload_size() + + # Assertions + assert gpkg_size == 6144 # Only GPKG files without diffs are included + + +def test_local_changes_post_init(): + """Test the __post_init__ method of LocalChanges.""" + # Create more than MAX_UPLOAD_CHANGES changes + added = [LocalChange(path=f"file{i}.txt", checksum="abc123", size=1024, mtime=datetime.now()) for i in range(80)] + updated = [LocalChange(path=f"file{i}.txt", checksum="xyz789", size=2048, mtime=datetime.now()) for i in range(21)] + + # Initialize LocalChanges + local_changes = LocalChanges(added=added, updated=updated) + + # Assertions + assert len(local_changes.added) == 80 # All 80 added changes are included + assert len(local_changes.updated) == 20 # Only 20 updated changes are included to respect the limit + assert len(local_changes.added) + len(local_changes.updated) == 100 # Total is limited to MAX_UPLOAD_CHANGES From 50b62244b53c33e4fdc221f452c7e80069c5ffcd Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Thu, 11 Sep 2025 18:09:05 +0200 Subject: [PATCH 02/10] black --- mergin/test/test_common.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mergin/test/test_common.py b/mergin/test/test_common.py index d86229e1..7a1dbbdf 100644 --- a/mergin/test/test_common.py +++ b/mergin/test/test_common.py @@ -1,5 +1,6 @@ from ..common import ClientError, ErrorCode + def test_client_error_is_blocked_sync(): """Test the is_blocked_sync method of ClientError.""" error = ClientError(detail="", server_code=None) @@ -12,6 +13,7 @@ def test_client_error_is_blocked_sync(): error.server_code = ErrorCode.ProjectVersionExists.value assert error.is_blocking_sync() is True + def test_client_error_is_rate_limit(): """Test the is_rate_limit method of ClientError.""" error = ClientError(detail="", http_error=None) @@ -21,6 +23,7 @@ def test_client_error_is_rate_limit(): error.http_error = 429 assert error.is_rate_limit() is True + def test_client_error_is_retryable_sync(): """Test the is_retryable_sync method of ClientError.""" error = ClientError(detail="", server_code=None, http_error=None) @@ -43,4 +46,4 @@ def test_client_error_is_retryable_sync(): error.http_error = 500 assert error.is_retryable_sync() is False error.http_error = 429 - assert error.is_retryable_sync() is True \ No newline at end of file + assert error.is_retryable_sync() is True From 0fc2a9eff0cf1781aa4fdbd5ef5255e36306e0c1 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Fri, 12 Sep 2025 10:02:30 +0200 Subject: [PATCH 03/10] Find just one file over limit in transaction --- mergin/client_push.py | 11 +++-- mergin/common.py | 4 +- mergin/local_changes.py | 25 +++++----- mergin/test/test_client.py | 18 +++++++ mergin/test/test_local_changes.py | 79 +++++++++++++++++++++---------- 5 files changed, 94 insertions(+), 43 deletions(-) diff --git a/mergin/client_push.py b/mergin/client_push.py index 188167db..bfe8132b 100644 --- a/mergin/client_push.py +++ b/mergin/client_push.py @@ -486,14 +486,17 @@ def get_push_changes_batch(mc, mp: MerginProject) -> Tuple[LocalChanges, int]: updated=[LocalChange(**change) for change in changes["updated"]], removed=[LocalChange(**change) for change in changes["removed"]], ) - if local_changes.get_media_upload_size() > MAX_UPLOAD_MEDIA_SIZE: + + over_limit_media = local_changes.get_media_upload_over_size(MAX_UPLOAD_MEDIA_SIZE) + if over_limit_media: raise ClientError( - f"Total size of media files to upload exceeds the maximum allowed size of {MAX_UPLOAD_MEDIA_SIZE / (1024**3)} GiB." + f"File {over_limit_media.path} to upload exceeds the maximum allowed size of {MAX_UPLOAD_MEDIA_SIZE / (1024**3)} GB." ) - if local_changes.get_gpgk_upload_size() > MAX_UPLOAD_VERSIONED_SIZE: + over_limit_gpkg = local_changes.get_gpgk_upload_over_size(MAX_UPLOAD_VERSIONED_SIZE) + if over_limit_gpkg: raise ClientError( - f"Total size of GPKG files to upload exceeds the maximum allowed size of {MAX_UPLOAD_VERSIONED_SIZE / (1024**3)} GiB." + f"Geopackage {over_limit_gpkg.path} to upload exceeds the maximum allowed size of {MAX_UPLOAD_VERSIONED_SIZE / (1024**3)} GB." ) return local_changes, sum(len(v) for v in changes.values()) diff --git a/mergin/common.py b/mergin/common.py index bc4c60ce..25df4f4d 100644 --- a/mergin/common.py +++ b/mergin/common.py @@ -24,10 +24,10 @@ # seconds to wait between sync callback calls SYNC_CALLBACK_WAIT = 0.01 -# maximum size of media files able to upload in one push (in bytes) +# maximum size of media file able to upload in one push (in bytes) MAX_UPLOAD_MEDIA_SIZE = 10 * (1024**3) -# maximum size of GPKG files able to upload in one push (in bytes) +# maximum size of GPKG file able to upload in one push (in bytes) MAX_UPLOAD_VERSIONED_SIZE = 5 * (1024**3) # default URL for submitting logs diff --git a/mergin/local_changes.py b/mergin/local_changes.py index a73be299..06c5872d 100644 --- a/mergin/local_changes.py +++ b/mergin/local_changes.py @@ -113,23 +113,22 @@ def update_chunks(self, server_chunks: List[Tuple[str, str]]) -> None: for change in self.updated: change.chunks = self._map_unique_chunks(change.chunks, server_chunks) - def get_media_upload_size(self) -> int: + def get_media_upload_over_size(self, size_limit: int) -> Optional[LocalChange]: """ - Calculate the total size of media files in added and updated changes. + Find the first media file in added and updated changes that exceeds the size limit. + :return: The first LocalChange that exceeds the size limit, or None if no such file exists. """ - total_size = 0 for change in self.get_upload_changes(): - if not is_versioned_file(change.path): - total_size += change.size - return total_size + if not is_versioned_file(change.path) and change.size > size_limit: + return change - def get_gpgk_upload_size(self) -> int: + def get_gpgk_upload_over_size(self, size_limit: int) -> Optional[LocalChange]: """ - Calculate the total size of gpgk files in added and updated changes. - Do not calculate diffs (only new or overwriten files). + Find the first GPKG file in added and updated changes that exceeds the size limit. + Do not include diffs (only new or overwritten files). + :param size_limit: The size limit in bytes. + :return: The first LocalChange that exceeds the size limit, or None if no such file exists. """ - total_size = 0 for change in self.get_upload_changes(): - if is_versioned_file(change.path) and not change.diff: - total_size += change.size - return total_size + if is_versioned_file(change.path) and not change.diff and change.size > size_limit: + return change diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index fdd988a3..a529d027 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -3211,3 +3211,21 @@ def test_client_project_sync_retry(mc): with pytest.raises(ClientError): mc.sync_project(project_dir) assert mock_push_project_async.call_count == 2 + +def test_push_file_limits(mc): + test_project = "test_push_file_limits" + project = API_USER + "/" + test_project + project_dir = os.path.join(TMP_DIR, test_project) + cleanup(mc, project, [project_dir]) + mc.create_project(test_project) + mc.download_project(project, project_dir) + shutil.copy(os.path.join(TEST_DATA_DIR, "base.gpkg"), project_dir) + # setting to some minimal value to mock limit hit + with patch("mergin.client_push.MAX_UPLOAD_VERSIONED_SIZE", 1): + with pytest.raises(ClientError, match=f"base.gpkg to upload exceeds the maximum allowed size of {1/1024**3}"): + mc.push_project(project_dir) + + shutil.copy(os.path.join(TEST_DATA_DIR, "test.txt"), project_dir) + with patch("mergin.client_push.MAX_UPLOAD_MEDIA_SIZE", 1): + with pytest.raises(ClientError, match=f"test.txt to upload exceeds the maximum allowed size of {1/1024**3}"): + mc.push_project(project_dir) diff --git a/mergin/test/test_local_changes.py b/mergin/test/test_local_changes.py index 8c263c7d..dceab27a 100644 --- a/mergin/test/test_local_changes.py +++ b/mergin/test/test_local_changes.py @@ -1,6 +1,6 @@ from datetime import datetime -from ..local_changes import LocalChange, LocalChanges +from ..local_changes import LocalChange, LocalChanges, MAX_UPLOAD_CHANGES def test_local_changes_from_dict(): @@ -120,60 +120,91 @@ def test_local_changes_get_upload_changes(): assert upload_changes[1].path == "file2.txt" # Second change is from updated -def test_local_changes_get_media_upload_size(): - """Test the get_media_upload_size method of LocalChanges.""" +def test_local_changes_get_media_upload_over_size(): + """Test the get_media_upload_file method of LocalChanges.""" + # Define constants + SIZE_LIMIT_MB = 10 + SIZE_LIMIT_BYTES = SIZE_LIMIT_MB * 1024 * 1024 + SMALL_FILE_SIZE = 1024 + LARGE_FILE_SIZE = 15 * 1024 * 1024 + # Create sample LocalChange instances added = [ - LocalChange(path="file1.txt", checksum="abc123", size=1024, mtime=datetime.now()), - LocalChange(path="file2.jpg", checksum="xyz789", size=2048, mtime=datetime.now()), + LocalChange(path="file1.txt", checksum="abc123", size=SMALL_FILE_SIZE, mtime=datetime.now()), + LocalChange(path="file2.jpg", checksum="xyz789", size=LARGE_FILE_SIZE, mtime=datetime.now()), # Over limit ] updated = [ - LocalChange(path="file3.mp4", checksum="lmn456", size=5120, mtime=datetime.now()), - LocalChange(path="file4.gpkg", checksum="opq123", size=1024, mtime=datetime.now()), + LocalChange(path="file3.mp4", checksum="lmn456", size=5 * 1024 * 1024, mtime=datetime.now()), + LocalChange(path="file4.gpkg", checksum="opq123", size=SMALL_FILE_SIZE, mtime=datetime.now()), ] # Initialize LocalChanges local_changes = LocalChanges(added=added, updated=updated) - # Call get_media_upload_size - media_size = local_changes.get_media_upload_size() + # Call get_media_upload_file with a size limit + media_file = local_changes.get_media_upload_over_size(SIZE_LIMIT_BYTES) # Assertions - assert media_size == 8192 # Only non-versioned files (txt, jpg, mp4) are included + assert media_file is not None + assert media_file.path == "file2.jpg" # The first file over the limit + assert media_file.size == LARGE_FILE_SIZE + +def test_local_changes_get_gpgk_upload_over_size(): + """Test the get_gpgk_upload_file method of LocalChanges.""" + # Define constants + SIZE_LIMIT_MB = 10 + SIZE_LIMIT_BYTES = SIZE_LIMIT_MB * 1024 * 1024 + SMALL_FILE_SIZE = 1024 + LARGE_FILE_SIZE = 15 * 1024 * 1024 -def test_local_changes_get_gpgk_upload_size(): - """Test the get_gpgk_upload_size method of LocalChanges.""" # Create sample LocalChange instances added = [ - LocalChange(path="file1.gpkg", checksum="abc123", size=1024, mtime=datetime.now()), - LocalChange(path="file2.gpkg", checksum="xyz789", size=2048, mtime=datetime.now(), diff={"path": "diff1"}), + LocalChange(path="file1.gpkg", checksum="abc123", size=SMALL_FILE_SIZE, mtime=datetime.now()), + LocalChange( + path="file2.gpkg", checksum="xyz789", size=LARGE_FILE_SIZE, mtime=datetime.now(), diff=None + ), # Over limit ] updated = [ - LocalChange(path="file3.gpkg", checksum="lmn456", size=5120, mtime=datetime.now()), - LocalChange(path="file4.txt", checksum="opq123", size=1024, mtime=datetime.now()), + LocalChange(path="file3.gpkg", checksum="lmn456", size=5 * 1024 * 1024, mtime=datetime.now()), + LocalChange(path="file4.txt", checksum="opq123", size=SMALL_FILE_SIZE, mtime=datetime.now()), ] # Initialize LocalChanges local_changes = LocalChanges(added=added, updated=updated) - # Call get_gpgk_upload_size - gpkg_size = local_changes.get_gpgk_upload_size() + # Call get_gpgk_upload_file with a size limit + gpkg_file = local_changes.get_gpgk_upload_over_size(SIZE_LIMIT_BYTES) # Assertions - assert gpkg_size == 6144 # Only GPKG files without diffs are included + assert gpkg_file is not None + assert gpkg_file.path == "file2.gpkg" # The first GPKG file over the limit + assert gpkg_file.size == LARGE_FILE_SIZE + assert gpkg_file.diff is None # Ensure it doesn't include diffs def test_local_changes_post_init(): """Test the __post_init__ method of LocalChanges.""" + # Define constants + ADDED_COUNT = 80 + UPDATED_COUNT = 21 + SMALL_FILE_SIZE = 1024 + LARGE_FILE_SIZE = 2048 + # Create more than MAX_UPLOAD_CHANGES changes - added = [LocalChange(path=f"file{i}.txt", checksum="abc123", size=1024, mtime=datetime.now()) for i in range(80)] - updated = [LocalChange(path=f"file{i}.txt", checksum="xyz789", size=2048, mtime=datetime.now()) for i in range(21)] + added = [ + LocalChange(path=f"file{i}.txt", checksum="abc123", size=SMALL_FILE_SIZE, mtime=datetime.now()) + for i in range(ADDED_COUNT) + ] + updated = [ + LocalChange(path=f"file{i}.txt", checksum="xyz789", size=LARGE_FILE_SIZE, mtime=datetime.now()) + for i in range(UPDATED_COUNT) + ] # Initialize LocalChanges local_changes = LocalChanges(added=added, updated=updated) # Assertions - assert len(local_changes.added) == 80 # All 80 added changes are included - assert len(local_changes.updated) == 20 # Only 20 updated changes are included to respect the limit - assert len(local_changes.added) + len(local_changes.updated) == 100 # Total is limited to MAX_UPLOAD_CHANGES + assert len(local_changes.added) == ADDED_COUNT # All added changes are included + assert len(local_changes.updated) == MAX_UPLOAD_CHANGES - ADDED_COUNT # Only enough updated changes are included + assert len(local_changes.added) + len(local_changes.updated) == MAX_UPLOAD_CHANGES # Total is limited From 607e148e997b4fefef7460c62453b9b7a1caa8cd Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Fri, 12 Sep 2025 10:05:54 +0200 Subject: [PATCH 04/10] black swan --- mergin/test/test_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index a529d027..4f49d090 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -3212,6 +3212,7 @@ def test_client_project_sync_retry(mc): mc.sync_project(project_dir) assert mock_push_project_async.call_count == 2 + def test_push_file_limits(mc): test_project = "test_push_file_limits" project = API_USER + "/" + test_project @@ -3224,7 +3225,7 @@ def test_push_file_limits(mc): with patch("mergin.client_push.MAX_UPLOAD_VERSIONED_SIZE", 1): with pytest.raises(ClientError, match=f"base.gpkg to upload exceeds the maximum allowed size of {1/1024**3}"): mc.push_project(project_dir) - + shutil.copy(os.path.join(TEST_DATA_DIR, "test.txt"), project_dir) with patch("mergin.client_push.MAX_UPLOAD_MEDIA_SIZE", 1): with pytest.raises(ClientError, match=f"test.txt to upload exceeds the maximum allowed size of {1/1024**3}"): From 5e4c38ad342299bbd015160fd36c479380066bd2 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Fri, 12 Sep 2025 10:06:46 +0200 Subject: [PATCH 05/10] debug test --- .github/workflows/autotests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autotests.yml b/.github/workflows/autotests.yml index fc658a58..bc7631fb 100644 --- a/.github/workflows/autotests.yml +++ b/.github/workflows/autotests.yml @@ -28,7 +28,7 @@ jobs: - name: Run tests run: | - pytest --cov=mergin --cov-report=lcov mergin/test/ + pytest --cov=mergin --cov-report=lcov mergin/test/ -k gpkg_schema - name: Submit coverage to Coveralls uses: coverallsapp/github-action@v2 From 7e660a2f103d177741c927bf2851c5916152c783 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Fri, 12 Sep 2025 10:07:58 +0200 Subject: [PATCH 06/10] downgrade py --- .github/workflows/autotests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autotests.yml b/.github/workflows/autotests.yml index bc7631fb..8daccdeb 100644 --- a/.github/workflows/autotests.yml +++ b/.github/workflows/autotests.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.10' - name: Install python package dependencies run: | From 309315db5930cd29fd72d3a03dbf388e40c1376e Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Fri, 12 Sep 2025 10:11:17 +0200 Subject: [PATCH 07/10] back to normal py --- .github/workflows/autotests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autotests.yml b/.github/workflows/autotests.yml index 8daccdeb..bc7631fb 100644 --- a/.github/workflows/autotests.yml +++ b/.github/workflows/autotests.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: '3.10' + python-version: '3.x' - name: Install python package dependencies run: | From e1675cc358044a7bf9b412c3bdb19b769fc0810f Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Fri, 12 Sep 2025 10:15:25 +0200 Subject: [PATCH 08/10] close conns properly --- mergin/test/test_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index 4f49d090..7edc1941 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -1364,6 +1364,7 @@ def _create_test_table(db_file): cursor.execute("CREATE TABLE test (fid SERIAL, txt TEXT);") cursor.execute("INSERT INTO test VALUES (123, 'hello');") cursor.execute("COMMIT;") + con.close() def _create_spatial_table(db_file): @@ -1479,6 +1480,7 @@ def test_push_gpkg_schema_change(mc): # open a connection and keep it open (qgis does this with a pool of connections too) acon2 = AnotherSqliteConn(test_gpkg) acon2.run("select count(*) from simple;") + acon2.close() # add a new table to ensure that geodiff will fail due to unsupported change # (this simulates an independent reader/writer like GDAL) From 678fa8d297c3577a8fef0d89e27d930506da654f Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Fri, 12 Sep 2025 10:18:29 +0200 Subject: [PATCH 09/10] double con close get rid of failing test on app.dev without push --- mergin/test/test_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index 7edc1941..aa499157 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -1480,7 +1480,6 @@ def test_push_gpkg_schema_change(mc): # open a connection and keep it open (qgis does this with a pool of connections too) acon2 = AnotherSqliteConn(test_gpkg) acon2.run("select count(*) from simple;") - acon2.close() # add a new table to ensure that geodiff will fail due to unsupported change # (this simulates an independent reader/writer like GDAL) @@ -3071,6 +3070,9 @@ def test_validate_auth(mc: MerginClient): def test_uploaded_chunks_cache(mc): """Create a new project, download it, add a file and then do sync - it should not fail""" + # This test does not make sense on servers without v2 push + if not mc.server_features().get("v2_push_enabled"): + return test_project = "test_uploaded_chunks_cache" project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) # primary project dir for updates From 027ba431c7d6ec28bc4ca5017c04bf1471b4c12e Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Fri, 12 Sep 2025 10:21:36 +0200 Subject: [PATCH 10/10] enable all tests --- .github/workflows/autotests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autotests.yml b/.github/workflows/autotests.yml index bc7631fb..fc658a58 100644 --- a/.github/workflows/autotests.yml +++ b/.github/workflows/autotests.yml @@ -28,7 +28,7 @@ jobs: - name: Run tests run: | - pytest --cov=mergin --cov-report=lcov mergin/test/ -k gpkg_schema + pytest --cov=mergin --cov-report=lcov mergin/test/ - name: Submit coverage to Coveralls uses: coverallsapp/github-action@v2