From 4c186f56d23e4f56b41f80846e7d82a40e224f67 Mon Sep 17 00:00:00 2001 From: ansipunk Date: Tue, 11 Feb 2025 02:23:40 +0500 Subject: [PATCH 1/5] MySQL backend --- .github/workflows/test.yaml | 24 ++++++++-- LICENSE.txt | 2 +- README.md | 21 ++++++--- based/__init__.py | 2 +- based/backends/__init__.py | 91 ++++++++++++++++++------------------- based/backends/mysql.py | 91 +++++++++++++++++++++++++++++++++++++ based/database.py | 9 +++- pyproject.toml | 10 ++-- tests/conftest.py | 12 +++-- 9 files changed, 196 insertions(+), 66 deletions(-) create mode 100644 based/backends/mysql.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b13b36e..23f0d2b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ jobs: strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] services: postgres: @@ -29,6 +29,14 @@ jobs: - 5432:5432 options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5 + mysql: + image: mariadb:latest + env: + MARIADB_ROOT_PASSWORD: based + MARIADB_DB: based + ports: + - 5432:5432 + steps: - uses: "actions/checkout@v4" - uses: "actions/setup-python@v5" @@ -41,7 +49,8 @@ jobs: - name: "Run tests" env: BASED_TEST_DB_URLS: | - postgresql://based:based@localhost:5432/based + postgresql://based:based@localhost:5432/based, + mysql://root:based@127.0.0.1:3306/based run: "make test" coverage: @@ -59,6 +68,14 @@ jobs: - 5432:5432 options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + mysql: + image: mariadb:latest + env: + MARIADB_ROOT_PASSWORD: based + MARIADB_DB: based + ports: + - 5432:5432 + steps: - uses: "actions/checkout@v4" - uses: "actions/setup-python@v5" @@ -69,7 +86,8 @@ jobs: - name: "Run tests" env: BASED_TEST_DB_URLS: | - postgresql://based:based@localhost:5432/based + postgresql://based:based@localhost:5432/based, + mysql://root:based@127.0.0.1:3306/based run: "make test" - name: Coverage report uses: irongut/CodeCoverageSummary@v1.3.0 diff --git a/LICENSE.txt b/LICENSE.txt index 10bebc8..8d5ef5c 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2024 ansipunk +Copyright (c) 2024 ansipunk Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 610c568..e8fa8d2 100644 --- a/README.md +++ b/README.md @@ -3,17 +3,17 @@ A based asynchronous database connection manager. Based is designed to be used with SQLAlchemy Core requests. Currently, the only -supported databases are SQLite and PostgreSQL. It's fairly simple to add a new -backend, should you need one. Work in progress - any contributions - issues or -pull requests - are very welcome. API might change, as library is still at its -early experiment stage. +supported databases are SQLite, PostgreSQL and MySQL. It's fairly simple to add +a new backend, should you need one. Work in progress - any contributions - +issues or pull requests - are very welcome. API might change, as library is +still at its early experiment stage. This library is inspired by [databases](https://github.com/encode/databases). ## Usage ```bash -pip install based[sqlite] # or based[postgres] +pip install based[sqlite] # or based[postgres] or based[mysql] ``` ```python @@ -99,13 +99,22 @@ need to implement `Backend` class and add its initialization to the `Database` class. You only need to implement methods that raise `NotImplementedError` in the base class, adding private helpers as needed. +### Testing + +Pass database URLs for those you want to run the tests against. Comma separated +list. + +```bash +BASED_TEST_DB_URLS='postgresql://postgres:postgres@localhost:5432/postgres,mysql://root:mariadb@127.0.0.1:3306/mariadb' make test` +``` + ## TODO - [x] CI/CD - [x] Building and uploading packages to PyPi - [x] Testing with multiple Python versions - [ ] Database URL parsing and building -- [ ] MySQL backend +- [x] MySQL backend - [x] Add comments and docstrings - [x] Add lock for PostgreSQL in `force_rollback` mode and SQLite in both modes - [x] Refactor tests diff --git a/based/__init__.py b/based/__init__.py index 0819391..7275819 100644 --- a/based/__init__.py +++ b/based/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.4.2" +__version__ = "0.5.0" from based.backends import Session from based.database import Database diff --git a/based/backends/__init__.py b/based/backends/__init__.py index de218fc..2ee6e0f 100644 --- a/based/backends/__init__.py +++ b/based/backends/__init__.py @@ -122,10 +122,12 @@ def __init__( # noqa: D107 async def _execute( self, query: typing.Union[ClauseElement, str], - params: typing.Optional[typing.Union[ - typing.Dict[str, typing.Any], - typing.List[typing.Any], - ]] = None, + params: typing.Optional[ + typing.Union[ + typing.Dict[str, typing.Any], + typing.List[typing.Any], + ] + ] = None, ) -> typing.Any: # noqa: ANN401 """Execute the provided query and return a corresponding Cursor object. @@ -145,34 +147,17 @@ async def _execute( """ return await self._conn.execute(query, params) - async def _execute_within_transaction( - self, - query: typing.Union[ClauseElement, str], - params: typing.Optional[typing.Union[ - typing.Dict[str, typing.Any], - typing.List[typing.Any], - ]] = None, - ) -> typing.Any: # noqa: ANN401 - await self.create_transaction() - - try: - cursor = await self._conn.execute(query, params) - except Exception: - await self.cancel_transaction() - raise - else: - await self.commit_transaction() - - return cursor - def _compile_query( - self, query: ClauseElement, + self, + query: ClauseElement, ) -> typing.Tuple[ str, - typing.Optional[typing.Union[ - typing.Dict[str, typing.Any], - typing.List[typing.Any], - ]], + typing.Optional[ + typing.Union[ + typing.Dict[str, typing.Any], + typing.List[typing.Any], + ] + ], ]: compiled_query = query.compile( dialect=self._dialect, @@ -182,7 +167,9 @@ def _compile_query( return str(compiled_query), compiled_query.params def _cast_row( - self, cursor: typing.Any, row: typing.Any, # noqa: ANN401 + self, + cursor: typing.Any, # noqa: ANN401 + row: typing.Any, # noqa: ANN401 ) -> typing.Dict[str, typing.Any]: """Cast a driver specific Row object to a more general mapping.""" fields = [column[0] for column in cursor.description] @@ -191,10 +178,12 @@ def _cast_row( async def execute( self, query: typing.Union[ClauseElement, str], - params: typing.Optional[typing.Union[ - typing.Dict[str, typing.Any], - typing.List[typing.Any], - ]] = None, + params: typing.Optional[ + typing.Union[ + typing.Dict[str, typing.Any], + typing.List[typing.Any], + ] + ] = None, ) -> None: """Execute the provided query. @@ -207,15 +196,17 @@ async def execute( """ if isinstance(query, ClauseElement): query, params = self._compile_query(query) - await self._execute_within_transaction(query, params) + await self._execute(query, params) async def fetch_one( self, query: typing.Union[ClauseElement, str], - params: typing.Optional[typing.Union[ - typing.Dict[str, typing.Any], - typing.List[typing.Any], - ]] = None, + params: typing.Optional[ + typing.Union[ + typing.Dict[str, typing.Any], + typing.List[typing.Any], + ] + ] = None, ) -> typing.Optional[typing.Dict[str, typing.Any]]: """Execute the provided query. @@ -234,19 +225,23 @@ async def fetch_one( if isinstance(query, ClauseElement): query, params = self._compile_query(query) - cursor = await self._execute_within_transaction(query, params) + cursor = await self._execute(query, params) row = await cursor.fetchone() if not row: return None - return self._cast_row(cursor, row) + row = self._cast_row(cursor, row) + await cursor.close() + return row async def fetch_all( self, query: typing.Union[ClauseElement, str], - params: typing.Optional[typing.Union[ - typing.Dict[str, typing.Any], - typing.List[typing.Any], - ]] = None, + params: typing.Optional[ + typing.Union[ + typing.Dict[str, typing.Any], + typing.List[typing.Any], + ] + ] = None, ) -> typing.List[typing.Dict[str, typing.Any]]: """Execute the provided query. @@ -264,9 +259,11 @@ async def fetch_all( if isinstance(query, ClauseElement): query, params = self._compile_query(query) - cursor = await self._execute_within_transaction(query, params) + cursor = await self._execute(query, params) rows = await cursor.fetchall() - return [self._cast_row(cursor, row) for row in rows] + rows = [self._cast_row(cursor, row) for row in rows] + await cursor.close() + return rows async def create_transaction(self) -> None: """Create a transaction and add it to the transaction stack.""" diff --git a/based/backends/mysql.py b/based/backends/mysql.py new file mode 100644 index 0000000..0277d45 --- /dev/null +++ b/based/backends/mysql.py @@ -0,0 +1,91 @@ +import typing +from contextlib import asynccontextmanager + +import asyncmy +from sqlalchemy import URL, make_url +from sqlalchemy.dialects.mysql.asyncmy import dialect +from sqlalchemy.engine.interfaces import Dialect +from sqlalchemy.sql import ClauseElement + +from based.backends import Backend, Session + + +class MySQL(Backend): + """A MySQL backend for based.Database using asyncmy.""" + + _url: URL + _pool: asyncmy.Pool + _force_rollback: bool + _force_rollback_connection: asyncmy.Connection + _dialect: Dialect + + def __init__(self, url: str, *, force_rollback: bool = False) -> None: # noqa: D107 + self._url = make_url(url) + self._force_rollback = force_rollback + self._dialect = dialect() # type: ignore + + async def _connect(self) -> None: + self._pool = await asyncmy.create_pool( + user=self._url.username, + password=self._url.password, + host=self._url.host, + port=self._url.port, + database=self._url.database, + ) + + if self._force_rollback: + self._force_rollback_connection = await self._pool.acquire() + + async def _disconnect(self) -> None: + if self._force_rollback: + await self._force_rollback_connection.rollback() + self._pool.release(self._force_rollback_connection) + + self._pool.close() + await self._pool.wait_closed() + + @asynccontextmanager + async def _session(self) -> typing.AsyncGenerator["Session", None]: + if self._force_rollback: + connection = self._force_rollback_connection + else: + connection = await self._pool.acquire() + + session = _MySQLSession(connection, self._dialect) + + if self._force_rollback: + await session.create_transaction() + + try: + yield session + except Exception: + await session.cancel_transaction() + raise + else: + await session.commit_transaction() + else: + try: + yield session + except Exception: + await connection.rollback() + raise + else: + await connection.commit() + finally: + self._pool.release(connection) + + +class _MySQLSession(Session): + async def _execute( + self, + query: typing.Union[ClauseElement, str], + params: typing.Optional[ + typing.Union[ + typing.Dict[str, typing.Any], + typing.List[typing.Any], + ] + ] = None, + ) -> asyncmy.cursors.Cursor: + cursor = self._conn.cursor() + await cursor.execute(query, params) + return cursor diff --git a/based/database.py b/based/database.py index 16e6978..b4f808e 100644 --- a/based/database.py +++ b/based/database.py @@ -63,13 +63,20 @@ def __init__( if schema == "sqlite": from based.backends.sqlite import SQLite + sqlite_url = url_parts[1][1:] self._backend = SQLite( - sqlite_url, force_rollback=force_rollback, + sqlite_url, + force_rollback=force_rollback, ) elif schema == "postgresql": from based.backends.postgresql import PostgreSQL + self._backend = PostgreSQL(url, force_rollback=force_rollback) + elif schema == "mysql": + from based.backends.mysql import MySQL + + self._backend = MySQL(url, force_rollback=force_rollback) else: raise ValueError(f"Unknown database schema: {schema}") diff --git a/pyproject.toml b/pyproject.toml index 6bfd8e2..4740c47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,11 +4,11 @@ build-backend = "setuptools.build_meta" [project] name = "based" -authors = [{ name = "ansipunk" }] +authors = [{ name = "ansipunk", email = "kysput@gmail.com" }] description = "A based asynchronous database connection manager." readme = "README.md" license = { file = "LICENSE.txt" } -keywords = [ "database", "sqlalchemy", "sqlite", "asyncio" ] +keywords = [ "database", "sqlalchemy", "sqlite", "asyncio", "postgresql", "mysql" ] requires-python = ">=3.8" dynamic = [ "version" ] classifiers = [ @@ -23,6 +23,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ] dependencies = [ "SQLAlchemy>=2,<2.1" ] @@ -34,6 +35,9 @@ postgres = [ "psycopg>=3,<4", "psycopg_pool>=3,<4", ] +mysql = [ + "asyncmy>=0.2,<1", +] dev = [ "SQLAlchemy-Utils>=0.41,<1", "mypy>=1.12,<2", @@ -53,7 +57,7 @@ version = { attr = "based.__version__" } addopts = "-n 2 --cov=based --cov-report=term-missing --cov-report=html --cov-report=xml" testpaths = [ "tests" ] asyncio_mode = "auto" -asyncio_default_fixture_loop_scope = "session" +asyncio_default_fixture_loop_scope = "function" [tool.ruff.lint] select = [ diff --git a/tests/conftest.py b/tests/conftest.py index b3fcf68..6e306ab 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -108,20 +108,24 @@ def _context( @pytest.fixture async def database(database_url: str, mocker: pytest_mock.MockerFixture): database = based.Database(database_url, force_rollback=True) + await database.connect() if database_url.startswith("postgresql"): getconn_mock = mocker.spy(database._backend._pool, "getconn") putconn_mock = mocker.spy(database._backend._pool, "putconn") - - await database.connect() + elif database_url.startswith("mysql"): + getconn_mock = mocker.spy(database._backend._pool, "acquire") + putconn_mock = mocker.spy(database._backend._pool, "release") try: yield database finally: await database.disconnect() - if database_url.startswith("postgresql"): - assert getconn_mock.call_count == putconn_mock.call_count + if database_url.startswith(("postgresql", "mysql")): + # 1 is subtracted because one connection is always automatically + # acquired when force rollback mode is engaged. + assert getconn_mock.call_count == putconn_mock.call_count - 1 @pytest.fixture From 5df295bf6cf10777d4e09592ff0cb0c5c45dba3b Mon Sep 17 00:00:00 2001 From: ansipunk Date: Tue, 11 Feb 2025 02:27:39 +0500 Subject: [PATCH 2/5] fuck mypy --- .gitignore | 1 - Makefile | 4 ++-- pyproject.toml | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index fb70281..e96584e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ .venv *.egg-info -.mypy_cache .ruff_cache .coverage diff --git a/Makefile b/Makefile index f92b18a..28cc424 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help bootstrap lint test clean +.PHONY: help bootstrap lint test build clean DEFAULT: help VENV = .venv @@ -9,6 +9,7 @@ help: @echo " bootstrap - setup development environment" @echo " lint - run static code analysis" @echo " test - run project tests" + @echo " build - build packages" @echo " clean - clean environment and remove development artifacts" bootstrap: @@ -18,7 +19,6 @@ bootstrap: lint: $(VENV) $(PYTHON) -m ruff check based tests - $(PYTHON) -m mypy --strict based test: $(VENV) $(PYTHON) -m pytest diff --git a/pyproject.toml b/pyproject.toml index 4740c47..7010755 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,6 @@ mysql = [ ] dev = [ "SQLAlchemy-Utils>=0.41,<1", - "mypy>=1.12,<2", "psycopg2-binary>=2.9,<3", "pytest-asyncio>=0.24,<1", "pytest-cov>=5,<6", From af2226567e43b673edf317d5738922d92c62a646 Mon Sep 17 00:00:00 2001 From: ansipunk Date: Tue, 11 Feb 2025 02:31:01 +0500 Subject: [PATCH 3/5] fix broken pipelines --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 23f0d2b..80430ae 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -35,7 +35,7 @@ jobs: MARIADB_ROOT_PASSWORD: based MARIADB_DB: based ports: - - 5432:5432 + - 3306:3306 steps: - uses: "actions/checkout@v4" @@ -74,7 +74,7 @@ jobs: MARIADB_ROOT_PASSWORD: based MARIADB_DB: based ports: - - 5432:5432 + - 3306:3306 steps: - uses: "actions/checkout@v4" From fd94adb6e423fd63eba796bc2f62cb188788720b Mon Sep 17 00:00:00 2001 From: ansipunk Date: Tue, 11 Feb 2025 02:33:38 +0500 Subject: [PATCH 4/5] ... --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7010755..a55bcc3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,14 +39,15 @@ mysql = [ "asyncmy>=0.2,<1", ] dev = [ - "SQLAlchemy-Utils>=0.41,<1", + "mysqlclient>=2.2,<3", "psycopg2-binary>=2.9,<3", + "pytest>=8,<9", "pytest-asyncio>=0.24,<1", "pytest-cov>=5,<6", "pytest-mock>=3,<4", "pytest-xdist>=3,<4", - "pytest>=8,<9", "ruff>=0.7,<0.8", + "SQLAlchemy-Utils>=0.41,<1", ] [tool.setuptools.dynamic] From dcd7a425dbd7ea5e17a8544c2f0276dd225db434 Mon Sep 17 00:00:00 2001 From: ansipunk Date: Tue, 11 Feb 2025 02:36:10 +0500 Subject: [PATCH 5/5] install mysql libraries --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 28cc424..b69d340 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ help: bootstrap: python3 -m venv $(VENV) $(PYTHON) -m pip install --upgrade pip==24.2 setuptools==75.2.0 wheel==0.44.0 build==1.2.2.post1 - $(PYTHON) -m pip install -e .[postgres,sqlite,dev] + $(PYTHON) -m pip install -e .[postgres,sqlite,mysql,dev] lint: $(VENV) $(PYTHON) -m ruff check based tests