Project linting: support for Python 3.14 and general cleanup

pyproject.toml

@@ -41,9 +41,9 @@ path = "src/databricks/labs/blueprint/__about__.py"
 dependencies = [
     "databricks-labs-blueprint[yaml]",
     "coverage[toml]~=7.4.4",
-    "mypy~=1.9.0",
-    "pylint~=3.1.0",
-    "pylint-pytest==2.0.0a0",
+    "mypy~=1.18.0",
+    "pylint~=4.0.0",
+    # "pylint-pytest==2.0.0a0"  # Incorrect dependency constraint (pylint<4), installed separately below.
     "databricks-labs-pylint~=0.3.0",
     "pytest~=8.1.0",
     "pytest-cov~=4.1.0",
@@ -55,6 +55,10 @@ dependencies = [
     "types-requests~=2.31.0",
 ]
 
+post-install-commands = [
+    "pip install --no-deps pylint-pytest==2.0.0a0",  # See above; installed here to avoid dependency conflict.
+]
+
 # store virtual env as the child of this folder. Helps VSCode (and PyCharm) to run better
 path = ".venv"
 
@@ -207,10 +211,6 @@ py-version = "3.10"
 # source root.
 # source-roots =
 
-# When enabled, pylint would attempt to guess common misconfiguration and emit
-# user-friendly hints instead of false-positive error messages.
-suggestion-mode = true
-
 # Allow loading of arbitrary C extensions. Extensions are imported into the
 # active Python interpreter and may run arbitrary code.
 # unsafe-load-any-extension =

src/databricks/labs/blueprint/paths.py

@@ -15,7 +15,7 @@
 from collections.abc import Generator, Iterable, Sequence
 from io import BytesIO, StringIO
 from pathlib import Path, PurePath
-from typing import BinaryIO, Literal, NoReturn, TextIO, TypeVar
+from typing import BinaryIO, ClassVar, Literal, NoReturn, TextIO, TypeVar
 from urllib.parse import quote_from_bytes as urlquote_from_bytes
 
 from databricks.sdk import WorkspaceClient
@@ -127,7 +127,7 @@ class _DatabricksPath(Path, abc.ABC):  # pylint: disable=too-many-public-methods
     _str: str
     _hash: int
 
-    parser = _posixpath
+    parser: ClassVar = _posixpath
 
     # Compatibility attribute, for when superclass implementations get invoked on python <= 3.11.
     _flavour = object()
@@ -246,10 +246,10 @@ def open(
     ): ...
 
     @abstractmethod
-    def is_dir(self) -> bool: ...
+    def is_dir(self, *, follow_symlinks: bool = True) -> bool: ...
 
     @abstractmethod
-    def is_file(self) -> bool: ...
+    def is_file(self, *, follow_symlinks: bool = True) -> bool: ...
 
     @abstractmethod
     def rename(self: P, target: str | bytes | os.PathLike) -> P: ...
@@ -258,7 +258,7 @@ def rename(self: P, target: str | bytes | os.PathLike) -> P: ...
     def replace(self: P, target: str | bytes | os.PathLike) -> P: ...
 
     @abstractmethod
-    def iterdir(self: P) -> Generator[P, None, None]: ...
+    def iterdir(self: P) -> Generator[P]: ...
 
     def __reduce__(self) -> NoReturn:
         # Cannot support pickling because we can't pickle the workspace client.
@@ -587,7 +587,10 @@ def glob(
         pattern: str | bytes | os.PathLike,
         *,
         case_sensitive: bool | None = None,
-    ) -> Generator[P, None, None]:
+        recurse_symlinks: bool = False,
+    ) -> Generator[P]:
+        if recurse_symlinks:
+            raise NotImplementedError("recurse_symlinks is not supported for Databricks paths")
         pattern_parts = self._prepare_pattern(pattern)
         if case_sensitive is None:
             case_sensitive = True
@@ -599,7 +602,10 @@ def rglob(
         pattern: str | bytes | os.PathLike,
         *,
         case_sensitive: bool | None = None,
-    ) -> Generator[P, None, None]:
+        recurse_symlinks: bool = False,
+    ) -> Generator[P]:
+        if recurse_symlinks:
+            raise NotImplementedError("recurse_symlinks is not supported for Databricks paths")
         pattern_parts = ("**", *self._prepare_pattern(pattern))
         if case_sensitive is None:
             case_sensitive = True
@@ -731,18 +737,22 @@ def stat(self, *, follow_symlinks=True) -> os.stat_result:
         )  # 8
         return os.stat_result(seq)
 
-    def is_dir(self) -> bool:
+    def is_dir(self, *, follow_symlinks: bool = True) -> bool:
         """Return True if the path points to a DBFS directory."""
+        if not follow_symlinks:
+            raise NotImplementedError("follow_symlinks is not supported for DBFS paths")
         try:
             return bool(self._file_info.is_dir)
         except DatabricksError:
             return False
 
-    def is_file(self) -> bool:
-        """Return True if the path points to a file in Databricks Workspace."""
+    def is_file(self, *, follow_symlinks: bool = True) -> bool:
+        """Return True if the path points to a DBFS file."""
+        if not follow_symlinks:
+            raise NotImplementedError("follow_symlinks is not supported for DBFS paths")
         return not self.is_dir()
 
-    def iterdir(self) -> Generator[DBFSPath, None, None]:
+    def iterdir(self) -> Generator[DBFSPath]:
         for child in self._ws.dbfs.list(self.as_posix()):
             yield self._from_file_info(self._ws, child)
 
@@ -842,8 +852,8 @@ def open(
             return _TextUploadIO(self._ws, self.as_posix())
         raise ValueError(f"invalid mode: {mode}")
 
-    def read_text(self, encoding=None, errors=None):
-        with self.open(mode="r", encoding=encoding, errors=errors) as f:
+    def read_text(self, encoding=None, errors=None, newline=None) -> str:
+        with self.open(mode="r", encoding=encoding, errors=errors, newline=newline) as f:
             return f.read()
 
     @property
@@ -881,15 +891,19 @@ def stat(self, *, follow_symlinks=True) -> os.stat_result:
         seq[stat.ST_CTIME] = float(self._object_info.created_at) / 1000.0 if self._object_info.created_at else -1.0  # 9
         return os.stat_result(seq)
 
-    def is_dir(self) -> bool:
+    def is_dir(self, *, follow_symlinks: bool = True) -> bool:
         """Return True if the path points to a directory in Databricks Workspace."""
+        if not follow_symlinks:
+            raise NotImplementedError("follow_symlinks is not supported for Workspace paths")
         try:
             return self._object_info.object_type == ObjectType.DIRECTORY
         except DatabricksError:
             return False
 
-    def is_file(self) -> bool:
+    def is_file(self, *, follow_symlinks: bool = True) -> bool:
         """Return True if the path points to a file in Databricks Workspace."""
+        if not follow_symlinks:
+            raise NotImplementedError("follow_symlinks is not supported for Workspace paths")
         try:
             return self._object_info.object_type == ObjectType.FILE
         except DatabricksError:
@@ -902,7 +916,7 @@ def is_notebook(self) -> bool:
         except DatabricksError:
             return False
 
-    def iterdir(self) -> Generator[WorkspacePath, None, None]:
+    def iterdir(self) -> Generator[WorkspacePath]:
         for child in self._ws.workspace.list(self.as_posix()):
             yield self._from_object_info(self._ws, child)