Skip to content

Commit 2f836db

Browse files
sveinugusveinugu
committed
Huge commit fixing typing and autocomplete for task and flow variants
Typing fixes everywhere - too many to mention. Fir both mypy and pyright
1 parent eac0be3 commit 2f836db

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+1548
-930
lines changed

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,3 +140,7 @@ poetry.lock
140140

141141
# Documentation
142142
html
143+
144+
# IPython / Jupyter
145+
146+
.ipynb_checkpoints

‎pyproject.toml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ pytest-mypy-plugins = "^3.1.2"
6060
devtools = "^0.12.2"
6161
mypy = "^1.11.2"
6262
pytest-aiohttp = "^1.0.5"
63+
pyright = "^1.1.389"
6364

6465
[tool.poetry.group.docs.dependencies]
6566
portray = {git = "https://github.com/HeinrichAD/portray.git"}
@@ -137,7 +138,7 @@ follow_imports = "silent"
137138
warn_redundant_casts = true
138139
warn_unused_ignores = true
139140
warn_unused_configs = true
140-
warn_unreachable = true
141+
warn_unreachable = false
141142
show_error_codes = true
142143

143144
# Explicit is better than implicit
@@ -170,6 +171,9 @@ disallow_untyped_decorators = false
170171
#init_typed = false
171172
#warn_required_dynamic_aliases = true
172173

174+
[tool.pyright]
175+
reportFunctionMemberAccess = 'warning'
176+
173177
[tool.pytest.ini_options]
174178
addopts = "--tb=native"
175179
testpaths = [

‎src/omnipy/api/protocols/private/compute/job.py‎

Lines changed: 71 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
from datetime import datetime
22
from inspect import BoundArguments
33
from types import MappingProxyType
4-
from typing import Any, Callable, Mapping, Protocol, Type
4+
from typing import Any, Callable, Iterable, Mapping, ParamSpec, Protocol, runtime_checkable
5+
6+
from typing_extensions import TypeVar
57

68
from omnipy.api.enums import (OutputStorageProtocolOptions,
79
PersistOutputsOptions,
@@ -19,8 +21,12 @@
1921
TaskTemplateCovT,
2022
TaskTemplateT)
2123

24+
CallP = ParamSpec('CallP')
25+
RetCovT = TypeVar('RetCovT', covariant=True)
26+
RetContraT = TypeVar('RetContraT', contravariant=True)
27+
2228

23-
class IsJobBase(CanLog, IsUniquelyNamedJob, Protocol):
29+
class IsJobBase(CanLog, IsUniquelyNamedJob, Protocol[JobTemplateT, JobT, CallP, RetCovT]):
2430
""""""
2531
@property
2632
def _job_creator(self) -> IsJobCreator:
@@ -38,68 +44,81 @@ def engine(self) -> IsEngine | None:
3844
def in_flow_context(self) -> bool:
3945
...
4046

41-
def __eq__(self, other: object):
47+
def __eq__(self, other: object) -> bool:
4248
...
4349

4450
@classmethod
45-
def _create_job_template(cls, *args: object, **kwargs: object) -> 'IsJobTemplate':
51+
def _create_job_template(cls, *args: object, **kwargs: object) -> JobTemplateT:
4652
...
4753

4854
@classmethod
49-
def _create_job(cls, *args: object, **kwargs: object) -> 'IsJob':
55+
def _create_job(cls, *args: object, **kwargs: object) -> JobT:
56+
...
57+
58+
def _apply(self) -> JobT:
5059
...
5160

52-
def _apply(self) -> 'IsJob':
61+
def _refine(self, *args: Any, update: bool = True, **kwargs: object) -> JobTemplateT:
5362
...
5463

55-
def _refine(self, *args: Any, update: bool = True, **kwargs: object) -> 'IsJobTemplate':
64+
def _revise(self) -> JobTemplateT:
5665
...
5766

58-
def _revise(self) -> 'IsJobTemplate':
67+
def _call_job_template(self, *args: CallP.args, **kwargs: CallP.kwargs) -> RetCovT:
5968
...
6069

61-
def _call_job_template(self, *args: object, **kwargs: object) -> object:
70+
def _call_job(self, *args: CallP.args, **kwargs: CallP.kwargs) -> RetCovT:
6271
...
6372

64-
def _call_job(self, *args: object, **kwargs: object) -> object:
73+
74+
class IsJobBaseCallable(IsJobBase[JobTemplateT, JobT, CallP, RetCovT],
75+
Protocol[JobTemplateT, JobT, CallP, RetCovT]):
76+
def __call__(self, *args: CallP.args, **kwargs: CallP.kwargs) -> RetCovT:
6577
...
6678

6779

68-
class IsJob(IsJobBase, Protocol):
80+
@runtime_checkable
81+
class IsJob(IsJobBaseCallable[JobTemplateT, JobT, CallP, RetCovT],
82+
Protocol[JobTemplateT, JobT, CallP, RetCovT]):
6983
""""""
7084
@property
7185
def time_of_cur_toplevel_flow_run(self) -> datetime | None:
7286
...
7387

7488
@classmethod
75-
def create_job(cls, *args: object, **kwargs: object) -> 'IsJob':
89+
def create_job(cls, *args: object, **kwargs: object) -> JobT:
7690
...
7791

78-
def __call__(self, *args: object, **kwargs: object) -> object:
92+
def revise(self) -> JobTemplateT:
7993
...
8094

8195
def _apply_engine_decorator(self, engine: IsEngine) -> None:
8296
...
8397

8498

85-
class IsJobTemplate(IsJobBase, Protocol):
99+
@runtime_checkable
100+
class IsJobTemplate(IsJobBaseCallable[JobTemplateT, JobT, CallP, RetCovT],
101+
Protocol[JobTemplateT, JobT, CallP, RetCovT]):
86102
""""""
87103
@classmethod
88-
def create_job_template(cls, *args: object, **kwargs: object) -> 'IsJobTemplate':
104+
def create_job_template(cls, *args: object, **kwargs: object) -> JobTemplateT:
105+
...
106+
107+
def run(self, *args: CallP.args, **kwargs: CallP.kwargs) -> RetCovT:
89108
...
90109

91-
def run(self, *args: object, **kwargs: object) -> object:
110+
def apply(self) -> JobT:
92111
...
93112

94113

95-
class IsFuncArgJobBase(IsJob, Protocol):
114+
class IsFuncArgJobBase(Protocol):
96115
""""""
97116
@property
98117
def param_signatures(self) -> MappingProxyType:
99118
...
100119

101120
@property
102-
def return_type(self) -> Type[object]:
121+
def return_type(self) -> type:
103122
...
104123

105124
@property
@@ -161,26 +180,25 @@ def get_bound_args(self, *args: object, **kwargs: object) -> BoundArguments:
161180
...
162181

163182

183+
# Change?
164184
class IsPlainFuncArgJobBase(Protocol):
165185
""""""
186+
166187
_job_func: Callable
167188

168189
def _accept_call_func_decorator(self, call_func_decorator: GeneralDecorator) -> None:
169190
...
170191

171192

172-
class IsFuncArgJob(IsFuncArgJobBase, Protocol[JobT]):
173-
""""""
174-
def revise(self) -> JobT:
175-
...
193+
CallableT = TypeVar('CallableT', bound=Callable)
176194

177195

178-
class IsFuncArgJobTemplateCallable(Protocol[
179-
JobTemplateT,
180-
]):
196+
class HasFuncArgJobTemplateInit(Protocol[JobTemplateT, CallP, RetContraT]):
181197
""""""
182198
def __call__(
183199
self,
200+
job_func: Callable[CallP, RetContraT],
201+
*,
184202
name: str | None = None,
185203
iterate_over_data_files: bool = False,
186204
output_dataset_param: str | None = None,
@@ -189,14 +207,16 @@ def __call__(
189207
persist_outputs: PersistOutputsOptions | None = None,
190208
restore_outputs: RestoreOutputsOptions | None = None,
191209
result_key: str | None = None,
192-
fixed_params: Mapping[str, object] | None = None,
193-
param_key_map: Mapping[str, str] | None = None,
210+
fixed_params: Mapping[str, object] | Iterable[tuple[str, object]] | None = None,
211+
param_key_map: Mapping[str, str] | Iterable[tuple[str, str]] | None = None,
194212
**kwargs: object,
195-
) -> Callable[[Callable], JobTemplateT]:
213+
) -> JobTemplateT:
196214
...
197215

198216

199-
class IsFuncArgJobTemplate(IsJobTemplate, IsFuncArgJobBase, Protocol[JobTemplateT, JobT]):
217+
class IsFuncArgJobTemplate(IsJobTemplate[JobTemplateT, JobT, CallP, RetCovT],
218+
IsFuncArgJobBase,
219+
Protocol[JobTemplateT, JobT, CallP, RetCovT]):
200220
""""""
201221
def refine(self,
202222
*args: Any,
@@ -209,13 +229,16 @@ def refine(self,
209229
persist_outputs: PersistOutputsOptions | None = None,
210230
restore_outputs: RestoreOutputsOptions | None = None,
211231
result_key: str | None = None,
212-
fixed_params: Mapping[str, object] | None = None,
213-
param_key_map: Mapping[str, str] | None = None,
232+
fixed_params: Mapping[str, object] | Iterable[tuple[str, object]] | None = None,
233+
param_key_map: Mapping[str, str] | Iterable[tuple[str, str]] | None = None,
214234
**kwargs: object) -> JobTemplateT:
215235
...
216236

217-
def apply(self) -> JobT:
218-
...
237+
238+
class IsFuncArgJob(IsJob[JobTemplateT, JobT, CallP, RetCovT],
239+
IsFuncArgJobBase,
240+
Protocol[JobTemplateT, JobT, CallP, RetCovT]):
241+
""""""
219242

220243

221244
class IsTaskTemplateArgsJobBase(IsFuncArgJobBase, Protocol[TaskTemplateCovT]):
@@ -226,15 +249,20 @@ def task_templates(self) -> tuple[TaskTemplateCovT, ...]:
226249

227250

228251
class IsTaskTemplateArgsJob(IsTaskTemplateArgsJobBase[TaskTemplateCovT],
229-
IsFuncArgJob[JobT],
230-
Protocol[TaskTemplateCovT, JobT]):
252+
IsFuncArgJob[JobTemplateT, JobT, CallP, RetCovT],
253+
Protocol[TaskTemplateCovT, JobTemplateT, JobT, CallP, RetCovT]):
231254
""""""
232255

233256

234-
class IsTaskTemplateArgsJobTemplateCallable(Protocol[TaskTemplateContraT, JobTemplateT]):
257+
class HasTaskTemplateArgsJobTemplateInit(Protocol[JobTemplateT,
258+
TaskTemplateContraT,
259+
CallP,
260+
RetContraT]):
235261
""""""
236262
def __call__(
237263
self,
264+
job_func: Callable[CallP, RetContraT],
265+
/,
238266
*task_templates: TaskTemplateContraT,
239267
name: str | None = None,
240268
iterate_over_data_files: bool = False,
@@ -244,16 +272,16 @@ def __call__(
244272
persist_outputs: PersistOutputsOptions | None = None,
245273
restore_outputs: RestoreOutputsOptions | None = None,
246274
result_key: str | None = None,
247-
fixed_params: Mapping[str, object] | None = None,
248-
param_key_map: Mapping[str, str] | None = None,
275+
fixed_params: Mapping[str, object] | Iterable[tuple[str, object]] | None = None,
276+
param_key_map: Mapping[str, str] | Iterable[tuple[str, str]] | None = None,
249277
**kwargs: object,
250-
) -> Callable[[Callable], JobTemplateT]:
278+
) -> JobTemplateT:
251279
...
252280

253281

254-
class IsTaskTemplateArgsJobTemplate(IsFuncArgJobTemplate[JobTemplateT, JobT],
282+
class IsTaskTemplateArgsJobTemplate(IsFuncArgJobTemplate[JobTemplateT, JobT, CallP, RetCovT],
255283
IsTaskTemplateArgsJobBase[TaskTemplateT],
256-
Protocol[TaskTemplateT, JobTemplateT, JobT]):
284+
Protocol[TaskTemplateT, JobTemplateT, JobT, CallP, RetCovT]):
257285
""""""
258286
def refine(self,
259287
*task_templates: TaskTemplateT,
@@ -263,8 +291,8 @@ def refine(self,
263291
output_dataset_param: str | None = None,
264292
output_dataset_cls: type[IsDataset] | None = None,
265293
auto_async: bool = True,
266-
fixed_params: Mapping[str, object] | None = None,
267-
param_key_map: Mapping[str, str] | None = None,
294+
fixed_params: Mapping[str, object] | Iterable[tuple[str, object]] | None = None,
295+
param_key_map: Mapping[str, str] | Iterable[tuple[str, str]] | None = None,
268296
result_key: str | None = None,
269297
persist_outputs: PersistOutputsOptions | None = None,
270298
restore_outputs: RestoreOutputsOptions | None = None,

‎src/omnipy/api/protocols/private/util.py‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,14 @@
11
from types import TracebackType
2-
from typing import Callable, Protocol, runtime_checkable, TypeVar
2+
from typing import Callable, Protocol, runtime_checkable
3+
4+
from typing_extensions import TypeVar
35

4-
from omnipy.api.typedefs import DecoratorClassT
56
from omnipy.util.setdeque import SetDeque
67

7-
_ObjT = TypeVar('_ObjT', bound=object)
88
_ObjContraT = TypeVar('_ObjContraT', contravariant=True, bound=object)
99
_AnyKeyT = TypeVar('_AnyKeyT', contravariant=True, bound=object)
1010
_ValT = TypeVar('_ValT', bound=object)
1111
_ContentsT = TypeVar('_ContentsT', bound=object)
12-
_ContentCovT = TypeVar('_ContentCovT', covariant=True, bound=object)
13-
_ContentContraT = TypeVar('_ContentContraT', contravariant=True, bound=object)
1412
_HasContentsT = TypeVar('_HasContentsT', bound='HasContents')
1513

1614

@@ -21,13 +19,6 @@ def __call__(self, callable_arg: Callable, /, *args: object, **kwargs: object) -
2119
...
2220

2321

24-
@runtime_checkable
25-
class IsCallableClass(Protocol[DecoratorClassT]):
26-
""""""
27-
def __call__(self, *args: object, **kwargs: object) -> Callable[[Callable], DecoratorClassT]:
28-
...
29-
30-
3122
@runtime_checkable
3223
class HasContents(Protocol[_ContentsT]):
3324
@property

0 commit comments

Comments
 (0)