Generalize the PETS input writer and remove legacy/unused code (#134)

* Add generalized handling for PETS input using a dedicated factory

* Since we need to support python 3.7 and 3.8, move csv file inside py

* Fix name of produced pets file to `pets.pts`

* Fix creating `pets.pts` if prefix or suffix are not defined

* Split PETS affix to blocks pre-adding: rejects only duplicate lines, not all affix

* Add doc, tests, move csv to resources, bump min Python version to 3.9

* GitHub tests fail for Python 3.13, may be not supported by some libraries yet

* Remove `[tool.setuptools.package-data]` from `pyproject.toml`

Co-authored-by: Stef Smeets <stefsmeets@users.noreply.github.com>

* Fix typo in src/instamatic/processing/ImgConversion.py

Co-authored-by: Stef Smeets <stefsmeets@users.noreply.github.com>

* Move `PetsInputWarning` to the top of the `PETS_input_factory.py` file

* Make `PETS_KEYWORDS` uppercase, fix `PetsKeywords.from_file` type hint

* Bump the suggested conda-forge Python version to 3.12

---------

Co-authored-by: Stef Smeets <stefsmeets@users.noreply.github.com>

Author: Baharis

.github/workflows/test.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12', ]
 
     steps:
     - uses: actions/checkout@v3

MANIFEST.in

@@ -27,3 +27,4 @@ include src/instamatic/config/camera/*.yaml
 include src/instamatic/config/microscope/*.yaml
 include src/instamatic/config/scripts/*.md
 include src/instamatic/neural_network/*.p
+include src/instamatic/processing/PETS_input_keywords.csv

docs/config.md

@@ -249,6 +249,42 @@ This file holds the specifications of the camera. This file is must be located t
   DiffShift: {gridsize: 5, stepsize: 300}
 ```
 
+**pets_prefix**
+: Arbitrary information to be added at the beginning of the `.pts` file created after an experiment. The prefix can include any [valid PETS2 input lines](http://pets.fzu.cz/download/pets2_manual.pdf). In the case of duplicate commands, prefix lines take precedence over hard-coded and suffix commands, and prevent the latter ones from being added. Additionally, this field can contain new python-style [replacement fields](https://pyformat.info/) which, if present among the `ImgConversion` instance attributes, will be filled automatically after each experiment (see the `pets_suffix` example). A typical `pets_prefix`, capable of overwriting the default detector specification output can look like this:
+```yaml
+pets_prefix: "noiseparameters 4.2 0\nreflectionsize 8\ndetector asi"
+```
+
+**pets_suffix**
+: Arbitrary information to be added at the end of the `.pts` file created after an experiment. Similarly to the `pets_prefix`, the suffix can include any [valid PETS2 input lines](http://pets.fzu.cz/download/pets2_manual.pdf) as well as new python-style [replacement fields](https://pyformat.info/). In contrast to prefix, any duplicate commands added to suffix will be ignored. This field can be useful to add backup or meta information about the experiment:
+```yaml
+pets_suffix: |
+  cifentries
+  _exptl_special_details
+  ;
+  {method} data collected using Instamatic.
+  Tilt step:   	{osc_angle:.3f} deg
+  Exposure:    	{headers[0][ImageExposureTime]:.6f} s per frame
+  ;
+  _diffrn_ambient_temperature          	?
+  _diffrn_source                       	'Lanthanum hexaboride cathode'
+  _diffrn_source_voltage               	200
+  _diffrn_radiation_type               	electron
+  _diffrn_radiation_wavelength         	0.0251
+  _diffrn_measurement_device           	'Transmission electron microscope'
+  _diffrn_measurement_device_type      	'FEI Tecnai G2 20'
+  _diffrn_detector                     	'ASI Cheetah'
+  _diffrn_measurement_method           	'{method}'
+  _diffrn_measurement_specimen_support 	'Cu grid with amorphous carbon foil'
+  _diffrn_standards_number             	0
+  endcifentries
+
+  badpixels
+  359 32
+  279 513
+  endbadpixels
+```
+
 ## microscope.yaml
 
 This file holds all the specifications of the microscope as necessary. It is important to set up the camera lengths, magnifications, and magnification modes. This file is must be located the `microscope/camera` directory, and can have any name as defined in `settings.yaml`.

environment.yml

@@ -4,4 +4,4 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python==3.7
+  - python==3.12

pyproject.toml

@@ -8,7 +8,7 @@ name = "instamatic"
 version = "2.1.1"
 description = "Python program for automated electron diffraction data collection"
 readme = "readme.md"
-requires-python = ">=3.7"
+requires-python = ">=3.9"
 authors = [
 	{name = "Stef Smeets", email = "s.smeets@esciencecenter.nl"},
 ]
@@ -24,11 +24,10 @@ keywords = [
 ]
 license = {text = "BSD License"}
 classifiers = [
-    "Programming Language :: Python :: 3.7",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Development Status :: 5 - Production/Stable",
     "Intended Audience :: Science/Research",
     "License :: OSI Approved :: BSD License",
@@ -123,8 +122,13 @@ publishing = [
 # setup
 "instamatic.autoconfig" = "instamatic.config.autoconfig:main"
 
+[tool.setuptools]
+packages = ["instamatic"]
+package-dir = {"" = "src"}
+include-package-data = true
+
 [tool.ruff]
-target-version = 'py37'
+target-version = 'py39'
 line-length = 96
 
 [tool.ruff.lint]

readme.md

@@ -39,7 +39,7 @@ conda create -n instamatic python=3.11
 conda activate instamatic
 ```
 
-Install using pip, works with python versions 3.7 or newer:
+Install using pip, works with python versions 3.9 or newer:
 
 ```bash
 pip install instamatic

src/instamatic/_collections.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
+import string
 from collections import UserDict
-from typing import Any
+from typing import Any, Tuple
 
 
 class NoOverwriteDict(UserDict):
@@ -11,3 +12,29 @@ def __setitem__(self, key: Any, value: Any) -> None:
         if key in self.data:
             raise KeyError(f'Key "{key}" already exists and cannot be overwritten.')
         super().__setitem__(key, value)
+
+
+class PartialFormatter(string.Formatter):
+    """`str.format` alternative, allows for partial replacement of {fields}"""
+
+    def __init__(self, missing: str = '{{{}}}') -> None:
+        super().__init__()
+        self.missing: str = missing  # used instead of missing values
+
+    def get_field(self, field_name: str, args, kwargs) -> Tuple[Any, str]:
+        """When field can't be found, return placeholder text instead."""
+        try:
+            obj, used_key = super().get_field(field_name, args, kwargs)
+            return obj, used_key
+        except (KeyError, AttributeError, IndexError, TypeError):
+            return self.missing.format(field_name), field_name
+
+    def format_field(self, value: Any, format_spec: str) -> str:
+        """If the field was not found, format placeholder as string instead."""
+        try:
+            return super().format_field(value, format_spec)
+        except (ValueError, TypeError):
+            return str(value)
+
+
+partial_formatter = PartialFormatter()

src/instamatic/_typing.py

@@ -1,6 +1,10 @@
 from __future__ import annotations
 
+import os
+from typing import Union
+
 from typing_extensions import Annotated
 
+AnyPath = Union[str, bytes, os.PathLike]
 int_nm = Annotated[int, 'Length expressed in nanometers']
 float_deg = Annotated[float, 'Angle expressed in degrees']

src/instamatic/processing/ImgConversion.py

@@ -4,13 +4,15 @@
 import logging
 import time
 from datetime import datetime
-from math import cos
+from pathlib import Path
 
 import numpy as np
 
 from instamatic import config
+from instamatic._typing import AnyPath
 from instamatic.formats import read_tiff, write_adsc, write_mrc, write_tiff
 from instamatic.processing.flatfield import apply_flatfield_correction
+from instamatic.processing.PETS_input_factory import PetsInputFactory
 from instamatic.processing.stretch_correction import affine_transform_ellipse_to_circle
 from instamatic.tools import (
     find_beam_center,
@@ -131,6 +133,7 @@ def __init__(
         rotation_axis: float,  # radians, specifies the position of the rotation axis
         acquisition_time: float,  # seconds, acquisition time (exposure time + overhead)
         flatfield: str = 'flatfield.tiff',
+        method: str = 'continuous-rotation 3D ED',  # or 'stills' or 'precession', used for CIF/documentation
     ):
         if flatfield is not None:
             flatfield, h = read_tiff(flatfield)
@@ -188,6 +191,7 @@ def __init__(
         # self.rotation_speed = get_calibrated_rotation_speed(osc_angle / self.acquisition_time)
 
         self.name = 'Instamatic'
+        self.method = method
 
         from .XDS_template import (
             XDS_template,  # hook XDS_template here, because it is difficult to override as a global
@@ -644,93 +648,36 @@ def write_beam_centers(self, path: str) -> None:
 
         np.savetxt(path / 'beam_centers.txt', centers, fmt='%10.4f')
 
-    def write_pets_inp(self, path: str, tiff_path: str = 'tiff') -> None:
-        """Write PETS input file `pets.pts` in directory `path`"""
-        if self.start_angle > self.end_angle:
-            sign = -1
-        else:
-            sign = 1
-
-        omega = np.degrees(self.rotation_axis)
-
-        # for pets, 0 <= omega <= 360
-        if omega < 0:
-            omega += 360
-        elif omega > 360:
-            omega -= 360
-
-        with open(path / 'pets.pts', 'w') as f:
-            date = str(time.ctime())
-            print(
-                '# PETS input file for Rotation Electron Diffraction generated by `instamatic`',
-                file=f,
-            )
-            print(f'# {date}', file=f)
-            print('# For definitions of input parameters, see:', file=f)
-            print('# http://pets.fzu.cz/ ', file=f)
-            print('', file=f)
-            print(f'lambda {self.wavelength}', file=f)
-            print(f'Aperpixel {self.pixelsize}', file=f)
-            print(f'phi {float(self.osc_angle) / 2}', file=f)
-            print(f'omega {omega}', file=f)
-            print('bin 1', file=f)
-            print('reflectionsize 20', file=f)
-            print('noiseparameters 3.5 38', file=f)
-            print('', file=f)
-            # print("reconstructions", file=f)
-            # print("endreconstructions", file=f)
-            # print("", file=f)
-            # print("distortions", file=f)
-            # print("enddistortions", file=f)
-            # print("", file=f)
-            print('imagelist', file=f)
-            for i in self.observed_range:
-                fn = f'{i:05d}.tiff'
-                angle = self.start_angle + sign * self.osc_angle * i
-                print(f'{tiff_path}/{fn} {angle:10.4f} 0.00', file=f)
-            print('endimagelist', file=f)
+    def write_pets_inp(self, path: AnyPath, tiff_path: str = 'tiff') -> None:
+        sign = 1 if self.start_angle < self.end_angle else -1
+        omega = np.degrees(self.rotation_axis) % 360
 
-    def write_pets2_inp(self, path: str, tiff_path: str = 'tiff') -> None:
-        """Write PETS 2 input file `pets.pts2` in directory `path`"""
-        path.mkdir(exist_ok=True, parents=True)
-
-        if self.start_angle > self.end_angle:
-            sign = -1
+        if 'continuous' in self.method:
+            geometry = 'continuous'
+        elif 'precess' in self.method:
+            geometry = 'precession'
         else:
-            sign = 1
-
-        omega = np.degrees(self.rotation_axis)
+            geometry = 'static'
+
+        p = PetsInputFactory()
+        p.add('geometry', geometry)
+        p.add('lambda', self.wavelength)
+        p.add('Aperpixel', self.pixelsize)
+        p.add('phi', float(self.osc_angle) / 2)
+        p.add('omega', omega)
+        p.add('bin', 1)
+        p.add('reflectionsize', 20)
+        p.add('noiseparameters', 3.5, 38)
+        p.add('')
+
+        s = []
+        for i in self.observed_range:
+            angle = self.start_angle + sign * self.osc_angle * i
+            s.append(f'{tiff_path}/{i:05d}.tiff {angle:10.4f} 0.00')
+        p.add('imagelist', *s)
 
-        with open(path / 'pets.pts2', 'w') as f:
-            date = str(time.ctime())
-            print(
-                '# PETS 2 input file for Rotation Electron Diffraction generated by `instamatic`',
-                file=f,
-            )
-            print(f'# {date}', file=f)
-            print('# For definitions of input parameters, see:', file=f)
-            print('# http://pets.fzu.cz/ ', file=f)
-            print('', file=f)
-            print('geometry continuous', file=f)
-            print(f'lambda {self.wavelength}', file=f)
-            print(f'Aperpixel {self.pixelsize}', file=f)
-            print(f'phi {float(self.osc_angle) / 2}', file=f)
-            print(f'omega {omega}', file=f)
-            print('bin 1', file=f)
-            print('reflectionsize 15', file=f)
-            print('noiseparameters 25 10', file=f)
-            print('i/sigma    5.00   10.00', file=f)
-            print('', file=f)
-            print('imagelist', file=f)
-            tiff_set = {0}
-            last_img = len(self.observed_range)
-            tiff_set.update(self.observed_range)
-            tiff_set.remove(last_img)
-            for i in tiff_set:
-                fn = f'{i:04d}.tiff'
-                angle = self.start_angle + sign * self.osc_angle * i
-                print(f'{tiff_path}/{fn} {angle:10.4f} 0.00', file=f)
-            print('endimagelist', file=f)
+        with open(Path(path) / 'pets.pts', 'w') as f:
+            f.write(str(p.compile(self.__dict__)))
 
     def write_REDp_shiftcorrection(self, path: str) -> None:
         """Write .sc (shift correction) file for REDp in directory `path`"""

src/instamatic/processing/ImgConversionDM.py

@@ -24,6 +24,7 @@ def __init__(
         pixelsize: float = None,  # p/Angstrom, size of the pixels (overrides camera_length)
         physical_pixelsize: float = None,  # mm, physical size of the pixels (overrides camera length)
         wavelength: float = None,  # Angstrom, relativistic wavelength of the electron beam
+        method: str = 'continuous-rotation 3D ED',  # or 'stills' or 'precession', used for CIF/documentation
     ):
         if flatfield is not None:
             flatfield, h = read_tiff(flatfield)
@@ -69,6 +70,7 @@ def __init__(
         logger.debug(f'Primary beam at: {self.mean_beam_center}')
 
         self.name = 'DigitalMicrograph'
+        self.method = method
 
         from .XDS_templateDM import XDS_template