Add software sync support

Software sync brings two cameras frames into synchronisation. It
takes some time, however, and image metadata tells us when this
synchronisation moment has been reached.

The extra functionality includes:

* The ability for encoders to ignore frames until sync is achieved.

* Applications can wait for the encoder to signal that a synchronised
  recording has actually started.

* We can easily capture the very first request where sync was
  achieved.

Some examples and tests also been included.

Signed-off-by: David Plowman <david.plowman@raspberrypi.com>

Author: davidplowman

examples/sync_capture.py

@@ -0,0 +1,28 @@
+#!/usr/bin/python3
+
+from libcamera import controls
+
+from picamera2 import Picamera2
+
+# Show how to do software-synchronised capture.
+#
+# This code is for the server (which controls the synchronisation), but the code
+# for the client is almost identical. We just change controls.rpi.SyncModeEnum.Server
+# into controls.rpi.SyncModeEnum.Client and it will listen out for and follow
+# the server.
+#
+# Usually the best advice is to start the client first. It will sit there doing
+# nothing at all until the server has been started and tells it "now".
+
+picam2 = Picamera2()
+ctrls = {'FrameRate': 30.0, 'SyncMode': controls.rpi.SyncModeEnum.Server}
+# You can use a still capture mode, but would probably need more buffers so that
+# you don't get lots of frame drops.
+config = picam2.create_preview_configuration(controls=ctrls)
+
+picam2.start(config)
+
+req = picam2.capture_sync_request()
+# THe 'timer' tells us how many microseconds we were away from the synchronisation point.
+# Normally this should be small, but may be much larger if frames are being dropped.
+print("Lag:", req.get_metadata()['SyncTimer'])

examples/sync_recording.py

@@ -0,0 +1,37 @@
+#!/usr/bin/python3
+
+import time
+
+from libcamera import controls
+
+from picamera2 import Picamera2
+from picamera2.encoders import H264Encoder
+
+# Show how to do software-synchronised camera recordings.
+#
+# This code is for the server (which controls the synchronisation), but the code
+# for the client is almost identical. We just change controls.rpi.SyncModeEnum.Server
+# into controls.rpi.SyncModeEnum.Client and it will listen out for and follow
+# the server.
+#
+# Usually the best advice is to start the client first. It will sit there recording
+# nothing at all until the server has been started and tells it to "go now".
+
+picam2 = Picamera2()
+ctrls = {'SyncMode': controls.rpi.SyncModeEnum.Server}
+config = picam2.create_video_configuration(controls=ctrls)
+encoder = H264Encoder(bitrate=5000000)
+encoder.sync_enable = True  # this tells the encoder to wait until synchronisation
+output = "server.h264"
+
+picam2.start(config)
+picam2.start_encoder(encoder, output)
+
+# This event being signalled tells us that recording has started.
+encoder.sync.wait()
+print("Recording has started")
+
+time.sleep(5)
+
+picam2.stop_encoder(encoder)
+picam2.stop()

picamera2/encoders/encoder.py

@@ -69,6 +69,9 @@ def __init__(self):
         self.audio_sync = -100000  # in us, so by default, delay audio by 100ms
         self._audio_start = threading.Event()
         self.frames_encoded = 0
+        # For camera sync.
+        self.sync_enable = False
+        self.sync = threading.Event()
 
     @property
     def running(self):
@@ -239,6 +242,17 @@ def encode(self, stream, request):
         """
         if self.audio:
             self._audio_start.set()  # Signal the audio encode thread to start.
+
+        # If "sync" has been requested, we must wait for the image metadata to say that we
+        # don't need to wait any more. While waiting, we simply don't encode any frames.
+        if self.sync_enable:
+            metadata = request.get_metadata()
+            if metadata.get('SyncReady', False):
+                self.sync_enable = False
+                self.sync.set()
+            else:
+                return
+
         if self._skip_count == 0:
             with self._lock:
                 if not self._running:

picamera2/picamera2.py

@@ -1528,6 +1528,18 @@ def captured_request(self, wait=None, flush=None):
         finally:
             request.release()
 
+    @contextlib.contextmanager
+    def captured_sync_request(self, wait=None):
+        """Capture the first synchronised request using the context manager which guarantees its release.
+
+        Only for use when running with the software sync algorith.
+        """
+        request = self.capture_sync_request(wait=wait)
+        try:
+            yield request
+        finally:
+            request.release()
+
     def capture_metadata_(self):
         if not self.completed_requests:
             return (False, None)
@@ -2006,3 +2018,24 @@ def wait_for_af_state(self, states):
                      partial(wait_for_af_state, self,
                              {controls.AfStateEnum.Focused, controls.AfStateEnum.Failed, controls.AfStateEnum.Idle})]
         return self.dispatch_functions(functions, wait, signal_function)
+
+    def capture_sync_request(self, wait=None, signal_function=None):
+        """Return the first request when the camera system has reached sychronisation point.
+
+        This method can be used when this camera is the sychronisation server or client
+        for the software sync algorithm.
+        """
+
+        def capture_sync_request_(self):
+            if not self.completed_requests:
+                return (False, None)
+            req = self.completed_requests.pop(0)
+            sync_ready = req.get_metadata().get('SyncReady', False)
+            if not sync_ready:
+                # Not yet synced. Discard this request and wait some more.
+                req.release()
+                return (False, None)
+            # Sync achieved. Return this request.
+            return (True, req)
+
+        return self.dispatch_functions([partial(capture_sync_request_, self)], wait, signal_function)

tests/sync_test.py

@@ -0,0 +1,44 @@
+#!/usr/bin/python3
+
+import io
+import time
+
+from libcamera import controls
+
+from picamera2 import Picamera2
+from picamera2.encoders import H264Encoder
+from picamera2.outputs import FileOutput
+
+picam2 = Picamera2()
+ctrls = {'SyncMode': controls.rpi.SyncModeEnum.Server, 'SyncFrames': 300}
+config = picam2.create_video_configuration(controls=ctrls)
+encoder = H264Encoder(bitrate=5000000)
+encoder.sync_enable = True  # this tells the encoder to wait until synchronisation
+buffer = io.BytesIO()
+output = FileOutput(buffer)
+
+picam2.start(config)
+picam2.start_encoder(encoder, output)
+
+# The synchronisation delay is now 300 frames, or 10 seconds at 30fps, so
+# for 5 seconds we can be quite sure that nothing should be recorded.
+
+time.sleep(5)
+if buffer.tell():
+    print("ERROR: bytes recorded before synchronisation")
+else:
+    print("No bytes during synchronisation period")
+
+encoder.sync.wait()
+print("Recording has started")
+
+# Now, if we wait a bit longer, there had better be some data!
+
+time.sleep(5)
+if buffer.tell():
+    print(buffer.tell(), "bytes record after synchronisation period")
+else:
+    print("ERROR: still no bytes after synchronisation")
+
+picam2.stop_encoder(encoder)
+picam2.stop()

tests/test_list.txt

@@ -43,6 +43,8 @@ examples/still_during_video.py
 examples/switch_mode.py
 examples/switch_mode_2.py
 examples/switch_mode_persist.py
+examples/sync_capture.py
+examples/sync_recording.py
 examples/timestamp_capture.py
 examples/title_bar.py
 examples/tuning_file.py
@@ -85,6 +87,7 @@ tests/preview_location_test.py
 tests/preview_start_stop.py
 tests/quality_check.py
 tests/qt_gl_preview_test.py
+tests/sync_test.py
 tests/stop_slow_framerate.py
 tests/allocator_test.py
 tests/allocator_leak_test.py