WIP QC-1251 possible solution for postprocessing without accessing DB

Framework/basic-latetask-async.json

@@ -0,0 +1,55 @@
+{
+  "qc": {
+    "config": {
+      "database": {
+        "implementation": "CCDB",
+        "host": "ccdb-test.cern.ch:8080"
+      },
+      "Activity": {
+        "number": "42",
+        "type": "NONE",
+        "provenance": "qc_async"
+      },
+      "monitoring": {
+        "url": "infologger:///debug?qc"
+      },
+      "consul": {
+        "url": ""
+      },
+      "conditionDB": {
+        "url": "ccdb-test.cern.ch:8080"
+      },
+      "infologger": {
+        "filterDiscardDebug": "false",
+        "filterDiscardLevel": "12"
+      },
+      "bookkeeping": {
+        "url": ""
+      }
+    },
+    "tasks": {},
+    "checks": {},
+    "lateTasks": {
+      "QcTaskLate": {                     "": "this is a way to configure an asynchronous late task",
+        "active": "true",
+        "className": "o2::quality_control::core::GeneralPurposeLateTask",
+        "moduleName": "QualityControl",
+        "detectorName": "TST",
+        "policy": "OnAll",                "": "just like Checks and Aggregators, late tasks support update policies",
+        "dataSources": [{                 "" : "provides the late task with an MO",
+          "type": "repository",
+          "detectorName": "TST",
+          "taskName": "QcTask",
+          "MOs": ["example", "example2"], "": "framework should attempt to retrieve the two objects corresponding to the same validity",
+          "restrict": {
+            "number": "",                 "": "takes any run number",
+            "periodName": "LHC23k",       "": "objects should belong to period LHC23k",
+            "passName": "derived",        "": "objects should belongs to the same pass name as the globally defined one in this file"
+          }
+        }],
+        "TODO": "how to describe expected plots? i guess this should be task-specific"
+      }
+    }
+  },
+  "dataSamplingPolicies": []
+}

Framework/basic-latetask.json

@@ -0,0 +1,96 @@
+{
+  "qc": {
+    "config": {
+      "database": {
+        "implementation": "CCDB",
+        "host": "ccdb-test.cern.ch:8080"
+      },
+      "Activity": {
+        "number": "42",
+        "type": "NONE"
+      },
+      "monitoring": {
+        "url": "infologger:///debug?qc"
+      },
+      "consul": {
+        "url": ""
+      },
+      "conditionDB": {
+        "url": "ccdb-test.cern.ch:8080"
+      },
+      "infologger": {
+        "filterDiscardDebug": "false",
+        "filterDiscardLevel": "12"
+      },
+      "bookkeeping": {
+        "url": ""
+      }
+    },
+    "tasks": {
+      "QcTask": {
+        "active": "true",
+        "className": "o2::quality_control_modules::skeleton::SkeletonTask",
+        "moduleName": "QcSkeleton",
+        "detectorName": "TST",
+        "cycleDurations": [
+          {"cycleDurationSeconds": 10, "validitySeconds": 35},
+          {"cycleDurationSeconds": 12, "validitySeconds": 1}
+        ],
+        "dataSource": {
+          "type": "dataSamplingPolicy",
+          "name": "tst-raw"
+        }
+      }
+    },
+    "checks": {
+      "QcCheck": {
+        "active": "true",
+        "className": "o2::quality_control_modules::skeleton::SkeletonCheck",
+        "moduleName": "QcSkeleton",
+        "policy": "OnAny",
+        "detectorName": "TST",
+        "dataSource": [{
+          "type": "Task",
+          "name": "QcTask",
+          "MOs": ["example"]
+        }]
+      }
+    },
+    "lateTasks": {
+      "QcTaskLate": {
+        "active": "true",
+        "className": "o2::quality_control::core::GeneralPurposeLateTask",
+        "moduleName": "QualityControl",
+        "detectorName": "TST",
+        "policy": "OnAll",                "": "just like Checks and Aggregators, late tasks support update policies",
+        "dataSources": [{                 "" : "provides the late task with an MO",
+          "type": "Task",
+          "name": "QcTask",
+          "MOs": ["example"]
+        }, {
+          "type": "TaskReduced",          "": "provides the late task with a bunch of statistics derived from declared object(s)",
+          "name": "QcTask",
+          "MOs": ["example"],
+          "reducerName": "o2::quality_control_modules::common::TH1Reducer",
+          "moduleName": "QcCommon"
+        }],
+        "TODO": "how to describe expected plots? i guess this should be task-specific"
+      }
+    }
+  },
+  "dataSamplingPolicies": [
+    {
+      "id": "tst-raw",
+      "active": "true",
+      "machines": [],
+      "query": "data:TST/RAWDATA/0",
+      "samplingConditions": [
+        {
+          "condition": "random",
+          "fraction": "0.1",
+          "seed": "1234"
+        }
+      ]
+    }
+  ]
+}

Framework/include/QualityControl/LateTaskInterface.h

@@ -0,0 +1,113 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+///
+/// \file   LateTaskInterface.h
+/// \author Piotr Konopka
+///
+
+#ifndef LATETASKINTERFACE_H
+#define LATETASKINTERFACE_H
+
+#include <memory>
+// O2
+#include <Framework/InitContext.h>
+#include <Framework/ProcessingContext.h>
+// QC
+#include "QualityControl/Activity.h"
+#include "QualityControl/ObjectsManager.h"
+#include "QualityControl/UserCodeInterface.h"
+#include "QualityControl/MonitorObject.h"
+#include "QualityControl/QualityObject.h"
+
+namespace o2::monitoring
+{
+class Monitoring;
+}
+
+//namespace o2::globaltracking
+//{
+//struct DataRequest;
+//}
+
+namespace o2::framework
+{
+struct ConcreteDataMatcher;
+}
+
+namespace o2::quality_control::core
+{
+
+/// \brief  Skeleton of a late QC task.
+///
+/// Purely abstract class defining the skeleton and the common interface of a late QC task.
+/// It is therefore the parent class of any late QC task.
+/// It is responsible for the instantiation, modification and destruction of the TObjects that are published.
+///
+/// Late tasks can process any output of a Task, Check or Aggregator and produce new MonitorObjects.
+/// In a multinode setup, they always run on remote (QC) nodes, so they can access merged MonitorObjects and any
+/// QualityObjects. Thus, it is not possible to run late Tasks on FLPs or EPNs.
+/// In async QC, late Tasks can be used in combination with a QCDB reader (not implemented yet) to perform
+/// any trends or correlations on series of objects which are available only in the QCDB.
+///
+/// TODO: one could even consider allowing to feed late tasks with output of Reductors.
+///   It could be an opportunity to refactor them as well (and rename them to Reducers, which sounds more like English).
+/// TODO: to allow for more structured configuration, i see no other choice than providing the user with the late task config tree.
+///   CustomParameters do not support tree-like structures. one could consider extending it, but i'm not sure if we can be fully backwards compatible.
+/// TODO: think how to allow to produce new plots after each `process()` in sync, while producing just one at the end for async.
+/// \author Piotr Konopka
+class LateTaskInterface : public UserCodeInterface
+{
+ public:
+  /// \brief Constructor
+  /// Can't be used when dynamically loading the class with ROOT.
+  /// @param objectsManager
+  explicit LateTaskInterface(ObjectsManager* objectsManager);
+
+  /// \brief Default constructor
+  LateTaskInterface() = default;
+
+  /// \brief Destructor
+  virtual ~LateTaskInterface() noexcept = default;
+  /// Copy constructor
+  LateTaskInterface(const LateTaskInterface& other) = default;
+  /// Move constructor
+  LateTaskInterface(LateTaskInterface&& other) noexcept = default;
+  /// Copy assignment operator
+  LateTaskInterface& operator=(const LateTaskInterface& other) = default;
+  /// Move assignment operator
+  LateTaskInterface& operator=(LateTaskInterface&& other) /* noexcept */ = default; // error with gcc if noexcept
+
+  // Definition of the methods for the template method pattern
+  virtual void initialize(o2::framework::InitContext& ctx) = 0;
+  virtual void startOfActivity(const Activity& activity) = 0;
+  virtual void process(std::map<std::string, std::shared_ptr<const core::MonitorObject>>& moMap, std::map<std::string, std::shared_ptr<const core::QualityObject>>& qoMap) = 0;
+  virtual void endOfActivity(const Activity& activity) = 0;
+  virtual void reset() = 0;
+
+  /// \brief Called each time mCustomParameters is updated.
+  virtual void configure() override;
+
+  // Setters and getters
+  void setObjectsManager(std::shared_ptr<ObjectsManager> objectsManager);
+  void setMonitoring(const std::shared_ptr<o2::monitoring::Monitoring>& mMonitoring);
+
+ protected:
+  std::shared_ptr<ObjectsManager> getObjectsManager();
+  std::shared_ptr<o2::monitoring::Monitoring> mMonitoring;
+
+ private:
+  std::shared_ptr<ObjectsManager> mObjectsManager;
+};
+
+} // namespace o2::quality_control::core
+
+#endif // LATETASKINTERFACE_H