From 064588b8d9a26c819dd8d758fce1a59307e26519 Mon Sep 17 00:00:00 2001 From: Piotr Konopka Date: Thu, 6 Feb 2025 16:33:56 +0100 Subject: [PATCH] WIP QC-1251 possible solution for postprocessing without accessing DB This proposes "late tasks", which can take as input merged MOs and any QOs within the same workflow. See JIRA for more deliberations about why it's probably the best approach. Definitely not finished, I'm just saving the progress and making it available for early review. --- Framework/basic-latetask-async.json | 55 +++++++++ Framework/basic-latetask.json | 96 +++++++++++++++ .../QualityControl/LateTaskInterface.h | 113 ++++++++++++++++++ 3 files changed, 264 insertions(+) create mode 100644 Framework/basic-latetask-async.json create mode 100644 Framework/basic-latetask.json create mode 100644 Framework/include/QualityControl/LateTaskInterface.h diff --git a/Framework/basic-latetask-async.json b/Framework/basic-latetask-async.json new file mode 100644 index 0000000000..79ae8c3ae9 --- /dev/null +++ b/Framework/basic-latetask-async.json @@ -0,0 +1,55 @@ +{ + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080" + }, + "Activity": { + "number": "42", + "type": "NONE", + "provenance": "qc_async" + }, + "monitoring": { + "url": "infologger:///debug?qc" + }, + "consul": { + "url": "" + }, + "conditionDB": { + "url": "ccdb-test.cern.ch:8080" + }, + "infologger": { + "filterDiscardDebug": "false", + "filterDiscardLevel": "12" + }, + "bookkeeping": { + "url": "" + } + }, + "tasks": {}, + "checks": {}, + "lateTasks": { + "QcTaskLate": { "": "this is a way to configure an asynchronous late task", + "active": "true", + "className": "o2::quality_control::core::GeneralPurposeLateTask", + "moduleName": "QualityControl", + "detectorName": "TST", + "policy": "OnAll", "": "just like Checks and Aggregators, late tasks support update policies", + "dataSources": [{ "" : "provides the late task with an MO", + "type": "repository", + "detectorName": "TST", + "taskName": "QcTask", + "MOs": ["example", "example2"], "": "framework should attempt to retrieve the two objects corresponding to the same validity", + "restrict": { + "number": "", "": "takes any run number", + "periodName": "LHC23k", "": "objects should belong to period LHC23k", + "passName": "derived", "": "objects should belongs to the same pass name as the globally defined one in this file" + } + }], + "TODO": "how to describe expected plots? i guess this should be task-specific" + } + } + }, + "dataSamplingPolicies": [] +} diff --git a/Framework/basic-latetask.json b/Framework/basic-latetask.json new file mode 100644 index 0000000000..bc6e20240d --- /dev/null +++ b/Framework/basic-latetask.json @@ -0,0 +1,96 @@ +{ + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080" + }, + "Activity": { + "number": "42", + "type": "NONE" + }, + "monitoring": { + "url": "infologger:///debug?qc" + }, + "consul": { + "url": "" + }, + "conditionDB": { + "url": "ccdb-test.cern.ch:8080" + }, + "infologger": { + "filterDiscardDebug": "false", + "filterDiscardLevel": "12" + }, + "bookkeeping": { + "url": "" + } + }, + "tasks": { + "QcTask": { + "active": "true", + "className": "o2::quality_control_modules::skeleton::SkeletonTask", + "moduleName": "QcSkeleton", + "detectorName": "TST", + "cycleDurations": [ + {"cycleDurationSeconds": 10, "validitySeconds": 35}, + {"cycleDurationSeconds": 12, "validitySeconds": 1} + ], + "dataSource": { + "type": "dataSamplingPolicy", + "name": "tst-raw" + } + } + }, + "checks": { + "QcCheck": { + "active": "true", + "className": "o2::quality_control_modules::skeleton::SkeletonCheck", + "moduleName": "QcSkeleton", + "policy": "OnAny", + "detectorName": "TST", + "dataSource": [{ + "type": "Task", + "name": "QcTask", + "MOs": ["example"] + }] + } + }, + "lateTasks": { + "QcTaskLate": { + "active": "true", + "className": "o2::quality_control::core::GeneralPurposeLateTask", + "moduleName": "QualityControl", + "detectorName": "TST", + "policy": "OnAll", "": "just like Checks and Aggregators, late tasks support update policies", + "dataSources": [{ "" : "provides the late task with an MO", + "type": "Task", + "name": "QcTask", + "MOs": ["example"] + }, { + "type": "TaskReduced", "": "provides the late task with a bunch of statistics derived from declared object(s)", + "name": "QcTask", + "MOs": ["example"], + "reducerName": "o2::quality_control_modules::common::TH1Reducer", + "moduleName": "QcCommon" + }], + "TODO": "how to describe expected plots? i guess this should be task-specific" + } + } + }, + "dataSamplingPolicies": [ + { + "id": "tst-raw", + "active": "true", + "machines": [], + "query": "data:TST/RAWDATA/0", + "samplingConditions": [ + { + "condition": "random", + "fraction": "0.1", + "seed": "1234" + } + ] + } + ] +} diff --git a/Framework/include/QualityControl/LateTaskInterface.h b/Framework/include/QualityControl/LateTaskInterface.h new file mode 100644 index 0000000000..a68dd5c630 --- /dev/null +++ b/Framework/include/QualityControl/LateTaskInterface.h @@ -0,0 +1,113 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file LateTaskInterface.h +/// \author Piotr Konopka +/// + +#ifndef LATETASKINTERFACE_H +#define LATETASKINTERFACE_H + +#include +// O2 +#include +#include +// QC +#include "QualityControl/Activity.h" +#include "QualityControl/ObjectsManager.h" +#include "QualityControl/UserCodeInterface.h" +#include "QualityControl/MonitorObject.h" +#include "QualityControl/QualityObject.h" + +namespace o2::monitoring +{ +class Monitoring; +} + +//namespace o2::globaltracking +//{ +//struct DataRequest; +//} + +namespace o2::framework +{ +struct ConcreteDataMatcher; +} + +namespace o2::quality_control::core +{ + +/// \brief Skeleton of a late QC task. +/// +/// Purely abstract class defining the skeleton and the common interface of a late QC task. +/// It is therefore the parent class of any late QC task. +/// It is responsible for the instantiation, modification and destruction of the TObjects that are published. +/// +/// Late tasks can process any output of a Task, Check or Aggregator and produce new MonitorObjects. +/// In a multinode setup, they always run on remote (QC) nodes, so they can access merged MonitorObjects and any +/// QualityObjects. Thus, it is not possible to run late Tasks on FLPs or EPNs. +/// In async QC, late Tasks can be used in combination with a QCDB reader (not implemented yet) to perform +/// any trends or correlations on series of objects which are available only in the QCDB. +/// +/// TODO: one could even consider allowing to feed late tasks with output of Reductors. +/// It could be an opportunity to refactor them as well (and rename them to Reducers, which sounds more like English). +/// TODO: to allow for more structured configuration, i see no other choice than providing the user with the late task config tree. +/// CustomParameters do not support tree-like structures. one could consider extending it, but i'm not sure if we can be fully backwards compatible. +/// TODO: think how to allow to produce new plots after each `process()` in sync, while producing just one at the end for async. +/// \author Piotr Konopka +class LateTaskInterface : public UserCodeInterface +{ + public: + /// \brief Constructor + /// Can't be used when dynamically loading the class with ROOT. + /// @param objectsManager + explicit LateTaskInterface(ObjectsManager* objectsManager); + + /// \brief Default constructor + LateTaskInterface() = default; + + /// \brief Destructor + virtual ~LateTaskInterface() noexcept = default; + /// Copy constructor + LateTaskInterface(const LateTaskInterface& other) = default; + /// Move constructor + LateTaskInterface(LateTaskInterface&& other) noexcept = default; + /// Copy assignment operator + LateTaskInterface& operator=(const LateTaskInterface& other) = default; + /// Move assignment operator + LateTaskInterface& operator=(LateTaskInterface&& other) /* noexcept */ = default; // error with gcc if noexcept + + // Definition of the methods for the template method pattern + virtual void initialize(o2::framework::InitContext& ctx) = 0; + virtual void startOfActivity(const Activity& activity) = 0; + virtual void process(std::map>& moMap, std::map>& qoMap) = 0; + virtual void endOfActivity(const Activity& activity) = 0; + virtual void reset() = 0; + + /// \brief Called each time mCustomParameters is updated. + virtual void configure() override; + + // Setters and getters + void setObjectsManager(std::shared_ptr objectsManager); + void setMonitoring(const std::shared_ptr& mMonitoring); + + protected: + std::shared_ptr getObjectsManager(); + std::shared_ptr mMonitoring; + + private: + std::shared_ptr mObjectsManager; +}; + +} // namespace o2::quality_control::core + +#endif // LATETASKINTERFACE_H