From c3447555b042dbfbbfb62e53f63bdc401f7d28e3 Mon Sep 17 00:00:00 2001 From: Piotr Konopka Date: Mon, 5 May 2025 15:12:46 +0200 Subject: [PATCH] [build] remove walnut Closes OCTRL-934. --- Makefile | 7 +- cmd/walnut/main.go | 32 -- docs/building.md | 5 +- walnut/README.md | 75 ---- walnut/app/constants.go | 31 -- walnut/cmd/about.go | 83 ----- walnut/cmd/check.go | 75 ---- walnut/cmd/convert.go | 245 ------------- walnut/cmd/root.go | 100 ------ walnut/converter/converter.go | 225 ------------ walnut/converter/converter_test.go | 115 ------ walnut/converter/dump.json | 390 --------------------- walnut/converter/dump.yaml | 83 ----- walnut/converter/importer.go | 78 ----- walnut/converter/importer_test.go | 44 --- walnut/converter/small.yaml | 43 --- walnut/converter/testvalues.go | 248 ------------- walnut/doc/generate.go | 43 --- walnut/doc/walnut.md | 24 -- walnut/doc/walnut_about.md | 29 -- walnut/doc/walnut_check.md | 34 -- walnut/doc/walnut_convert.md | 31 -- walnut/includeschemata.go | 65 ---- walnut/schemata/Task.json | 186 ---------- walnut/schemata/Workflow.json | 248 ------------- walnut/schemata/includeschemata.go | 61 ---- walnut/schemata/includeschemata_test.go | 33 -- walnut/schemata/schemata.go | 442 ------------------------ walnut/schemata/task_test.yaml | 24 -- walnut/schemata/validate.go | 72 ---- walnut/schemata/validate_test.go | 48 --- walnut/schemata/workflow_test.yaml | 24 -- 32 files changed, 5 insertions(+), 3238 deletions(-) delete mode 100644 cmd/walnut/main.go delete mode 100644 walnut/README.md delete mode 100644 walnut/app/constants.go delete mode 100644 walnut/cmd/about.go delete mode 100644 walnut/cmd/check.go delete mode 100644 walnut/cmd/convert.go delete mode 100644 walnut/cmd/root.go delete mode 100644 walnut/converter/converter.go delete mode 100644 walnut/converter/converter_test.go delete mode 100644 walnut/converter/dump.json delete mode 100644 walnut/converter/dump.yaml delete mode 100644 walnut/converter/importer.go delete mode 100644 walnut/converter/importer_test.go delete mode 100644 walnut/converter/small.yaml delete mode 100644 walnut/converter/testvalues.go delete mode 100644 walnut/doc/generate.go delete mode 100644 walnut/doc/walnut.md delete mode 100644 walnut/doc/walnut_about.md delete mode 100644 walnut/doc/walnut_check.md delete mode 100644 walnut/doc/walnut_convert.md delete mode 100644 walnut/includeschemata.go delete mode 100644 walnut/schemata/Task.json delete mode 100644 walnut/schemata/Workflow.json delete mode 100644 walnut/schemata/includeschemata.go delete mode 100644 walnut/schemata/includeschemata_test.go delete mode 100644 walnut/schemata/schemata.go delete mode 100644 walnut/schemata/task_test.yaml delete mode 100644 walnut/schemata/validate.go delete mode 100644 walnut/schemata/validate_test.go delete mode 100644 walnut/schemata/workflow_test.yaml diff --git a/Makefile b/Makefile index 929854bf5..8a3a5deee 100644 --- a/Makefile +++ b/Makefile @@ -58,18 +58,17 @@ BK_LHCFILL_PROTO="https://raw.githubusercontent.com/AliceO2Group/Bookkeeping/mai VERBOSE_1 := -v VERBOSE_2 := -v -x -WHAT := o2-aliecs-core o2-aliecs-executor coconut peanut walnut o2-apricot +WHAT := o2-aliecs-core o2-aliecs-executor coconut peanut o2-apricot WHAT_o2-aliecs-core_BUILD_FLAGS=$(BUILD_ENV_FLAGS) WHAT_o2-aliecs-executor_BUILD_FLAGS=$(BUILD_ENV_FLAGS) WHAT_coconut_BUILD_FLAGS=$(BUILD_ENV_FLAGS) WHAT_peanut_BUILD_FLAGS=$(BUILD_ENV_FLAGS) -WHAT_walnut_BUILD_FLAGS=$(BUILD_ENV_FLAGS) WHAT_o2-apricot_BUILD_FLAGS=$(BUILD_ENV_FLAGS) INSTALL_WHAT:=$(patsubst %, install_%, $(WHAT)) -GENERATE_DIRS := ./apricot ./coconut/cmd ./common ./common/runtype ./common/system ./core ./core/integration/ccdb ./core/integration/dcs ./core/integration/ddsched ./core/integration/kafka ./core/integration/odc ./executor ./walnut ./core/integration/trg ./core/integration/bookkeeping -SRC_DIRS := ./apricot ./cmd/* ./core ./coconut ./executor ./common ./configuration ./occ/peanut ./walnut +GENERATE_DIRS := ./apricot ./coconut/cmd ./common ./common/runtype ./common/system ./core ./core/integration/ccdb ./core/integration/dcs ./core/integration/ddsched ./core/integration/kafka ./core/integration/odc ./executor ./core/integration/trg ./core/integration/bookkeeping +SRC_DIRS := ./apricot ./cmd/* ./core ./coconut ./executor ./common ./configuration ./occ/peanut TEST_DIRS := ./apricot/local ./common/gera ./common/utils ./common/utils/safeacks ./configuration/cfgbackend ./configuration/componentcfg ./configuration/template ./core/task/sm ./core/workflow ./core/integration/odc/fairmq ./core/integration/ccdb ./core/integration ./core/environment GO_TEST_DIRS := ./core/repos ./core/integration/dcs ./common/monitoring diff --git a/cmd/walnut/main.go b/cmd/walnut/main.go deleted file mode 100644 index 5c41f873b..000000000 --- a/cmd/walnut/main.go +++ /dev/null @@ -1,32 +0,0 @@ -/* - * === This file is part of ALICE O² === - * - * Copyright 2020 CERN and copyright holders of ALICE O². - * Author: Ayaan Zaidi - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * In applying this license CERN does not waive the privileges and - * immunities granted to it by virtue of its status as an - * Intergovernmental Organization or submit itself to any jurisdiction. - */ - -// Package walnut implements the O² Workflow Administration and Linting Utility. -package main - -import "github.com/AliceO2Group/Control/walnut/cmd" - -func main() { - cmd.Execute() -} diff --git a/docs/building.md b/docs/building.md index b738d33e2..ffd5fbe1b 100644 --- a/docs/building.md +++ b/docs/building.md @@ -12,7 +12,6 @@ AliECS consists of: * coconut - Go, runs locally or wherever you like and connects to core * process control plugin + library (OCC) - C++, runs throughout the cluster linked by user processes * peanut - Go, local debugging tool -* walnut - Go, local workflow conversion tool ## Building with aliBuild @@ -29,7 +28,7 @@ For development purposes, due to the significant differences in building and dep $ aliBuild init --defaults o2-dataflow Control-OCCPlugin $ aliBuild build --defaults o2-dataflow Control-OCCPlugin -# Includes core and executor in default build, can be used to build coconut, peanut and walnut locally +# Includes core and executor in default build, can be used to build coconut and peanut locally $ aliBuild init --defaults o2-dataflow Control-Core $ aliBuild build --defaults o2-dataflow Control-Core ``` @@ -87,4 +86,4 @@ For subsequent builds (after the first one), plain `make` (instead of `make all` If you wish to also build the process control library and/or plugin, see [the OCC readme](./occ/README.md). -This build of AliECS can be run locally and connected to an existing O²/FLP Suite cluster by passing a `--mesosUrl` parameter. If you do this, remember to `systemctl stop o2-aliecs-core` on the head node, in order to stop the core that came with the O²/FLP Suite and use your own. \ No newline at end of file +This build of AliECS can be run locally and connected to an existing O²/FLP Suite cluster by passing a `--mesosUrl` parameter. If you do this, remember to `systemctl stop o2-aliecs-core` on the head node, in order to stop the core that came with the O²/FLP Suite and use your own. diff --git a/walnut/README.md b/walnut/README.md deleted file mode 100644 index 61e437428..000000000 --- a/walnut/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# `walnut` - the O² workflow administration and linting utility - -The O² **w**orkflow **a**dministration and **l**i**n**ting **ut**ility is a command line program for workflow configuration -and import validation. - -## Configuration file - -You can check the local `walnut` configuration with -```bash -$ walnut about -``` - -## Using `walnut` - -At any step, you can type `$ walnut help ` to get information on what you can do and how, for example: - -```bash -$ walnut help -walnut is a command line program for interacting with the O² Workflow Administration and Linting Utility. - -For more information on the available commands, see the individual documentation for each command. - -Usage: - walnut [command] - -Available Commands: - check check the file passed against a specified schema. - convert Converts a DPL Dump to the required formats. - help Help about any command - -Flags: - --config string optional configuration file for walnut (default $HOME/.config/walnut/settings.yaml) - -h, --help help for walnut - -Use "walnut [command] --help" for more information about a command. -``` - -### Validating a file - -To check if a file conforms to the pre-defined schemas, run the `check` command: -```bash -$ walnut check -``` -Example usage: -```bash -$ walnut check dump.json --format workflow -``` - -A successful validation will return no output while any errors will cause the program to exit. - -#### SEE ALSO - -* [walnut check](./doc/walnut_check.md) - - -### Converting a DPL Dump to Task/Workflow Templates - -The goal of `walnut` is to convert O²/DPL generated dumps to the required number of task templates and workflow templates. The -command used is `convert`: -```bash -$ walnut convert -``` - -Convert can also receive optional arguments to specify which modules to consider while generating task templates. - -Example usage: -```bash -$ walnut convert dump.json --modules "TestValue1 TestValue2 TestValue3" -``` - -A successful validation will return no output while any errors will cause the program to exit. - -#### SEE ALSO - -* [walnut convert](./doc/walnut_convert.md) \ No newline at end of file diff --git a/walnut/app/constants.go b/walnut/app/constants.go deleted file mode 100644 index e7e5a2bc6..000000000 --- a/walnut/app/constants.go +++ /dev/null @@ -1,31 +0,0 @@ -/* - * === This file is part of ALICE O² === - * - * Copyright 2020 CERN and copyright holders of ALICE O². - * Author: Ayaan Zaidi - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * In applying this license CERN does not waive the privileges and - * immunities granted to it by virtue of its status as an - * Intergovernmental Organization or submit itself to any jurisdiction. - */ - -package app - -const ( - NAME = "walnut" - PRETTY_SHORTNAME = "walnut" - PRETTY_FULLNAME = "O² Workflow Administration and Linting Utility" -) diff --git a/walnut/cmd/about.go b/walnut/cmd/about.go deleted file mode 100644 index ac8039341..000000000 --- a/walnut/cmd/about.go +++ /dev/null @@ -1,83 +0,0 @@ -/* - * === This file is part of ALICE O² === - * - * Copyright 2020 CERN and copyright holders of ALICE O². - * Author: Ayaan Zaidi - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * In applying this license CERN does not waive the privileges and - * immunities granted to it by virtue of its status as an - * Intergovernmental Organization or submit itself to any jurisdiction. - */ - -package cmd - -import ( - "fmt" - "time" - - "github.com/AliceO2Group/Control/walnut/app" - "github.com/fatih/color" - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -// aboutCmd represents the about command -var aboutCmd = &cobra.Command{ - Use: "about", - Aliases: []string{}, - Short: fmt.Sprintf("about %s", app.NAME), - Long: `The about command shows some basic information on this utility.`, - Run: func(*cobra.Command, []string) { - color.Set(color.FgHiWhite) - fmt.Print(app.PRETTY_SHORTNAME + " *** ") - color.Set(color.FgHiGreen) - fmt.Printf("The ALICE %s\n", app.PRETTY_FULLNAME) - color.Unset() - fmt.Printf(` -version: %s -config: %s -`, - color.HiGreenString(viper.GetString("version")), - color.HiGreenString(func() string { - if len(viper.ConfigFileUsed()) > 0 { - return viper.ConfigFileUsed() - } - return "builtin" - }())) - - color.Set(color.FgHiBlue) - fmt.Printf("\nCopyright 2017-%d CERN and the copyright holders of ALICE O².\n"+ - "This program is free software: you can redistribute it and/or modify \n"+ - "it under the terms of the GNU General Public License as published by \n"+ - "the Free Software Foundation, either version 3 of the License, or \n"+ - "(at your option) any later version.\n", time.Now().Year()) - color.Unset() - - fmt.Printf(` -bugs: %s -code: %s -maintainer: %s -`, - - color.HiBlueString("https://alice.its.cern.ch/jira/projects/OCTRL"), - color.HiBlueString("https://github.com/AliceO2Group/Control"), - color.HiBlueString("Ayaan Zaidi, ")) - }, -} - -func init() { - rootCmd.AddCommand(aboutCmd) -} diff --git a/walnut/cmd/check.go b/walnut/cmd/check.go deleted file mode 100644 index 587e0d4b5..000000000 --- a/walnut/cmd/check.go +++ /dev/null @@ -1,75 +0,0 @@ -/* - * === This file is part of ALICE O² === - * - * Copyright 2020 CERN and copyright holders of ALICE O². - * Author: Ayaan Zaidi - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * In applying this license CERN does not waive the privileges and - * immunities granted to it by virtue of its status as an - * Intergovernmental Organization or submit itself to any jurisdiction. - */ - -package cmd - -import ( - "fmt" - "io/ioutil" - "os" - - "github.com/spf13/viper" - - "github.com/AliceO2Group/Control/walnut/schemata" - "github.com/spf13/cobra" -) - -// checkCmd represents the check command -var checkCmd = &cobra.Command{ - Use: "check", - Short: "validate the file against a specified schema", - Long: `The check command validates the given file against a specified schema. This file can be a task template, -a workflow template or an O² DPL Dump. Each of these have a schema to validate against. - -Valid schema formats: - workflow task dpl_dump`, - - Run: func(cmd *cobra.Command, args []string) { - format, _ := cmd.Flags().GetString("format") - - for _, filename := range args { - file, err := ioutil.ReadFile(filename) - if err != nil { - fmt.Printf("failed to open file %s: %v", filename, err) - os.Exit(1) - } - err = schemata.Validate(file, format) - if err != nil { - fmt.Printf("validation failed: %v", err) - os.Exit(1) - } - } - }, - Args: cobra.MinimumNArgs(1), -} - -var format string - -func init() { - rootCmd.AddCommand(checkCmd) - - checkCmd.Flags().StringP("format", "f", "", "format to validate against") - viper.BindPFlag("format", checkCmd.Flags().Lookup("format")) - checkCmd.MarkFlagRequired("format") -} diff --git a/walnut/cmd/convert.go b/walnut/cmd/convert.go deleted file mode 100644 index fb675cc68..000000000 --- a/walnut/cmd/convert.go +++ /dev/null @@ -1,245 +0,0 @@ -/* - * === This file is part of ALICE O² === - * - * Copyright 2020 CERN and copyright holders of ALICE O². - * Author: Ayaan Zaidi - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * In applying this license CERN does not waive the privileges and - * immunities granted to it by virtue of its status as an - * Intergovernmental Organization or submit itself to any jurisdiction. - */ - -package cmd - -import ( - "errors" - "fmt" - "io/ioutil" - "log" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" - - "github.com/AlecAivazis/survey/v2" - "github.com/AliceO2Group/Control/core/task/taskclass" - "github.com/mitchellh/go-homedir" - "github.com/spf13/cobra" - "github.com/spf13/viper" - - "github.com/AliceO2Group/Control/common/utils" - "github.com/AliceO2Group/Control/core/workflow" - "github.com/AliceO2Group/Control/walnut/converter" -) - -var outputDir string -var modules []string -var graft string -var workflowName string -var configURIVarname string -var staticConfigURI string -var extraVars string -var taskNamePrefix string - -// convertCmd represents the convert command -var convertCmd = &cobra.Command{ - Use: "convert", - Short: "converts a DPL Dump to the required formats", - Long: `The convert command takes a DPL input and outputs task and workflow templates. Optional flags can be provided to -specify which modules should be used when generating task templates. Control-OCCPlugin is always used as module.`, - - Run: func(cmd *cobra.Command, args []string) { - for _, dumpFile := range args { - // Strip .json from end of filename - nameOfDump := filepath.Base(dumpFile)[:len(filepath.Base(dumpFile))-5] - defaults := map[string]string{ - "user": "flp", - nameOfDump + "_monitoring_url": "no-op://", - "dpl_config": "", - } - - // Open specified DPL Dump - file, err := ioutil.ReadFile(dumpFile) - if err != nil { - err = fmt.Errorf("failed to open file &s: &w", dumpFile, err) - fmt.Println(err.Error()) - os.Exit(1) - } - - if configURIVarname != "" { - defaults[configURIVarname] = staticConfigURI - } else { - defaults[nameOfDump+"_config_uri"] = staticConfigURI - } - - // Import the dump and convert it to []*task.Class - dplDump, err := converter.DPLImporter(file) - taskClass, err := converter.ExtractTaskClasses(dplDump, taskNamePrefix, modules) - - if outputDir == "" { - outputDir, _ = os.Getwd() - } - outputDir, _ = homedir.Expand(outputDir) - - extraVars = strings.TrimSpace(extraVars) - if cmd.Flags().Changed("extra-vars") && len(extraVars) == 0 { - fmt.Println(errors.New("empty list of extra-vars supplied").Error()) - os.Exit(1) - } - - extraVarsMap, err := utils.ParseExtraVars(extraVars) - if err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - - fmt.Printf("OPENED: %s", dumpFile) - - if graft == "" { - if workflowName == "" { - workflowName = nameOfDump - } - - // If not grafting, simply convert dump to WFTs and TTs - err = WriteTemplates(taskClass, workflowName, extraVarsMap, defaults) - if err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - } else { - rolePath := strings.Split(graft, ":") - workflowPath := rolePath[0] - targetRolePath := rolePath[1] - - if workflowName == "" { - workflowName = filepath.Base(workflowPath) - } - - // Open existing workflow - f, err := ioutil.ReadFile(workflowPath) - if err != nil { - log.Fatal(err) - } - root, err := workflow.LoadWorkflow(f) - if err != nil { - log.Fatal(err) - } - - // Convert DPL to yaml.Node - roleToGraft, err := workflow.LoadDPL(taskClass, dumpFile[:len(dumpFile)-5], extraVarsMap) - if err != nil { - log.Fatal(err) - } - roleData, err := workflow.RoleToYAML(roleToGraft) - if err != nil { - log.Fatal(err) - } - - grafted, err := workflow.Graft(&root, targetRolePath, roleData, workflowName) - if err != nil { - log.Fatal(err) - } - path := filepath.Join(outputDir, workflowName+".yaml") - fmt.Println("Writing to: ", path) - err = ioutil.WriteFile(path, grafted, 0644) - if err != nil { - log.Fatal(err) - } - } - } - - isGitRepo, _ := strconv.ParseBool(strings.TrimSpace( - runGitCmd([]string{"rev-parse", "--is-inside-work-tree"}))) - - if isGitRepo { - fmt.Printf(runGitCmd([]string{"status"})) - - result := true - prompt := &survey.Confirm{ - Message: "Would you like to view the git diff?", - Default: true, - } - _ = survey.AskOne(prompt, &result) - - if result { - fmt.Printf(runGitCmd([]string{"diff"})) - } - } - }, - Args: cobra.MinimumNArgs(1), -} - -func runGitCmd(args []string) string { - cmd := exec.Command("git", args...) - cmd.Dir = outputDir - out, err := cmd.Output() - if err != nil { - log.Fatal(err) - } - - return string(out) -} - -func WriteTemplates(taskClass []*taskclass.Class, nameOfDump string, extraVarsMap map[string]string, defaults map[string]string) (err error) { - err = converter.GenerateTaskTemplate(taskClass, outputDir, defaults) - if err != nil { - return fmt.Errorf("conversion to task failed for %s: %w", nameOfDump, err) - } - - role, err := workflow.LoadDPL(taskClass, nameOfDump, extraVarsMap) - err = converter.GenerateWorkflowTemplate(role, outputDir) - if err != nil { - return fmt.Errorf("conversion to workflow failed for %s: %w", nameOfDump, err) - } - - return nil -} - -func init() { - convertCmd.Flags().StringArrayVarP(&modules, "modules", "m", []string{}, "modules to load") - _ = viper.BindPFlag("modules", convertCmd.Flags().Lookup("modules")) - - convertCmd.PersistentFlags().StringVarP(&outputDir, "output-dir", "o", "", - "optional output directory") - _ = viper.BindPFlag("output-dir", rootCmd.Flags().Lookup("output-dir")) - - convertCmd.PersistentFlags().StringVarP(&graft, "graft", "g", "", - "graft converted DPL to an existing template") - _ = viper.BindPFlag("graft", rootCmd.Flags().Lookup("graft")) - - convertCmd.PersistentFlags().StringVarP(&workflowName, "workflow-name", "w", "", - "workflow to graft to") - _ = viper.BindPFlag("workflow-name", rootCmd.Flags().Lookup("workflow-name")) - - convertCmd.PersistentFlags().StringVarP(&configURIVarname, "config-uri-varname", "", "", - "name of config uri var") - _ = viper.BindPFlag("config-uri-varname", rootCmd.Flags().Lookup("config-uri-varname")) - - convertCmd.PersistentFlags().StringVarP(&staticConfigURI, "static-config-uri", "", "", - "path of config uri") - _ = viper.BindPFlag("static-config-uri", rootCmd.Flags().Lookup("static-config-uri")) - - convertCmd.PersistentFlags().StringVarP(&extraVars, "extra-vars", "", "", - "extra vars") - _ = viper.BindPFlag("extra-vars", rootCmd.Flags().Lookup("extra-vars")) - - convertCmd.PersistentFlags().StringVarP(&taskNamePrefix, "task-name-prefix", "", "", - "prefix for task name fields") - _ = viper.BindPFlag("task-name-prefix", rootCmd.Flags().Lookup("task-name-prefix")) - - rootCmd.AddCommand(convertCmd) -} diff --git a/walnut/cmd/root.go b/walnut/cmd/root.go deleted file mode 100644 index ef6a7db1b..000000000 --- a/walnut/cmd/root.go +++ /dev/null @@ -1,100 +0,0 @@ -/* - * === This file is part of ALICE O² === - * - * Copyright 2020 CERN and copyright holders of ALICE O². - * Author: Ayaan Zaidi - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * In applying this license CERN does not waive the privileges and - * immunities granted to it by virtue of its status as an - * Intergovernmental Organization or submit itself to any jurisdiction. - */ - -package cmd - -import ( - "fmt" - "os" - - "github.com/AliceO2Group/Control/walnut/app" - "github.com/spf13/cobra" - - homedir "github.com/mitchellh/go-homedir" - "github.com/spf13/viper" -) - -var cfgFile string - -// rootCmd represents the base command when called without any subcommands -var rootCmd = &cobra.Command{ - Use: app.NAME, - Short: app.PRETTY_FULLNAME, - Long: fmt.Sprintf(`%s is a command line program for interacting with the %s. - -For more information on the available commands, see the individual documentation for each command.`, app.PRETTY_SHORTNAME, app.PRETTY_FULLNAME), - // Uncomment the following line if your bare application - // has an action associated with it: - // Run: func(cmd *cobra.Command, args []string) { }, -} - -// Execute adds all child commands to the root command and sets flags appropriately. -// This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - if err := rootCmd.Execute(); err != nil { - fmt.Println(err) - os.Exit(1) - } -} - -func GetRootCmd() *cobra.Command { // Used for docs generator - return rootCmd -} - -func init() { - cobra.OnInitialize(initConfig) - - // Here you will define your flags and configuration settings. - // Cobra supports persistent flags, which, if defined here, - // will be global for your application. - - rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", - fmt.Sprintf("optional configuration file for %s (default $HOME/.config/%s/settings.yaml)", app.NAME, app.NAME)) -} - -// initConfig reads in config file and ENV variables if set. -func initConfig() { - if cfgFile != "" { - // Use config file from the flag. - viper.SetConfigFile(cfgFile) - } else { - // Find home directory. - home, err := homedir.Dir() - if err != nil { - fmt.Println(err) - os.Exit(1) - } - - // Search config in home directory with name ".walnut" (without extension). - viper.AddConfigPath(home) - viper.SetConfigName(".walnut") - } - - viper.AutomaticEnv() // read in environment variables that match - - // If a config file is found, read it in. - if err := viper.ReadInConfig(); err == nil { - fmt.Println("Using config file:", viper.ConfigFileUsed()) - } -} diff --git a/walnut/converter/converter.go b/walnut/converter/converter.go deleted file mode 100644 index 0c3bf75a8..000000000 --- a/walnut/converter/converter.go +++ /dev/null @@ -1,225 +0,0 @@ -/* - * === This file is part of ALICE O² === - * - * Copyright 2020 CERN and copyright holders of ALICE O². - * Author: Ayaan Zaidi - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * In applying this license CERN does not waive the privileges and - * immunities granted to it by virtue of its status as an - * Intergovernmental Organization or submit itself to any jurisdiction. - */ - -package converter - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" - - "github.com/AlecAivazis/survey/v2" - "github.com/AliceO2Group/Control/core/task/taskclass" - "github.com/AliceO2Group/Control/core/task/taskclass/port" - "github.com/mitchellh/go-homedir" - "gopkg.in/yaml.v3" - - "github.com/AliceO2Group/Control/common" - "github.com/AliceO2Group/Control/common/controlmode" - "github.com/AliceO2Group/Control/common/gera" - "github.com/AliceO2Group/Control/core/task/channel" - "github.com/AliceO2Group/Control/core/workflow" -) - -// return pointer to float64 -func createFloat(x float64) *float64 { - return &x -} - -// return pointer to bool -func createBool(x bool) *bool { - return &x -} - -// return pointer to string -func createString(x string) *string { - return &x -} - -// ExtractTaskClasses takes in a DPL Dump string and extracts -// an array of Tasks -func ExtractTaskClasses(dplDump Dump, taskNamePrefix string, envModules []string) (tasks []*taskclass.Class, err error) { - envModules = append(envModules, "Control-OCCPlugin") - - for index := range dplDump.Workflows { - taskName := dplDump.Workflows[index].Name - if taskNamePrefix != "" { - taskNamePrefix += "_" - } - - correspondingMetadata := index + 1 // offset to match workflowEntry with correct metadataEntry - channelNames := dplDump.Metadata[correspondingMetadata].Channels - - task := taskclass.Class{ - Identifier: taskclass.Id{ - Name: taskNamePrefix + taskName, - }, - Control: struct { - Mode controlmode.ControlMode "yaml:\"mode\"" - }{Mode: controlmode.FAIRMQ}, - Command: &common.CommandInfo{ - Env: []string{}, // -> Default to empty array - Shell: createBool(true), - Arguments: sanitizeCmdLineArgs(dplDump.Metadata[correspondingMetadata].CmdlLineArgs, - taskName), - }, - Wants: taskclass.ResourceWants{ - Cpu: createFloat(0.15), - Memory: createFloat(128), - Ports: port.Ranges{}, // begin - end OR range - }, - Properties: gera.MakeMapWithMap(map[string]string{ - "severity": "trace", - "color": "false", - }), - } - - value := fmt.Sprintf("eval `aliswmod load %s` &&\n%s", strings.Join(envModules, " "), - dplDump.Metadata[correspondingMetadata].Executable) - value = fmt.Sprintf("%s &&\n%s", value, "cat {{ dpl_config }} | dump-name") - task.Command.Value = &value - - for _, channelName := range channelNames { - // To avoid duplication, only "push" channels are included - if strings.Contains(channelName, "from_"+taskName) { - singleBind := channel.Inbound{ - Channel: channel.Channel{ - Name: channelName, - Type: channel.ChannelType("push"), - Transport: channel.TransportType("shmem"), - }, - Addressing: "ipc", - } - task.Bind = append(task.Bind, singleBind) - } - - if strings.Contains(channelName, "to_"+taskName) { - // String manipulation to generate channel target of the form: - // {{ Parent().Path }}.taskName:from_{initiator}_to_{receiver} - initiator := channelName[5:strings.Index(channelName, "_to")] - singleConnect := channel.Outbound{ - Channel: channel.Channel{ - Name: channelName, - Type: channel.ChannelType("pull"), - Transport: channel.TransportType("shmem"), - Target: fmt.Sprintf("{{ Parent().Path }}.%s:%s", initiator, channelName), - }, - } - task.Connect = append(task.Connect, singleConnect) - } - } - - tasks = append(tasks, &task) - } - return tasks, nil -} - -func sanitizeCmdLineArgs(input []string, taskName string) (output []string) { - for index, value := range input { - // Check args for dump arguments and remove them - if !(strings.Contains(value, "--dump-workflow") || - (strings.Contains(value, ".json") && input[index-1] == "--dump-workflow-file")) { - output = append(output, value) - } - } - // Add --id parameter along with name of task to arguments - output = append(output, "--id", taskName) - - return output -} - -// GenerateTaskTemplate takes as input an array of pointers to task.Class -// and writes them to a AliECS friendly YAML file -func GenerateTaskTemplate(extractedTasks []*taskclass.Class, outputDir string, defaults map[string]string) (err error) { - path := filepath.Join(outputDir, "tasks") - path, _ = homedir.Expand(path) - _ = os.MkdirAll(path, os.ModePerm) - - for _, SingleTask := range extractedTasks { - SingleTask.Defaults = gera.MakeMapWithMap(defaults) - SingleTask.Command.User = createString("{{ user }}") - - YAMLData, err := yaml.Marshal(&SingleTask) - if err != nil { - return fmt.Errorf("marshal failed: %v", err) - } - - fileName := filepath.Join(path, SingleTask.Identifier.Name+".yaml") - if _, err := os.Stat(fileName); !os.IsNotExist(err) { - result := false - prompt := &survey.Confirm{ - Message: fmt.Sprintf("%s already exists, overwrite?", fileName), - Default: false, - } - _ = survey.AskOne(prompt, &result) - if result { - continue - } - } - - f, err := os.Create(fileName) - defer f.Close() - - // Write marshaled YAML to file - err = ioutil.WriteFile(fileName, YAMLData, 0644) - if err != nil { - return fmt.Errorf("creating file failed: %v", err) - } - } - - return -} - -func GenerateWorkflowTemplate(input workflow.Role, outputDir string) (err error) { - path := filepath.Join(outputDir, "workflows") - path, _ = homedir.Expand(path) - _ = os.MkdirAll(path, os.ModePerm) - - yamlDATA, err := workflow.RoleToYAML(input) - if err != nil { - return fmt.Errorf("error converting role to YAML: %v", err) - } - - fileName := filepath.Join(path, input.GetName()+".yaml") - if _, err := os.Stat(fileName); !os.IsNotExist(err) { - result := false - prompt := &survey.Confirm{ - Message: fmt.Sprintf("%s already exists, overwrite?", fileName), - Default: false, - } - _ = survey.AskOne(prompt, &result) - if result { - return nil - } - } - - err = ioutil.WriteFile(fileName, yamlDATA, 0644) - if err != nil { - return fmt.Errorf("error writing role to file: %v", err) - } - - return -} diff --git a/walnut/converter/converter_test.go b/walnut/converter/converter_test.go deleted file mode 100644 index c32ae0add..000000000 --- a/walnut/converter/converter_test.go +++ /dev/null @@ -1,115 +0,0 @@ -/* - * === This file is part of ALICE O² === - * - * Copyright 2020 CERN and copyright holders of ALICE O². - * Author: Ayaan Zaidi - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * In applying this license CERN does not waive the privileges and - * immunities granted to it by virtue of its status as an - * Intergovernmental Organization or submit itself to any jurisdiction. - */ - -package converter - -import ( - "io/ioutil" - "os" - "path/filepath" - "testing" - - "github.com/k0kubun/pp" - - "github.com/AliceO2Group/Control/core/workflow" -) - -func TestExtractClass(t *testing.T) { - t.Run("Testing class extraction", func(t *testing.T) { - extractedClasses, err := ExtractTaskClasses(TestDump, "", []string{}) - pp.Print(extractedClasses) - if err != nil { - t.Errorf("extract Task Class failed: %v", err) - } - }) -} - -func TestGenerateTaskTemplate(t *testing.T) { - t.Run("Testing Task -> YAML", func(t *testing.T) { - allTasks, err := ExtractTaskClasses(TestDump, "testPrefix", []string{"QualityControl"}) - if err != nil { - t.Errorf("extract Task Class failed: %v", err) - } - - err = GenerateTaskTemplate(allTasks, "dump", defaults) - if err != nil { - t.Errorf("failed to write YAML to file: %v", err) - } - }) -} - -func TestTaskToRole(t *testing.T) { - t.Run("Testing Task -> workflow.Role", func(t *testing.T) { - allTasks, err := ExtractTaskClasses(TestDump, "", []string{}) - if err != nil { - t.Errorf("extract Task Class failed: %v", err) - } - - role, err := workflow.LoadDPL(allTasks, "dump", extraVars) - if err != nil { - t.Errorf("error loading task to role: %v", err) - } - _, _ = pp.Print(role) - }) -} - -func TestGenerateWorkflowTemplate(t *testing.T) { - t.Run("Testing Role -> workflow.yaml", func(t *testing.T) { - dumpFile, err := ioutil.ReadFile("dump.json") - dump, err := DPLImporter(dumpFile) - allTasks, err := ExtractTaskClasses(dump, "testPrefix", []string{}) - if err != nil { - t.Errorf("extract Task Class failed: %v", err) - } - - role, err := workflow.LoadDPL(allTasks, "dump", extraVars) - if err != nil { - t.Errorf("error loading task to role: %v", err) - } - - err = GenerateWorkflowTemplate(role, "dump") - if err != nil { - t.Errorf("error converting Role to YAML: %v", err) - } - }) -} - -func TestGraft(t *testing.T) { - t.Run("test grafting role", func(t *testing.T) { - f1, _ := ioutil.ReadFile("small.yaml") - root, _ := workflow.LoadWorkflow(f1) - f2, _ := ioutil.ReadFile("dump.yaml") - - result, err := workflow.Graft(&root, "readout.host-{{ it }}", f2, "small") - if err != nil { - t.Error(err) - } - - wd, _ := os.Getwd() - err = ioutil.WriteFile(filepath.Join(wd, "test", "grafted.yaml"), result, 0644) - if err != nil { - t.Error(err) - } - }) -} diff --git a/walnut/converter/dump.json b/walnut/converter/dump.json deleted file mode 100644 index 591b5c1ca..000000000 --- a/walnut/converter/dump.json +++ /dev/null @@ -1,390 +0,0 @@ -{ - "workflow": [ - { - "name": "producer-0", - "inputs": [], - "outputs": [ - { - "binding": "out", - "origin": "TST", - "description": "RAWDATA", - "subspec": 0, - "lifetime": 0 - } - ], - "options": [], - "rank": 0, - "nSlots": 1, - "inputTimeSliceId": 0, - "maxInputTimeslices": 1 - }, - { - "name": "Dispatcher", - "inputs": [ - { - "binding": "random", - "origin": "TST", - "description": "RAWDATA", - "subspec": 0, - "lifetime": 0 - }, - { - "binding": "timer-stats", - "origin": "DS", - "description": "TIMER-Dispatcher", - "subspec": 0, - "lifetime": 4 - } - ], - "outputs": [ - { - "binding": "random", - "origin": "DS", - "description": "tst-raw-0", - "subspec": 0, - "lifetime": 0 - } - ], - "options": [ - { - "name": "period-timer-stats", - "type": "0", - "defaultValue": "10000000", - "help": "Dispatcher's stats timer period" - } - ], - "rank": 0, - "nSlots": 1, - "inputTimeSliceId": 0, - "maxInputTimeslices": 1 - }, - { - "name": "QC-TASK-RUNNER-QcTask", - "inputs": [ - { - "binding": "random", - "origin": "DS", - "description": "tst-raw-0", - "subspec": 0, - "lifetime": 0 - }, - { - "binding": "timer-cycle", - "origin": "QC", - "description": "TIMER-QcTask-mo", - "subspec": 0, - "lifetime": 4 - } - ], - "outputs": [ - { - "binding": "mo", - "origin": "QC", - "description": "QcTask-mo", - "subspec": 0, - "lifetime": 0 - } - ], - "options": [ - { - "name": "period-timer-cycle", - "type": "0", - "defaultValue": "10000000", - "help": "timer period" - } - ], - "rank": 0, - "nSlots": 1, - "inputTimeSliceId": 0, - "maxInputTimeslices": 1 - }, - { - "name": "QC-CHECK-RUNNER-QcCheck", - "inputs": [ - { - "binding": "QcTask", - "origin": "QC", - "description": "QcTask-mo", - "subspec": 0, - "lifetime": 0 - } - ], - "outputs": [ - { - "binding": "QC/QcCheck-chk/0", - "origin": "QC", - "description": "QcCheck-chk", - "subspec": 0, - "lifetime": 0 - } - ], - "options": [], - "rank": 0, - "nSlots": 1, - "inputTimeSliceId": 0, - "maxInputTimeslices": 1 - }, - { - "name": "printer", - "inputs": [ - { - "binding": "checked-mo", - "origin": "QC", - "description": "QcCheck-chk", - "subspec": 0, - "lifetime": 0 - } - ], - "outputs": [], - "options": [], - "rank": 0, - "nSlots": 1, - "inputTimeSliceId": 0, - "maxInputTimeslices": 1 - } - ], - "metadata": [ - { - "name": "internal-dpl-clock", - "executable": "o2-qc-run-basic", - "cmdLineArgs": [ - "-b", - "--dump-workflow", - "--dump-workflow-file", - "dpl-dump.json" - ], - "workflowOptions": [ - { - "name": "config-path", - "type": "4", - "defaultValue": "", - "help": "Absolute path to the config file. Overwrite the default paths. Do not use with no-data-sampling." - }, - { - "name": "no-data-sampling", - "type": "5", - "defaultValue": "0", - "help": "Skips data sampling, connects directly the task to the producer." - }, - { - "name": "readers", - "type": "1", - "defaultValue": "1", - "help": "number of parallel readers to use" - }, - { - "name": "pipeline", - "type": "4", - "defaultValue": "", - "help": "override default pipeline size" - } - ], - "channels": [ - "from_internal-dpl-clock_to_producer-0", - "from_internal-dpl-clock_to_Dispatcher", - "from_internal-dpl-clock_to_QC-TASK-RUNNER-QcTask" - ] - }, - { - "name": "producer-0", - "executable": "o2-qc-run-basic", - "cmdLineArgs": [ - "-b", - "--dump-workflow", - "--dump-workflow-file", - "dpl-dump.json" - ], - "workflowOptions": [ - { - "name": "config-path", - "type": "4", - "defaultValue": "", - "help": "Absolute path to the config file. Overwrite the default paths. Do not use with no-data-sampling." - }, - { - "name": "no-data-sampling", - "type": "5", - "defaultValue": "0", - "help": "Skips data sampling, connects directly the task to the producer." - }, - { - "name": "readers", - "type": "1", - "defaultValue": "1", - "help": "number of parallel readers to use" - }, - { - "name": "pipeline", - "type": "4", - "defaultValue": "", - "help": "override default pipeline size" - } - ], - "channels": [ - "from_internal-dpl-clock_to_producer-0", - "from_producer-0_to_Dispatcher" - ] - }, - { - "name": "Dispatcher", - "executable": "o2-qc-run-basic", - "cmdLineArgs": [ - "-b", - "--dump-workflow", - "--dump-workflow-file", - "dpl-dump.json" - ], - "workflowOptions": [ - { - "name": "config-path", - "type": "4", - "defaultValue": "", - "help": "Absolute path to the config file. Overwrite the default paths. Do not use with no-data-sampling." - }, - { - "name": "no-data-sampling", - "type": "5", - "defaultValue": "0", - "help": "Skips data sampling, connects directly the task to the producer." - }, - { - "name": "readers", - "type": "1", - "defaultValue": "1", - "help": "number of parallel readers to use" - }, - { - "name": "pipeline", - "type": "4", - "defaultValue": "", - "help": "override default pipeline size" - } - ], - "channels": [ - "from_internal-dpl-clock_to_Dispatcher", - "from_producer-0_to_Dispatcher", - "from_Dispatcher_to_QC-TASK-RUNNER-QcTask" - ] - }, - { - "name": "QC-TASK-RUNNER-QcTask", - "executable": "o2-qc-run-basic", - "cmdLineArgs": [ - "-b", - "--dump-workflow", - "--dump-workflow-file", - "dpl-dump.json" - ], - "workflowOptions": [ - { - "name": "config-path", - "type": "4", - "defaultValue": "", - "help": "Absolute path to the config file. Overwrite the default paths. Do not use with no-data-sampling." - }, - { - "name": "no-data-sampling", - "type": "5", - "defaultValue": "0", - "help": "Skips data sampling, connects directly the task to the producer." - }, - { - "name": "readers", - "type": "1", - "defaultValue": "1", - "help": "number of parallel readers to use" - }, - { - "name": "pipeline", - "type": "4", - "defaultValue": "", - "help": "override default pipeline size" - } - ], - "channels": [ - "from_internal-dpl-clock_to_QC-TASK-RUNNER-QcTask", - "from_Dispatcher_to_QC-TASK-RUNNER-QcTask", - "from_QC-TASK-RUNNER-QcTask_to_QC-CHECK-RUNNER-QcCheck" - ] - }, - { - "name": "QC-CHECK-RUNNER-QcCheck", - "executable": "o2-qc-run-basic", - "cmdLineArgs": [ - "-b", - "--dump-workflow", - "--dump-workflow-file", - "dpl-dump.json" - ], - "workflowOptions": [ - { - "name": "config-path", - "type": "4", - "defaultValue": "", - "help": "Absolute path to the config file. Overwrite the default paths. Do not use with no-data-sampling." - }, - { - "name": "no-data-sampling", - "type": "5", - "defaultValue": "0", - "help": "Skips data sampling, connects directly the task to the producer." - }, - { - "name": "readers", - "type": "1", - "defaultValue": "1", - "help": "number of parallel readers to use" - }, - { - "name": "pipeline", - "type": "4", - "defaultValue": "", - "help": "override default pipeline size" - } - ], - "channels": [ - "from_QC-TASK-RUNNER-QcTask_to_QC-CHECK-RUNNER-QcCheck", - "from_QC-CHECK-RUNNER-QcCheck_to_printer" - ] - }, - { - "name": "printer", - "executable": "o2-qc-run-basic", - "cmdLineArgs": [ - "-b", - "--dump-workflow", - "--dump-workflow-file", - "dpl-dump.json" - ], - "workflowOptions": [ - { - "name": "config-path", - "type": "4", - "defaultValue": "", - "help": "Absolute path to the config file. Overwrite the default paths. Do not use with no-data-sampling." - }, - { - "name": "no-data-sampling", - "type": "5", - "defaultValue": "0", - "help": "Skips data sampling, connects directly the task to the producer." - }, - { - "name": "readers", - "type": "1", - "defaultValue": "1", - "help": "number of parallel readers to use" - }, - { - "name": "pipeline", - "type": "4", - "defaultValue": "", - "help": "override default pipeline size" - } - ], - "channels": [ - "from_QC-CHECK-RUNNER-QcCheck_to_printer" - ] - } - ] -} diff --git a/walnut/converter/dump.yaml b/walnut/converter/dump.yaml deleted file mode 100644 index 3d4ac88c2..000000000 --- a/walnut/converter/dump.yaml +++ /dev/null @@ -1,83 +0,0 @@ -name: readout-qc -defaults: - monitoring_qc_url: "no-op://" - monitoring_dd_url: "no-op://" - monitoring_readout_url: "no-op://" -roles: - - name: readout-{{ it }} - for: - range: "{{ hosts }}" - var: it - constraints: - - attribute: machine_id - value: "{{ it }}" - vars: - readout_cfg_uri: "file:/home/flp/readout.cfg" - roles: - - name: "readout" - task: - load: readout - - name: qc-single-subwf - defaults: - qchost: "{{ FromJson(hosts)[0] }}" - dpl_config: "/etc/flp.d/qc/readout-qc.dpl.json" - constraints: - - attribute: machine_id - value: "{{ qchost }}" - roles: - - name: "readout-proxy" - connect: - - name: "from_internal-dpl-clock_to_readout-proxy" - target: "{{Parent().Path}}.internal-dpl-clock:from_internal-dpl-clock_to_readout-proxy" - type: "pull" - rateLogging: "60" - - name: "readout-proxy" - target: "ipc:///tmp/readout-pipe-1" - # readout.cfg [consumer-data-sampling] is hardcoded as channel "data-out" with type: pub and transport: zeromq - # shmem is only possible in push-pull mode - type: "sub" - rateLogging: "1" - task: - load: readout-proxy - - name: "internal-dpl-clock" - task: - load: readout-internal-dpl-clock - - name: "Dispatcher" - connect: - - name: "from_readout-proxy_to_Dispatcher" - target: "{{Parent().Path}}.readout-proxy:from_readout-proxy_to_Dispatcher" - type: "pull" - rateLogging: "60" - - name: "from_internal-dpl-clock_to_Dispatcher" - target: "{{Parent().Path}}.internal-dpl-clock:from_internal-dpl-clock_to_Dispatcher" - type: "pull" - rateLogging: "60" - task: - load: readout-qc-dispatcher - - name: "QC-TASK-RUNNER-daqTask" - connect: - - name: "from_internal-dpl-clock_to_QC-TASK-RUNNER-daqTask" - target: "{{Parent().Path}}.internal-dpl-clock:from_internal-dpl-clock_to_QC-TASK-RUNNER-daqTask" - type: "pull" - rateLogging: "60" - - name: "from_Dispatcher_to_QC-TASK-RUNNER-daqTask" - target: "{{Parent().Path}}.Dispatcher:from_Dispatcher_to_QC-TASK-RUNNER-daqTask" - type: "pull" - rateLogging: "60" - task: - load: readout-qc-task-runner - - name: "QC-CHECK-RUNNER-Q4H0" - connect: - - name: "from_QC-TASK-RUNNER-daqTask_to_QC-CHECK-RUNNER-Q4H0" - target: "{{Parent().Path}}.QC-TASK-RUNNER-daqTask:from_QC-TASK-RUNNER-daqTask_to_QC-CHECK-RUNNER-Q4H0" - type: "pull" - rateLogging: "60" - task: - load: readout-qc-check-runner - - name: "internal-dpl-global-binary-file-sink" - connect: - - name: "from_QC-CHECK-RUNNER-Q4H0_to_internal-dpl-global-binary-file-sink" - target: "{{Parent().Path}}.QC-CHECK-RUNNER-Q4H0:from_QC-CHECK-RUNNER-Q4H0_to_internal-dpl-global-binary-file-sink" - type: "pull" - task: - load: readout-internal-dpl-global-binary-file-sink diff --git a/walnut/converter/importer.go b/walnut/converter/importer.go deleted file mode 100644 index a8e927396..000000000 --- a/walnut/converter/importer.go +++ /dev/null @@ -1,78 +0,0 @@ -/* - * === This file is part of ALICE O² === - * - * Copyright 2020 CERN and copyright holders of ALICE O². - * Author: Ayaan Zaidi - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * In applying this license CERN does not waive the privileges and - * immunities granted to it by virtue of its status as an - * Intergovernmental Organization or submit itself to any jurisdiction. - */ - -package converter - -import ( - "encoding/json" - "fmt" -) - -type io struct { - Binding string `json:"binding"` - Origin string `json:"origin"` - Description string `json:"description"` - Subspec int `json:"subspec"` - Lifetime int `json:"lifetime"` -} - -type options struct { - Name string `json:"name"` - Type string `json:"type"` - DefaultValue string `json:"defaultValue"` - Help string `json:"help"` -} -type workflowEntry struct { - Name string `json:"name"` - Inputs []io `json:"inputs"` - Outputs []io `json:"outputs"` - Options []options `json:"options"` - Rank int `json:"rank"` - NSlots int `json:"nSlots"` - InputTimeSliceID int `json:"inputTimeSliceId"` - MaxInputTimeslices int `json:"maxInputTimeslices"` -} - -type metadataEntry struct { - Name string `json:"name"` - Executable string `json:"executable"` - CmdlLineArgs []string `json:"cmdLineArgs"` - WorkflowOptions []options `json:"workflowOptions"` - Channels []string `json:"channels"` -} - -// Dump is a 1:1 struct representation of a DPL Dump -type Dump struct { - Workflows []workflowEntry `json:"workflow"` - Metadata []metadataEntry `json:"metadata"` -} - -func DPLImporter(input []byte) (importedJSON Dump, err error) { - err = json.Unmarshal(input, &importedJSON) - if err != nil { - return importedJSON, fmt.Errorf("JSON Unmarshal failed: %w", err) - } - - return importedJSON, nil -} diff --git a/walnut/converter/importer_test.go b/walnut/converter/importer_test.go deleted file mode 100644 index 1a53f2bb9..000000000 --- a/walnut/converter/importer_test.go +++ /dev/null @@ -1,44 +0,0 @@ -/* - * === This file is part of ALICE O² === - * - * Copyright 2020 CERN and copyright holders of ALICE O². - * Author: Ayaan Zaidi - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * In applying this license CERN does not waive the privileges and - * immunities granted to it by virtue of its status as an - * Intergovernmental Organization or submit itself to any jurisdiction. - */ - -package converter - -import ( - "reflect" - "testing" -) - -func TestDPLImporter(t *testing.T) { - t.Run("Testing dump reading", func(t *testing.T) { - got, err := DPLImporter([]byte(TestJSON)) - want := TestDump - if err != nil { - t.Errorf("Import failed: %v", err) - } else { - if !reflect.DeepEqual(got, want) { - t.Error("Unmarshalled value does not match") - } - } - }) -} diff --git a/walnut/converter/small.yaml b/walnut/converter/small.yaml deleted file mode 100644 index 1d4d055e2..000000000 --- a/walnut/converter/small.yaml +++ /dev/null @@ -1,43 +0,0 @@ -name: readout-stfb -defaults: - roc_ctp_emulator_enabled: "true" - stfb_enabled: "true" -roles: - - name: host-{{ it }} - for: - range: "{{ hosts }}" - var: it - vars: - readout_cfg_uri: '{{stfb_enabled == "true" ? "file:/home/flp/readout_stfb_emu.cfg" : "file:/home/flp/readout.cfg"}}' - constraints: - - attribute: machine_id - value: "{{ it }}" - roles: - - name: "readout" - task: - load: readout - - name: "stfb" - enabled: "{{ stfb_enabled }}" - connect: - - name: readout - type: pull - target: "{{ Parent().Path }}.readout:readout" - rateLogging: "1" - task: - load: stfbuilder-nooutput - - name: roc-ctp-emulators - enabled: "{{roc_ctp_emulator_enabled}}" - defaults: - roc_ctp_emulator_endpoints: '["#0"]' - roles: - - name: "endpoint-{{ endpoint_id }}" - for: - range: "{{roc_ctp_emulator_endpoints}}" - var: endpoint_id - roles: - - name: roc-ctp-emulator - task: - load: "roc-ctp-emulator" - trigger: "enter_RUNNING" - timeout: 10s - critical: false \ No newline at end of file diff --git a/walnut/converter/testvalues.go b/walnut/converter/testvalues.go deleted file mode 100644 index 70b178609..000000000 --- a/walnut/converter/testvalues.go +++ /dev/null @@ -1,248 +0,0 @@ -/* - * === This file is part of ALICE O² === - * - * Copyright 2020 CERN and copyright holders of ALICE O². - * Author: Ayaan Zaidi - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * In applying this license CERN does not waive the privileges and - * immunities granted to it by virtue of its status as an - * Intergovernmental Organization or submit itself to any jurisdiction. - */ - -package converter - -var defaults = map[string]string{ - "qc_config_uri": "json:///etc/flp.d/qc/readout.json", - "user": "test-user-1", -} - -var extraVars = map[string]string{ - "readout_cfg_uri": "file:/home/flp/readout_stfb_emu.cfg", -} - -var TestDump = Dump{ - Workflows: []workflowEntry{ - { - Name: "producer-0", - Inputs: []io{}, - Outputs: []io{ - { - Binding: "out", - Origin: "TST", - Description: "RAWDATA", - Subspec: 0, - Lifetime: 0, - }, - }, - Options: []options{}, - Rank: 0, - NSlots: 1, - InputTimeSliceID: 0, - MaxInputTimeslices: 1, - }, - }, - Metadata: []metadataEntry{ - { - Name: "internal-dpl-clock", - Executable: "o2-qc-run-basic", - CmdlLineArgs: []string{ - "-b", - "--dump-workflow", - "--dump-workflow-file", - "dpl-dump.json", - }, - WorkflowOptions: []options{ - { - Name: "config-path", - Type: "4", - DefaultValue: "", - Help: "Absolute path to the config file. Overwrite the default paths. Do not use with no-data-sampling.", - }, - { - Name: "no-data-sampling", - Type: "5", - DefaultValue: "0", - Help: "Skips data sampling, connects directly the task to the producer.", - }, - { - Name: "readers", - Type: "1", - DefaultValue: "1", - Help: "number of parallel readers to use", - }, - { - Name: "pipeline", - Type: "4", - DefaultValue: "", - Help: "override default pipeline size", - }, - }, - Channels: []string{ - "from_internal-dpl-clock_to_producer-0", - "from_internal-dpl-clock_to_Dispatcher", - "from_internal-dpl-clock_to_QC-TASK-RUNNER-QcTask", - }, - }, - { - Name: "producer-0", - Executable: "o2-qc-run-basic", - CmdlLineArgs: []string{ - "-b", - "--dump-workflow", - "--dump-workflow-file", - "dpl-dump.json", - }, - WorkflowOptions: []options{ - { - Name: "config-path", - Type: "4", - DefaultValue: "", - Help: "Absolute path to the config file. Overwrite the default paths. Do not use with no-data-sampling.", - }, - { - Name: "no-data-sampling", - Type: "5", - DefaultValue: "0", - Help: "Skips data sampling, connects directly the task to the producer.", - }, - { - Name: "readers", - Type: "1", - DefaultValue: "1", - Help: "number of parallel readers to use", - }, - { - Name: "pipeline", - Type: "4", - DefaultValue: "", - Help: "override default pipeline size", - }, - }, - Channels: []string{ - "from_internal-dpl-clock_to_producer-0", - "from_producer-0_to_Dispatcher", - }, - }, - }, -} - -var TestJSON = ` -{ - "workflow": [ - { - "name": "producer-0", - "inputs": [], - "outputs": [ - { - "binding": "out", - "origin": "TST", - "description": "RAWDATA", - "subspec": 0, - "lifetime": 0 - } - ], - "options": [], - "rank": 0, - "nSlots": 1, - "inputTimeSliceId": 0, - "maxInputTimeslices": 1 - } - ], - "metadata": [ - { - "name": "internal-dpl-clock", - "executable": "o2-qc-run-basic", - "cmdLineArgs": [ - "-b", - "--dump-workflow", - "--dump-workflow-file", - "dpl-dump.json" - ], - "workflowOptions": [ - { - "name": "config-path", - "type": "4", - "defaultValue": "", - "help": "Absolute path to the config file. Overwrite the default paths. Do not use with no-data-sampling." - }, - { - "name": "no-data-sampling", - "type": "5", - "defaultValue": "0", - "help": "Skips data sampling, connects directly the task to the producer." - }, - { - "name": "readers", - "type": "1", - "defaultValue": "1", - "help": "number of parallel readers to use" - }, - { - "name": "pipeline", - "type": "4", - "defaultValue": "", - "help": "override default pipeline size" - } - ], - "channels": [ - "from_internal-dpl-clock_to_producer-0", - "from_internal-dpl-clock_to_Dispatcher", - "from_internal-dpl-clock_to_QC-TASK-RUNNER-QcTask" - ] - }, - { - "name": "producer-0", - "executable": "o2-qc-run-basic", - "cmdLineArgs": [ - "-b", - "--dump-workflow", - "--dump-workflow-file", - "dpl-dump.json" - ], - "workflowOptions": [ - { - "name": "config-path", - "type": "4", - "defaultValue": "", - "help": "Absolute path to the config file. Overwrite the default paths. Do not use with no-data-sampling." - }, - { - "name": "no-data-sampling", - "type": "5", - "defaultValue": "0", - "help": "Skips data sampling, connects directly the task to the producer." - }, - { - "name": "readers", - "type": "1", - "defaultValue": "1", - "help": "number of parallel readers to use" - }, - { - "name": "pipeline", - "type": "4", - "defaultValue": "", - "help": "override default pipeline size" - } - ], - "channels": [ - "from_internal-dpl-clock_to_producer-0", - "from_producer-0_to_Dispatcher" - ] - } - ] -} -` diff --git a/walnut/doc/generate.go b/walnut/doc/generate.go deleted file mode 100644 index 8bb8159fc..000000000 --- a/walnut/doc/generate.go +++ /dev/null @@ -1,43 +0,0 @@ -/* - * === This file is part of ALICE O² === - * - * Copyright 2020 CERN and copyright holders of ALICE O². - * Author: Ayaan Zaidi - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * In applying this license CERN does not waive the privileges and - * immunities granted to it by virtue of its status as an - * Intergovernmental Organization or submit itself to any jurisdiction. - */ - -package main - -import ( - "github.com/AliceO2Group/Control/common/logger" - "github.com/AliceO2Group/Control/walnut/app" - "github.com/AliceO2Group/Control/walnut/cmd" - "github.com/sirupsen/logrus" - "github.com/spf13/cobra/doc" -) - -var log = logger.New(logrus.StandardLogger(), app.NAME) - -func main() { - rootCmd := cmd.GetRootCmd() - err := doc.GenMarkdownTree(rootCmd, "./") - if err != nil { - log.Fatal(err) - } -} diff --git a/walnut/doc/walnut.md b/walnut/doc/walnut.md deleted file mode 100644 index 8451401ab..000000000 --- a/walnut/doc/walnut.md +++ /dev/null @@ -1,24 +0,0 @@ -## walnut - -O² Workflow Administration and Linting Utility - -### Synopsis - -walnut is a command line program for interacting with the O² Workflow Administration and Linting Utility. - -For more information on the available commands, see the individual documentation for each command. - -### Options - -``` - --config string optional configuration file for walnut (default $HOME/.config/walnut/settings.yaml) - -h, --help help for walnut -``` - -### SEE ALSO - -* [walnut about](walnut_about.md) - about walnut -* [walnut check](walnut_check.md) - validate the file against a specified schema -* [walnut convert](walnut_convert.md) - converts a DPL Dump to the required formats - -###### Auto generated by spf13/cobra on 10-Jul-2020 diff --git a/walnut/doc/walnut_about.md b/walnut/doc/walnut_about.md deleted file mode 100644 index 9006334ad..000000000 --- a/walnut/doc/walnut_about.md +++ /dev/null @@ -1,29 +0,0 @@ -## walnut about - -about walnut - -### Synopsis - -The about command shows some basic information on this utility. - -``` -walnut about [flags] -``` - -### Options - -``` - -h, --help help for about -``` - -### Options inherited from parent commands - -``` - --config string optional configuration file for walnut (default $HOME/.config/walnut/settings.yaml) -``` - -### SEE ALSO - -* [walnut](walnut.md) - O² Workflow Administration and Linting Utility - -###### Auto generated by spf13/cobra on 10-Jul-2020 diff --git a/walnut/doc/walnut_check.md b/walnut/doc/walnut_check.md deleted file mode 100644 index 877c3450e..000000000 --- a/walnut/doc/walnut_check.md +++ /dev/null @@ -1,34 +0,0 @@ -## walnut check - -validate the file against a specified schema - -### Synopsis - -The check command validates the given file against a specified schema. This file can be a task template, -a workflow template or an O² DPL Dump. Each of these have a schema to validate against. - -Valid schema formats: - workflow task dpl_dump - -``` -walnut check [agrs] [flags] -``` - -### Options - -``` - --format string format to validate against - -h, --help help for check -``` - -### Options inherited from parent commands - -``` - --config string optional configuration file for walnut (default $HOME/.config/walnut/settings.yaml) -``` - -### SEE ALSO - -* [walnut](walnut.md) - O² Workflow Administration and Linting Utility - -###### Auto generated by spf13/cobra on 10-Jul-2020 diff --git a/walnut/doc/walnut_convert.md b/walnut/doc/walnut_convert.md deleted file mode 100644 index 872de6d46..000000000 --- a/walnut/doc/walnut_convert.md +++ /dev/null @@ -1,31 +0,0 @@ -## walnut convert - -converts a DPL Dump to the required formats - -### Synopsis - -The convert command takes a DPL input and outputs task and workflow templates. Optional flags can be provided to -specify which modules should be used when generating task templates. Control-OCCPlugin is always used as module. - -``` -walnut convert [args] [flags] -``` - -### Options - -``` - -h, --help help for convert - -m, --modules stringArray modules to include -``` - -### Options inherited from parent commands - -``` - --config string optional configuration file for walnut (default $HOME/.config/walnut/settings.yaml) -``` - -### SEE ALSO - -* [walnut](walnut.md) - O² Workflow Administration and Linting Utility - -###### Auto generated by spf13/cobra on 10-Jul-2020 diff --git a/walnut/includeschemata.go b/walnut/includeschemata.go deleted file mode 100644 index b5843ace0..000000000 --- a/walnut/includeschemata.go +++ /dev/null @@ -1,65 +0,0 @@ -/* - * === This file is part of ALICE O² === - * - * Copyright 2020 CERN and copyright holders of ALICE O². - * Author: Ayaan Zaidi - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * In applying this license CERN does not waive the privileges and - * immunities granted to it by virtue of its status as an - * Intergovernmental Organization or submit itself to any jurisdiction. - */ - -package main - -// Generate latest schemas from .json files -//go:generate go run includeschemata.go - -import ( - "io" - "io/ioutil" - "os" - "path/filepath" - "strings" -) - -// Reads all .json files in the current folder -// and encodes them as strings literals in schemata.go -func main() { - - schemasPath := "./schemata/" - - fs, err := ioutil.ReadDir(schemasPath) - if err != nil { - return - } - - out, err := os.Create(filepath.Join(schemasPath, "schemata.go")) - if err != nil { - return - } - - out.Write([]byte("package schemata \n\nconst (\n")) - for _, f := range fs { - if strings.HasSuffix(f.Name(), ".json") { - out.Write([]byte(strings.TrimSuffix(f.Name(), ".json") + " = `\n")) - f, _ := os.Open(filepath.Join(schemasPath, f.Name())) - io.Copy(out, f) - out.Write([]byte("`\n")) - } - } - out.Write([]byte(")\n")) - -} diff --git a/walnut/schemata/Task.json b/walnut/schemata/Task.json deleted file mode 100644 index 4f073ba27..000000000 --- a/walnut/schemata/Task.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/obviyus/Controlss/walnut/schemata/Task.json", - "title": "AliECS Task Template", - "description": "A task template gets loaded into a task.Class object and its template expressions are resolved. Such a task.Class represents a set of tasks (processes) that can be run throughout the cluster in the same way (i.e. with the same command line invocation).", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "description": "The unique identifier for a task", - "type": "string" - }, - "control": { - "type": "object", - "title": "Type of control", - "properties": { - "mode": { - "type": "string", - "title": "Mode", - "enum": [ - "fairmq", - "basic", - "direct" - ] - } - } - }, - "wants": { - "description": "CPU, memory and ports required for task execution", - "type": "object", - "properties": { - "cpu": { - "description": "Amount of CPU", - "type": "number" - }, - "memory": { - "description": "Amount of Memory", - "type": "number" - }, - "ports": { - "description": "Range of ports to use", - "type": "string" - } - } - }, - "connect": { - "title": "Connect", - "type": "array", - "items": { - "properties": { - "name": { - "description": "Name of the inbound channel", - "type": "string" - }, - "target": { - "description": "The target entry is a string, with some tree walk functions available for traversing the control tree.", - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "pub", - "sub", - "push", - "pull" - ] - }, - "sndBufSize": { - "title": "Send Buffer Size", - "type": [ - "integer", - "string" - ] - }, - "rcvBufSize": { - "title": "Receive Buffer Size", - "type": [ - "integer", - "string" - ] - }, - "rateLogging": { - "title": "Rate Logging", - "type": [ - "integer", - "string" - ] - } - }, - "required": [ - "name", - "type" - ] - } - }, - "bind": { - "type": "array", - "title": "Bind array schema", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "pub", - "sub", - "push", - "pull" - ] - }, - "sndBufSize": { - "title": "Send Buffer Size", - "type": [ - "integer", - "string" - ] - }, - "rcvBufSize": { - "title": "Receive Buffer Size", - "type": [ - "integer", - "string" - ] - }, - "rateLogging": { - "title": "Rate Logging", - "type": [ - "integer", - "string" - ] - }, - "transport": { - "type": "string", - "enum": [ - "default", - "zeromq", - "nanomsg", - "shmem" - ] - } - }, - "required": [ - "name", - "type" - ] - } - }, - "command": { - "type": "object", - "title": "Command for Task execution", - "required": [ - "value" - ], - "properties": { - "env": { - "type": "array", - "title": "Environment", - "items": { - "type": "string" - } - }, - "shell": { - "type": "boolean" - }, - "value": { - "type": "string" - }, - "arguments": { - "type": "array", - "title": "Arguments", - "items": { - "type": "string" - } - }, - "user": { - "type": "string" - } - } - } - } -} diff --git a/walnut/schemata/Workflow.json b/walnut/schemata/Workflow.json deleted file mode 100644 index f58beeb9a..000000000 --- a/walnut/schemata/Workflow.json +++ /dev/null @@ -1,248 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/obviyus/Control/walnut/schemata/Workflow.json", - "title": "AliECS Workflow Template", - "description": "Workflow template schema for AliECS", - "definitions": { - "roles": { - "title": "Roles", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "description": "Parametrized name of an iterator role", - "type": "string" - }, - "enabled": { - "type": "string" - }, - "for": { - "description": "Amount of Memory", - "type": "object", - "properties": { - "begin": { - "title": "Begin port", - "type": "string" - }, - "end": { - "title": "End port", - "type": "string" - }, - "var": { - "title": "variables", - "type": "string" - }, - "range": { - "title": "Port ranges", - "type": "string" - } - }, - "oneOf": [ - { - "required": [ - "begin", - "end", - "var" - ] - }, - { - "required": [ - "range", - "var" - ] - } - ] - }, - "constraints": { - "type": "array", - "properties": { - "attribute": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "vars": { - "type": "object", - "description": "User defined variables" - }, - "connect": { - "title": "Connect", - "type": "array", - "items": { - "properties": { - "name": { - "description": "Name of the inbound channel", - "type": "string" - }, - "target": { - "description": "The target entry is a string, with some tree walk functions available for traversing the control tree.", - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "pub", - "sub", - "push", - "pull" - ] - }, - "sndBufSize": { - "title": "Send Buffer Size", - "type": [ - "integer", - "string" - ] - }, - "rcvBufSize": { - "title": "Receive Buffer Size", - "type": [ - "integer", - "string" - ] - }, - "rateLogging": { - "title": "Rate Logging", - "type": [ - "integer", - "string" - ] - } - }, - "required": [ - "name", - "type", - "target" - ] - } - }, - "bind": { - "type": "array", - "title": "Bind array schema", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "pub", - "sub", - "push", - "pull" - ] - }, - "sndBufSize": { - "title": "Send Buffer Size", - "type": [ - "integer", - "string" - ] - }, - "rcvBufSize": { - "title": "Receive Buffer Size", - "type": [ - "integer", - "string" - ] - }, - "rateLogging": { - "title": "Rate Logging", - "type": [ - "integer", - "string" - ] - }, - "transport": { - "type": "string", - "enum": [ - "default", - "zeromq", - "nanomsg", - "shmem" - ] - } - }, - "required": [ - "name", - "type" - ] - } - }, - "task": { - "title": "Task entry", - "type": "object", - "required": [ - "load" - ], - "properties": { - "load": { - "title": "Task template to load", - "type": "string" - }, - "trigger": { - "type": "string" - }, - "timeout": { - "type": "string" - }, - "critical": { - "type": "boolean" - } - } - }, - "roles": { - "type": "array", - "items": { - "$ref": "#/definitions/roles" - } - } - }, - "oneOf": [ - { - "required": [ - "roles" - ] - }, - { - "required": [ - "task" - ] - } - ] - } - }, - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "description": "The name of the root role, which happens to be an aggregator role.", - "type": "string" - }, - "roles": { - "type": "array", - "items": { - "$ref": "#/definitions/roles" - } - }, - "defaults": { - "type": "object", - "title": "Variable definitions", - "description": "Variable definitions: defaults, overridden by anything" - }, - "vars": { - "type": "object", - "title": "User variable definitions", - "description": "Variable definitions: vars, override defaults, overridden by user vars." - } - } -} diff --git a/walnut/schemata/includeschemata.go b/walnut/schemata/includeschemata.go deleted file mode 100644 index a5098b0f4..000000000 --- a/walnut/schemata/includeschemata.go +++ /dev/null @@ -1,61 +0,0 @@ -/* - * === This file is part of ALICE O² === - * - * Copyright 2020 CERN and copyright holders of ALICE O². - * Author: Ayaan Zaidi - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * In applying this license CERN does not waive the privileges and - * immunities granted to it by virtue of its status as an - * Intergovernmental Organization or submit itself to any jurisdiction. - */ - -package schemata - -import ( - "fmt" - "io" - "io/ioutil" - "os" - "strings" -) - -// Reads all .json files in the current folder -// and encodes them as strings literals in schemata.go -func generateSchema() (err error) { - - fs, err := ioutil.ReadDir(".") - if err != nil { - return fmt.Errorf("error opening Schema: %w", err) - } - - out, err := os.Create("schemata.go") - if err != nil { - return fmt.Errorf("error creating schemata.go: %w", err) - } - - out.Write([]byte("package schemata \n\nconst (\n")) - for _, f := range fs { - if strings.HasSuffix(f.Name(), ".json") { - out.Write([]byte(strings.TrimSuffix(f.Name(), ".json") + " = `\n")) - f, _ := os.Open(f.Name()) - io.Copy(out, f) - out.Write([]byte("`\n")) - } - } - out.Write([]byte(")\n")) - - return nil -} diff --git a/walnut/schemata/includeschemata_test.go b/walnut/schemata/includeschemata_test.go deleted file mode 100644 index 9574388d2..000000000 --- a/walnut/schemata/includeschemata_test.go +++ /dev/null @@ -1,33 +0,0 @@ -/* - * === This file is part of ALICE O² === - * - * Copyright 2020 CERN and copyright holders of ALICE O². - * Author: Ayaan Zaidi - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * In applying this license CERN does not waive the privileges and - * immunities granted to it by virtue of its status as an - * Intergovernmental Organization or submit itself to any jurisdiction. - */ - -package schemata - -import "testing" - -func TestSchemaGeneration(t *testing.T) { - t.Run("Testing Schema Generation", func(t *testing.T) { - generateSchema() - }) -} diff --git a/walnut/schemata/schemata.go b/walnut/schemata/schemata.go deleted file mode 100644 index 1e6c96aea..000000000 --- a/walnut/schemata/schemata.go +++ /dev/null @@ -1,442 +0,0 @@ -package schemata - -const ( - Task = ` -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/obviyus/Controlss/walnut/schemata/Task.json", - "title": "AliECS Task Template", - "description": "A task template gets loaded into a task.Class object and its template expressions are resolved. Such a task.Class represents a set of tasks (processes) that can be run throughout the cluster in the same way (i.e. with the same command line invocation).", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "description": "The unique identifier for a task", - "type": "string" - }, - "control": { - "type": "object", - "title": "Type of control", - "properties": { - "mode": { - "type": "string", - "title": "Mode", - "enum": [ - "fairmq", - "basic", - "direct" - ] - } - } - }, - "wants": { - "description": "CPU, memory and ports required for task execution", - "type": "object", - "properties": { - "cpu": { - "description": "Amount of CPU", - "type": "number" - }, - "memory": { - "description": "Amount of Memory", - "type": "number" - }, - "ports": { - "description": "Range of ports to use", - "type": "string" - } - } - }, - "connect": { - "title": "Connect", - "type": "array", - "items": { - "properties": { - "name": { - "description": "Name of the inbound channel", - "type": "string" - }, - "target": { - "description": "The target entry is a string, with some tree walk functions available for traversing the control tree.", - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "pub", - "sub", - "push", - "pull" - ] - }, - "sndBufSize": { - "title": "Send Buffer Size", - "type": [ - "integer", - "string" - ] - }, - "rcvBufSize": { - "title": "Receive Buffer Size", - "type": [ - "integer", - "string" - ] - }, - "rateLogging": { - "title": "Rate Logging", - "type": [ - "integer", - "string" - ] - } - }, - "required": [ - "name", - "type" - ] - } - }, - "bind": { - "type": "array", - "title": "Bind array schema", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "pub", - "sub", - "push", - "pull" - ] - }, - "sndBufSize": { - "title": "Send Buffer Size", - "type": [ - "integer", - "string" - ] - }, - "rcvBufSize": { - "title": "Receive Buffer Size", - "type": [ - "integer", - "string" - ] - }, - "rateLogging": { - "title": "Rate Logging", - "type": [ - "integer", - "string" - ] - }, - "transport": { - "type": "string", - "enum": [ - "default", - "zeromq", - "nanomsg", - "shmem" - ] - } - }, - "required": [ - "name", - "type" - ] - } - }, - "command": { - "type": "object", - "title": "Command for Task execution", - "required": [ - "value" - ], - "properties": { - "env": { - "type": "array", - "title": "Environment", - "items": { - "type": "string" - } - }, - "shell": { - "type": "boolean" - }, - "value": { - "type": "string" - }, - "arguments": { - "type": "array", - "title": "Arguments", - "items": { - "type": "string" - } - }, - "user": { - "type": "string" - } - } - } - } -} -` - Workflow = ` -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/obviyus/Control/walnut/schemata/Workflow.json", - "title": "AliECS Workflow Template", - "description": "Workflow template schema for AliECS", - "definitions": { - "roles": { - "title": "Roles", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "description": "Parametrized name of an iterator role", - "type": "string" - }, - "enabled": { - "type": "string" - }, - "for": { - "description": "Amount of Memory", - "type": "object", - "properties": { - "begin": { - "title": "Begin port", - "type": "string" - }, - "end": { - "title": "End port", - "type": "string" - }, - "var": { - "title": "variables", - "type": "string" - }, - "range": { - "title": "Port ranges", - "type": "string" - } - }, - "oneOf": [ - { - "required": [ - "begin", - "end", - "var" - ] - }, - { - "required": [ - "range", - "var" - ] - } - ] - }, - "constraints": { - "type": "array", - "properties": { - "attribute": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "vars": { - "type": "object", - "description": "User defined variables" - }, - "connect": { - "title": "Connect", - "type": "array", - "items": { - "properties": { - "name": { - "description": "Name of the inbound channel", - "type": "string" - }, - "target": { - "description": "The target entry is a string, with some tree walk functions available for traversing the control tree.", - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "pub", - "sub", - "push", - "pull" - ] - }, - "sndBufSize": { - "title": "Send Buffer Size", - "type": [ - "integer", - "string" - ] - }, - "rcvBufSize": { - "title": "Receive Buffer Size", - "type": [ - "integer", - "string" - ] - }, - "rateLogging": { - "title": "Rate Logging", - "type": [ - "integer", - "string" - ] - } - }, - "required": [ - "name", - "type", - "target" - ] - } - }, - "bind": { - "type": "array", - "title": "Bind array schema", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "pub", - "sub", - "push", - "pull" - ] - }, - "sndBufSize": { - "title": "Send Buffer Size", - "type": [ - "integer", - "string" - ] - }, - "rcvBufSize": { - "title": "Receive Buffer Size", - "type": [ - "integer", - "string" - ] - }, - "rateLogging": { - "title": "Rate Logging", - "type": [ - "integer", - "string" - ] - }, - "transport": { - "type": "string", - "enum": [ - "default", - "zeromq", - "nanomsg", - "shmem" - ] - } - }, - "required": [ - "name", - "type" - ] - } - }, - "task": { - "title": "Task entry", - "type": "object", - "required": [ - "load" - ], - "properties": { - "load": { - "title": "Task template to load", - "type": "string" - }, - "trigger": { - "type": "string" - }, - "timeout": { - "type": "string" - }, - "critical": { - "type": "boolean" - } - } - }, - "roles": { - "type": "array", - "items": { - "$ref": "#/definitions/roles" - } - } - }, - "oneOf": [ - { - "required": [ - "roles" - ] - }, - { - "required": [ - "task" - ] - } - ] - } - }, - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "description": "The name of the root role, which happens to be an aggregator role.", - "type": "string" - }, - "roles": { - "type": "array", - "items": { - "$ref": "#/definitions/roles" - } - }, - "defaults": { - "type": "object", - "title": "Variable definitions", - "description": "Variable definitions: defaults, overridden by anything" - }, - "vars": { - "type": "object", - "title": "User variable definitions", - "description": "Variable definitions: vars, override defaults, overridden by user vars." - } - } -} -` -) diff --git a/walnut/schemata/task_test.yaml b/walnut/schemata/task_test.yaml deleted file mode 100644 index 7c7b194f4..000000000 --- a/walnut/schemata/task_test.yaml +++ /dev/null @@ -1,24 +0,0 @@ -name: readout # read by workflow -defaults: - readout_cfg_uri: "file:/home/flp/readout.cfg" -control: - mode: direct -wants: - cpu: 0.15 - memory: 128 -bind: - - name: readout - type: push - rateLogging: "1" - addressing: ipc - transport: shmem -properties: {} -command: - env: [] - shell: true - arguments: - - "{{ readout_cfg_uri }}" - value: >- - eval `aliswmod load Readout Control-OCCPlugin` && - o2-readout-exe - diff --git a/walnut/schemata/validate.go b/walnut/schemata/validate.go deleted file mode 100644 index ec9545b60..000000000 --- a/walnut/schemata/validate.go +++ /dev/null @@ -1,72 +0,0 @@ -/* - * === This file is part of ALICE O² === - * - * Copyright 2020 CERN and copyright holders of ALICE O². - * Author: Ayaan Zaidi - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * In applying this license CERN does not waive the privileges and - * immunities granted to it by virtue of its status as an - * Intergovernmental Organization or submit itself to any jurisdiction. - */ - -package schemata - -import ( - "errors" - "fmt" - "os" - - "gopkg.in/yaml.v3" - - "github.com/xeipuuv/gojsonschema" -) - -// Validate takes a YAML file and format then perform schema validation -// on the file against the schema specified (either workflow or task) -func Validate(input []byte, format string) (err error) { - - var inputData interface{} - if err := yaml.Unmarshal(input, &inputData); err != nil { - return fmt.Errorf("Unmarshaling YAML failed: %w", err) - } - - var schema string - switch format { - case "task": - schema = Task - case "workflow": - schema = Workflow - default: - err = errors.New("format not task or workflow") - return fmt.Errorf("Failed to obtain schema: %w", err) - } - - schemaLoader := gojsonschema.NewStringLoader(schema) // load schema - documentLoader := gojsonschema.NewGoLoader(inputData) // load unmarhsaled YAML - - result, err := gojsonschema.Validate(schemaLoader, documentLoader) - if err != nil { - return fmt.Errorf("Error loading data: %w", err) - } - - if result.Valid() { - os.Exit(0) - } else { - return fmt.Errorf("%s", result.Errors()[0]) - } - - return nil -} diff --git a/walnut/schemata/validate_test.go b/walnut/schemata/validate_test.go deleted file mode 100644 index 064d82fb9..000000000 --- a/walnut/schemata/validate_test.go +++ /dev/null @@ -1,48 +0,0 @@ -/* - * === This file is part of ALICE O² === - * - * Copyright 2020 CERN and copyright holders of ALICE O². - * Author: Ayaan Zaidi - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * In applying this license CERN does not waive the privileges and - * immunities granted to it by virtue of its status as an - * Intergovernmental Organization or submit itself to any jurisdiction. - */ - -package schemata - -import ( - "io/ioutil" - "testing" -) - -func TestTaskSchemaValidation(t *testing.T) { - t.Run("Testing Validation with Task", func(t *testing.T) { - testfile, _ := ioutil.ReadFile("task_test.yaml") - err := Validate(testfile, "task") - if err != nil { - t.Errorf("task validation failed: %v", err) - } - }) - - t.Run("Testing Validation with Workflow", func(t *testing.T) { - testfile, _ := ioutil.ReadFile("workflow_test.yaml") - err := Validate(testfile, "workflow") - if err != nil { - t.Errorf("workflow validation failed: %v", err) - } - }) -} diff --git a/walnut/schemata/workflow_test.yaml b/walnut/schemata/workflow_test.yaml deleted file mode 100644 index 9ae8a1d99..000000000 --- a/walnut/schemata/workflow_test.yaml +++ /dev/null @@ -1,24 +0,0 @@ -name: readout-stfb -roles: - - name: host-{{ it }} - for: - range: "{{ hosts }}" - var: it - vars: - readout_cfg_uri: "file:/home/flp/readout_stfb_emu.cfg" - constraints: - - attribute: machine_id - value: "{{ it }}" - roles: - - name: "readout" - task: - load: readout - - name: "stfb" - connect: - - name: readout - type: pull - target: "{{ Parent().Path }}.readout:readout" - rateLogging: "1" - task: - load: stfbuilder-nooutput - \ No newline at end of file