From a8ee90ff0b8f30ba9d1ad87c9f0703864e481a89 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Mon, 24 Mar 2025 11:30:00 +0100 Subject: [PATCH 01/50] Add a new mapper: tedge-gen-mapper This commit is a very first step, scaffolding a generic mapper. The aim is to let users define their own mapping rules to tranform, filter or enrich data received from various sources. The idea is to form pipelines of user-provided transformation functions that: - consume messages from MQTT, - stream these messages along the transformation stages, - publish back to MQTT the resulting messages. Signed-off-by: Didier Wenzek --- Cargo.lock | 10 ++++ Cargo.toml | 2 + crates/core/tedge_mapper/Cargo.toml | 1 + crates/core/tedge_mapper/src/gen/mod.rs | 26 ++++++++ crates/core/tedge_mapper/src/lib.rs | 5 ++ crates/extensions/tedge_gen_mapper/Cargo.toml | 17 ++++++ .../extensions/tedge_gen_mapper/src/actor.rs | 32 ++++++++++ crates/extensions/tedge_gen_mapper/src/lib.rs | 59 +++++++++++++++++++ 8 files changed, 152 insertions(+) create mode 100644 crates/core/tedge_mapper/src/gen/mod.rs create mode 100644 crates/extensions/tedge_gen_mapper/Cargo.toml create mode 100644 crates/extensions/tedge_gen_mapper/src/actor.rs create mode 100644 crates/extensions/tedge_gen_mapper/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 478bafb7113..db615d84ac2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4544,6 +4544,7 @@ dependencies = [ "tedge_config", "tedge_downloader_ext", "tedge_file_system_ext", + "tedge_gen_mapper", "tedge_health_ext", "tedge_http_ext", "tedge_mqtt_bridge", @@ -4793,6 +4794,15 @@ dependencies = [ "try-traits", ] +[[package]] +name = "tedge_gen_mapper" +version = "1.5.1" +dependencies = [ + "async-trait", + "tedge_actors", + "tedge_mqtt_ext", +] + [[package]] name = "tedge_health_ext" version = "1.5.1" diff --git a/Cargo.toml b/Cargo.toml index a53f2902f6a..059536d342a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ tedge_config_macros-impl = { path = "crates/common/tedge_config_macros/impl" } tedge_config_manager = { path = "crates/extensions/tedge_config_manager" } tedge_downloader_ext = { path = "crates/extensions/tedge_downloader_ext" } tedge_file_system_ext = { path = "crates/extensions/tedge_file_system_ext" } +tedge_gen_mapper = { path = "crates/extensions/tedge_gen_mapper" } tedge_health_ext = { path = "crates/extensions/tedge_health_ext" } tedge_http_ext = { path = "crates/extensions/tedge_http_ext" } tedge_log_manager = { path = "crates/extensions/tedge_log_manager" } @@ -163,6 +164,7 @@ regex = "1.4" reqwest = { version = "0.12", default-features = false } ron = "0.8" rpassword = "5.0" +rquickjs = { version = "0.9", default-features = false} rstest = "0.16.0" rumqttc = { git = "https://github.com/jarhodes314/rumqtt", rev = "8c489faf6af910956c97b55587ff3ecb2ac4e96f" } rumqttd = "0.19" diff --git a/crates/core/tedge_mapper/Cargo.toml b/crates/core/tedge_mapper/Cargo.toml index c9f02766e49..908c84dd9f0 100644 --- a/crates/core/tedge_mapper/Cargo.toml +++ b/crates/core/tedge_mapper/Cargo.toml @@ -28,6 +28,7 @@ tedge_api = { workspace = true } tedge_config = { workspace = true } tedge_downloader_ext = { workspace = true } tedge_file_system_ext = { workspace = true } +tedge_gen_mapper = { workspace = true } tedge_health_ext = { workspace = true } tedge_http_ext = { workspace = true } tedge_mqtt_bridge = { workspace = true } diff --git a/crates/core/tedge_mapper/src/gen/mod.rs b/crates/core/tedge_mapper/src/gen/mod.rs new file mode 100644 index 00000000000..50e80ad6b92 --- /dev/null +++ b/crates/core/tedge_mapper/src/gen/mod.rs @@ -0,0 +1,26 @@ +use crate::core::mapper::start_basic_actors; +use crate::TEdgeComponent; +use tedge_config::TEdgeConfig; +use tedge_gen_mapper::GenMapperBuilder; + +pub struct GenMapper; + +#[async_trait::async_trait] +impl TEdgeComponent for GenMapper { + async fn start( + &self, + tedge_config: TEdgeConfig, + _config_dir: &tedge_config::Path, + ) -> Result<(), anyhow::Error> { + let (mut runtime, mut mqtt_actor) = + start_basic_actors("tedge-gen-mapper", &tedge_config).await?; + + let mut wasm_mapper = GenMapperBuilder::new("/etc/tedge/gen-mapper"); + wasm_mapper.connect(&mut mqtt_actor); + + runtime.spawn(wasm_mapper).await?; + runtime.spawn(mqtt_actor).await?; + runtime.run_to_completion().await?; + Ok(()) + } +} diff --git a/crates/core/tedge_mapper/src/lib.rs b/crates/core/tedge_mapper/src/lib.rs index ca3eb34867b..7ef0cc3ad4e 100644 --- a/crates/core/tedge_mapper/src/lib.rs +++ b/crates/core/tedge_mapper/src/lib.rs @@ -8,6 +8,7 @@ use crate::az::mapper::AzureMapper; use crate::c8y::mapper::CumulocityMapper; use crate::collectd::mapper::CollectdMapper; use crate::core::component::TEdgeComponent; +use crate::gen::GenMapper; use anyhow::Context; use clap::Parser; use flockfile::check_another_instance_is_not_running; @@ -25,6 +26,7 @@ mod az; mod c8y; mod collectd; mod core; +mod gen; /// Set the cloud profile either from the CLI argument or env variable, /// then set the environment variable so child processes automatically @@ -60,6 +62,7 @@ fn lookup_component(component_name: MapperName) -> Box { MapperName::C8y { profile } => Box::new(CumulocityMapper { profile: read_and_set_var!(profile, "TEDGE_CLOUD_PROFILE"), }), + MapperName::Gen => Box::new(GenMapper), } } @@ -109,6 +112,7 @@ pub enum MapperName { profile: Option, }, Collectd, + Gen, } impl fmt::Display for MapperName { @@ -133,6 +137,7 @@ impl fmt::Display for MapperName { profile: Some(profile), } => write!(f, "tedge-mapper-c8y@{profile}"), MapperName::Collectd => write!(f, "tedge-mapper-collectd"), + MapperName::Gen => write!(f, "tedge-mapper-gen"), } } } diff --git a/crates/extensions/tedge_gen_mapper/Cargo.toml b/crates/extensions/tedge_gen_mapper/Cargo.toml new file mode 100644 index 00000000000..bcc8196a3e9 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "tedge_gen_mapper" +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true + +[dependencies] +async-trait = { workspace = true } +tedge_actors = { workspace = true } +tedge_mqtt_ext = { workspace = true } + +[lints] +workspace = true diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs new file mode 100644 index 00000000000..e9afb2a5d7b --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -0,0 +1,32 @@ +use async_trait::async_trait; +use tedge_actors::Actor; +use tedge_actors::MessageReceiver; +use tedge_actors::RuntimeError; +use tedge_actors::SimpleMessageBox; +use tedge_mqtt_ext::MqttMessage; + +pub struct GenMapper { + messages: SimpleMessageBox, +} + +#[async_trait] +impl Actor for GenMapper { + fn name(&self) -> &str { + "GenMapper" + } + + async fn run(mut self) -> Result<(), RuntimeError> { + while let Some(message) = self.messages.recv().await { + self.filter(message).await; + } + Ok(()) + } +} + +impl GenMapper { + pub fn new(messages: SimpleMessageBox) -> Self { + Self { messages } + } + + async fn filter(&mut self, _message: MqttMessage) {} +} diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs new file mode 100644 index 00000000000..9b71871925d --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -0,0 +1,59 @@ +mod actor; + +use crate::actor::GenMapper; +use std::convert::Infallible; +use std::path::Path; +use tedge_actors::Builder; +use tedge_actors::DynSender; +use tedge_actors::MessageSink; +use tedge_actors::MessageSource; +use tedge_actors::NoConfig; +use tedge_actors::RuntimeRequest; +use tedge_actors::RuntimeRequestSink; +use tedge_actors::SimpleMessageBoxBuilder; +use tedge_mqtt_ext::MqttMessage; +use tedge_mqtt_ext::TopicFilter; + +pub struct GenMapperBuilder { + message_box: SimpleMessageBoxBuilder, +} + +impl GenMapperBuilder { + pub fn new(config_dir: impl AsRef) -> Self { + let _config_dir = config_dir.as_ref(); + let messages = SimpleMessageBoxBuilder::new("WasmMapper", 16); + GenMapperBuilder { + message_box: messages, + } + } + + pub fn connect( + &mut self, + mqtt: &mut (impl MessageSource + MessageSink), + ) { + mqtt.connect_sink(self.topics(), &self.message_box); + self.message_box.connect_sink(NoConfig, mqtt); + } + + fn topics(&self) -> TopicFilter { + TopicFilter::empty() + } +} + +impl RuntimeRequestSink for GenMapperBuilder { + fn get_signal_sender(&self) -> DynSender { + self.message_box.get_signal_sender() + } +} + +impl Builder for GenMapperBuilder { + type Error = Infallible; + + fn try_build(self) -> Result { + Ok(self.build()) + } + + fn build(self) -> GenMapper { + GenMapper::new(self.message_box.build()) + } +} From 4e1613c01e586ba2a4efcfd17999b195deecdae9 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 25 Mar 2025 22:29:24 +0100 Subject: [PATCH 02/50] Sketch message transformation pipelines Signed-off-by: Didier Wenzek --- Cargo.lock | 7 ++ crates/core/tedge_mapper/src/gen/mod.rs | 7 +- crates/extensions/tedge_gen_mapper/Cargo.toml | 7 ++ .../extensions/tedge_gen_mapper/src/actor.rs | 62 ++++++++- .../extensions/tedge_gen_mapper/src/config.rs | 71 +++++++++++ .../tedge_gen_mapper/src/gen_filter.rs | 38 ++++++ crates/extensions/tedge_gen_mapper/src/lib.rs | 76 +++++++++-- .../tedge_gen_mapper/src/pipeline.rs | 118 ++++++++++++++++++ 8 files changed, 370 insertions(+), 16 deletions(-) create mode 100644 crates/extensions/tedge_gen_mapper/src/config.rs create mode 100644 crates/extensions/tedge_gen_mapper/src/gen_filter.rs create mode 100644 crates/extensions/tedge_gen_mapper/src/pipeline.rs diff --git a/Cargo.lock b/Cargo.lock index db615d84ac2..5f4b4010883 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4799,8 +4799,15 @@ name = "tedge_gen_mapper" version = "1.5.1" dependencies = [ "async-trait", + "camino", + "serde", "tedge_actors", "tedge_mqtt_ext", + "thiserror 1.0.69", + "time", + "tokio", + "toml 0.8.8", + "tracing", ] [[package]] diff --git a/crates/core/tedge_mapper/src/gen/mod.rs b/crates/core/tedge_mapper/src/gen/mod.rs index 50e80ad6b92..ceb8b1acb6c 100644 --- a/crates/core/tedge_mapper/src/gen/mod.rs +++ b/crates/core/tedge_mapper/src/gen/mod.rs @@ -15,10 +15,11 @@ impl TEdgeComponent for GenMapper { let (mut runtime, mut mqtt_actor) = start_basic_actors("tedge-gen-mapper", &tedge_config).await?; - let mut wasm_mapper = GenMapperBuilder::new("/etc/tedge/gen-mapper"); - wasm_mapper.connect(&mut mqtt_actor); + let mut gen_mapper = GenMapperBuilder::default(); + gen_mapper.load("/etc/tedge/gen-mapper").await; + gen_mapper.connect(&mut mqtt_actor); - runtime.spawn(wasm_mapper).await?; + runtime.spawn(gen_mapper).await?; runtime.spawn(mqtt_actor).await?; runtime.run_to_completion().await?; Ok(()) diff --git a/crates/extensions/tedge_gen_mapper/Cargo.toml b/crates/extensions/tedge_gen_mapper/Cargo.toml index bcc8196a3e9..ff51e439ed7 100644 --- a/crates/extensions/tedge_gen_mapper/Cargo.toml +++ b/crates/extensions/tedge_gen_mapper/Cargo.toml @@ -10,8 +10,15 @@ repository.workspace = true [dependencies] async-trait = { workspace = true } +camino = { workspace = true } +serde = { workspace = true, features = ["derive"] } tedge_actors = { workspace = true } tedge_mqtt_ext = { workspace = true } +thiserror = { workspace = true } +time = { workspace = true } +tokio = { workspace = true, features = ["fs", "macros", "time"] } +toml = { workspace = true, features = ["parse"] } +tracing = { workspace = true } [lints] workspace = true diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index e9afb2a5d7b..ac72cd562d7 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -1,12 +1,20 @@ +use crate::pipeline::Pipeline; use async_trait::async_trait; +use std::collections::HashMap; use tedge_actors::Actor; use tedge_actors::MessageReceiver; use tedge_actors::RuntimeError; +use tedge_actors::Sender; use tedge_actors::SimpleMessageBox; use tedge_mqtt_ext::MqttMessage; +use time::OffsetDateTime; +use tokio::time::interval; +use tokio::time::Duration; +use tracing::error; pub struct GenMapper { - messages: SimpleMessageBox, + pub(super) mqtt: SimpleMessageBox, + pub(super) pipelines: HashMap, } #[async_trait] @@ -16,17 +24,59 @@ impl Actor for GenMapper { } async fn run(mut self) -> Result<(), RuntimeError> { - while let Some(message) = self.messages.recv().await { - self.filter(message).await; + let mut interval = interval(Duration::from_secs(5)); + + loop { + tokio::select! { + _ = interval.tick() => { + self.tick().await?; + } + message = self.mqtt.recv() => { + match message { + Some(message) => self.filter(message).await?, + None => break, + } + } + } } Ok(()) } } impl GenMapper { - pub fn new(messages: SimpleMessageBox) -> Self { - Self { messages } + async fn filter(&mut self, message: MqttMessage) -> Result<(), RuntimeError> { + let timestamp = OffsetDateTime::now_utc(); + for (pipeline_id, pipeline) in self.pipelines.iter_mut() { + match pipeline.process(timestamp, &message) { + Ok(messages) => { + for message in messages { + self.mqtt.send(message).await?; + } + } + Err(err) => { + error!(target: "gen-mapper", "{pipeline_id}: {err}"); + } + } + } + + Ok(()) } - async fn filter(&mut self, _message: MqttMessage) {} + async fn tick(&mut self) -> Result<(), RuntimeError> { + let timestamp = OffsetDateTime::now_utc(); + for (pipeline_id, pipeline) in self.pipelines.iter_mut() { + match pipeline.tick(timestamp) { + Ok(messages) => { + for message in messages { + self.mqtt.send(message).await?; + } + } + Err(err) => { + error!(target: "gen-mapper", "{pipeline_id}: {err}"); + } + } + } + + Ok(()) + } } diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs new file mode 100644 index 00000000000..a445729f86f --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -0,0 +1,71 @@ +use crate::pipeline::Pipeline; +use crate::pipeline::Stage; +use crate::gen_filter::GenFilter; +use serde::Deserialize; +use std::path::PathBuf; +use tedge_mqtt_ext::TopicFilter; + +#[derive(Deserialize)] +pub struct PipelineConfig { + input_topics: Vec, + stages: Vec, +} + +#[derive(Deserialize)] +pub struct StageConfig { + filter: FilterSpec, + + #[serde(default)] + config_topics: Vec, +} + +#[derive(Deserialize)] +#[serde(untagged)] +pub enum FilterSpec { + JavaScript(PathBuf), +} + +#[derive(thiserror::Error, Debug)] +pub enum ConfigError { + #[error("Not a valid MQTT topic filter: {0}")] + IncorrectTopicFilter(String), +} + +impl TryFrom for Pipeline { + type Error = ConfigError; + + fn try_from(config: PipelineConfig) -> Result { + let input = topic_filters(&config.input_topics)?; + let stages = config + .stages + .into_iter() + .map(Stage::try_from) + .collect::, _>>()?; + Ok(Pipeline { input_topics: input, stages }) + } +} + +impl TryFrom for Stage { + type Error = ConfigError; + + fn try_from(config: StageConfig) -> Result { + let filter = match config.filter { + FilterSpec::JavaScript(path) => GenFilter::new(path), + }; + let config = topic_filters(&config.config_topics)?; + Ok(Stage { + filter: Box::new(filter), + config_topics: config, + }) + } +} + +fn topic_filters(patterns: &Vec) -> Result { + let mut topics = TopicFilter::empty(); + for pattern in patterns { + topics + .add(pattern.as_str()) + .map_err(|_| ConfigError::IncorrectTopicFilter(pattern.clone()))?; + } + Ok(topics) +} diff --git a/crates/extensions/tedge_gen_mapper/src/gen_filter.rs b/crates/extensions/tedge_gen_mapper/src/gen_filter.rs new file mode 100644 index 00000000000..038543c8340 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/src/gen_filter.rs @@ -0,0 +1,38 @@ +use crate::pipeline::Filter; +use crate::pipeline::FilterError; +use std::path::PathBuf; +use tedge_mqtt_ext::MqttMessage; +use time::OffsetDateTime; +use tracing::debug; + +/// User-defined filter +pub struct GenFilter {} + +impl GenFilter { + pub fn new(path: impl Into) -> Self { + let path = path.into(); + debug!(target: "MAPPING", "new({path:?})"); + GenFilter {} + } +} + +impl Filter for GenFilter { + fn process( + &mut self, + timestamp: OffsetDateTime, + message: &MqttMessage, + ) -> Result, FilterError> { + debug!(target: "MAPPING", "process({timestamp}, {message:?})"); + Ok(vec![message.clone()]) + } + + fn update_config(&mut self, config: &MqttMessage) -> Result<(), FilterError> { + debug!(target: "MAPPING", "update_config({config:?})"); + Ok(()) + } + + fn tick(&mut self, timestamp: OffsetDateTime) -> Result, FilterError> { + debug!(target: "MAPPING", "tick({timestamp})"); + Ok(vec![]) + } +} diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index 9b71871925d..b5aee325715 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -1,6 +1,12 @@ mod actor; +mod config; +mod pipeline; +mod gen_filter; use crate::actor::GenMapper; +use crate::pipeline::Pipeline; +use camino::Utf8Path; +use std::collections::HashMap; use std::convert::Infallible; use std::path::Path; use tedge_actors::Builder; @@ -13,18 +19,58 @@ use tedge_actors::RuntimeRequestSink; use tedge_actors::SimpleMessageBoxBuilder; use tedge_mqtt_ext::MqttMessage; use tedge_mqtt_ext::TopicFilter; +use tokio::fs::read_dir; +use tokio::fs::read_to_string; +use tracing::error; +use tracing::info; pub struct GenMapperBuilder { message_box: SimpleMessageBoxBuilder, + pipelines: HashMap, } -impl GenMapperBuilder { - pub fn new(config_dir: impl AsRef) -> Self { - let _config_dir = config_dir.as_ref(); - let messages = SimpleMessageBoxBuilder::new("WasmMapper", 16); +impl Default for GenMapperBuilder { + fn default() -> Self { GenMapperBuilder { - message_box: messages, + message_box: SimpleMessageBoxBuilder::new("GenMapper", 16), + pipelines: HashMap::default(), + } + } +} + +impl GenMapperBuilder { + pub async fn load(&mut self, config_dir: impl AsRef) { + let config_dir = config_dir.as_ref(); + let Ok(mut entries) = read_dir(config_dir).await.map_err(|err| + error!(target: "MAPPING", "Failed to read filters from {}: {err}", config_dir.display()) + ) else { + return; + }; + + while let Ok(Some(entry)) = entries.next_entry().await { + let Some(path) = Utf8Path::from_path(&entry.path()).map(|p| p.to_path_buf()) else { + error!(target: "MAPPING", "Skipping non UTF8 path: {}", entry.path().display()); + continue; + }; + if let Ok(file_type) = entry.file_type().await { + if file_type.is_file() && path.extension() == Some("toml") { + info!(target: "MAPPING", "Loading pipeline: {path}"); + if let Err(err) = self.load_pipeline(path).await { + error!(target: "MAPPING", "Failed to load pipeline: {err}"); + } + } + } + } + } + + async fn load_pipeline(&mut self, file: impl AsRef) -> Result<(), LoadError> { + if let Some(name) = file.as_ref().file_name() { + let specs = read_to_string(file.as_ref()).await?; + let pipeline: Pipeline = toml::from_str(&specs)?; + self.pipelines.insert(name.to_string(), pipeline); } + + Ok(()) } pub fn connect( @@ -36,7 +82,11 @@ impl GenMapperBuilder { } fn topics(&self) -> TopicFilter { - TopicFilter::empty() + let mut topics = TopicFilter::empty(); + for pipeline in self.pipelines.values() { + topics.add_all(pipeline.topics()) + } + topics } } @@ -54,6 +104,18 @@ impl Builder for GenMapperBuilder { } fn build(self) -> GenMapper { - GenMapper::new(self.message_box.build()) + GenMapper { + mqtt: self.message_box.build(), + pipelines: self.pipelines, + } } } + +#[derive(thiserror::Error, Debug)] +pub enum LoadError { + #[error(transparent)] + IoError(#[from] std::io::Error), + + #[error(transparent)] + TomlError(#[from] toml::de::Error), +} diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs new file mode 100644 index 00000000000..03adc975276 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -0,0 +1,118 @@ +use serde::Deserialize; +use tedge_mqtt_ext::MqttMessage; +use tedge_mqtt_ext::TopicFilter; +use time::OffsetDateTime; + +/// A chain of transformation of MQTT messages +#[derive(Deserialize)] +#[serde(try_from = "crate::config::PipelineConfig")] +pub struct Pipeline { + /// The source topics + pub input_topics: TopicFilter, + + /// Transformation stages to apply in order to the messages + pub stages: Vec, +} + +/// A message transformation stage +pub struct Stage { + pub filter: Box, + pub config_topics: TopicFilter, +} + +/// A filter process a stream of messages, producing a stream of transformed messages +/// +/// Filters are chained along pipelines, consuming MQTT messages as input +/// and producing MQTT messages as output. +/// +/// The behavior of a filter can be time related and +/// +/// Filters are dynamically configured. New partial configuration updates are sent overtime, +/// giving the opportunity for a filter to adapt its behavior. +pub trait Filter: 'static + Send + Sync { + /// Process a single message; producing zero, one or more transformed messages + fn process( + &mut self, + timestamp: OffsetDateTime, + message: &MqttMessage, + ) -> Result, FilterError>; + + /// Update the filter configuration + fn update_config(&mut self, config: &MqttMessage) -> Result<(), FilterError>; + + /// Close the current time-window; producing zero, one or more accumulated messages + fn tick(&mut self, timestamp: OffsetDateTime) -> Result, FilterError>; +} + +#[derive(thiserror::Error, Debug)] +pub enum FilterError { + #[error("Input message cannot be processed: {0}")] + UnsupportedMessage(String), + + #[error("No messages can be processed due to an incorrect setting: {0}")] + IncorrectSetting(String), +} + +impl Pipeline { + pub fn topics(&self) -> TopicFilter { + let mut topics = self.input_topics.clone(); + for stage in self.stages.iter() { + topics.add_all(stage.config_topics.clone()) + } + topics + } + + pub fn update_config(&mut self, message: &MqttMessage) -> Result<(), FilterError> { + for stage in self.stages.iter_mut() { + if stage.config_topics.accept(message) { + stage.filter.update_config(message)? + } + } + Ok(()) + } + + pub fn process( + &mut self, + timestamp: OffsetDateTime, + message: &MqttMessage, + ) -> Result, FilterError> { + self.update_config(message)?; + if !self.input_topics.accept(message) { + return Ok(vec![]); + } + + let mut messages = vec![message.clone()]; + for stage in self.stages.iter_mut() { + let mut transformed_messages = vec![]; + for filter_output in messages + .iter() + .map(|message| stage.filter.process(timestamp, message)) + { + transformed_messages.extend(filter_output?); + } + messages = transformed_messages; + } + Ok(messages) + } + + pub fn tick(&mut self, timestamp: OffsetDateTime) -> Result, FilterError> { + let mut messages = vec![]; + for stage in self.stages.iter_mut() { + // Process first the messages triggered upstream by the tick + let mut transformed_messages = vec![]; + for filter_output in messages + .iter() + .map(|message| stage.filter.process(timestamp, message)) + { + transformed_messages.extend(filter_output?); + } + + // Only then process the tick + transformed_messages.extend(stage.filter.tick(timestamp)?); + + // Iterate with all the messages collected at this stage + messages = transformed_messages; + } + Ok(messages) + } +} From e58c8cc1aee6b501f3c33733483baab82ccb9823 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Wed, 14 May 2025 19:23:51 +0200 Subject: [PATCH 03/50] Run JS transformation stages Signed-off-by: Didier Wenzek --- Cargo.lock | 1642 ++++++++++------- Cargo.toml | 2 +- crates/core/tedge_mapper/src/gen/mod.rs | 4 +- crates/extensions/tedge_gen_mapper/Cargo.toml | 4 +- .../pipelines/add_timestamp.js | 9 + .../tedge_gen_mapper/pipelines/collectd.toml | 6 + .../pipelines/measurements.toml | 6 + .../tedge_gen_mapper/pipelines/te_to_c8y.js | 6 + .../extensions/tedge_gen_mapper/src/actor.rs | 9 +- .../extensions/tedge_gen_mapper/src/config.rs | 50 +- .../tedge_gen_mapper/src/gen_filter.rs | 38 - .../tedge_gen_mapper/src/js_filter.rs | 250 +++ crates/extensions/tedge_gen_mapper/src/lib.rs | 84 +- .../tedge_gen_mapper/src/pipeline.rs | 64 +- 14 files changed, 1337 insertions(+), 837 deletions(-) create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/add_timestamp.js create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/collectd.toml create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/measurements.toml create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js delete mode 100644 crates/extensions/tedge_gen_mapper/src/gen_filter.rs create mode 100644 crates/extensions/tedge_gen_mapper/src/js_filter.rs diff --git a/Cargo.lock b/Cargo.lock index 5f4b4010883..ca9cdc35845 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,19 +4,13 @@ version = 3 [[package]] name = "addr2line" -version = "0.21.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - [[package]] name = "adler2" version = "2.0.0" @@ -29,16 +23,16 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ - "getrandom 0.2.12", + "getrandom 0.2.16", "once_cell", "version_check", ] [[package]] name = "ahash" -version = "0.8.7" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", "once_cell", @@ -48,24 +42,25 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] [[package]] name = "anstream" -version = "0.6.11" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", + "is_terminal_polyfill", "utf8parse", ] @@ -77,46 +72,47 @@ checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" -version = "0.2.3" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.2" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.2" +version = "3.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" dependencies = [ "anstyle", - "windows-sys 0.52.0", + "once_cell", + "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.79" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" dependencies = [ "backtrace", ] [[package]] name = "arc-swap" -version = "1.6.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" [[package]] name = "arrayvec" @@ -158,9 +154,9 @@ dependencies = [ [[package]] name = "asn1-rs" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "607495ec7113b178fbba7a6166a27f99e774359ef4823adbefd756b5b81d7970" +checksum = "56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60" dependencies = [ "asn1-rs-derive 0.6.0", "asn1-rs-impl 0.2.0", @@ -169,7 +165,7 @@ dependencies = [ "num-bigint", "num-traits", "rusticata-macros", - "thiserror 2.0.11", + "thiserror 2.0.12", ] [[package]] @@ -192,8 +188,8 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", - "synstructure 0.13.1", + "syn 2.0.101", + "synstructure 0.13.2", ] [[package]] @@ -204,8 +200,8 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", - "synstructure 0.13.1", + "syn 2.0.101", + "synstructure 0.13.2", ] [[package]] @@ -227,7 +223,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -242,14 +238,15 @@ dependencies = [ [[package]] name = "assert_cmd" -version = "2.0.13" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00ad3f3a942eee60335ab4342358c161ee296829e0d16ff42fc1d6cb07815467" +checksum = "2bd389a4b2970a01282ee455294913c0a43724daedcd1a24c3eb0ec1c1320b66" dependencies = [ "anstyle", "bstr", "doc-comment", - "predicates 3.1.0", + "libc", + "predicates 3.1.3", "predicates-core", "predicates-tree", "wait-timeout", @@ -263,9 +260,9 @@ checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" [[package]] name = "async-compat" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f68a707c1feb095d8c07f8a65b9f506b117d30af431cab89374357de7c11461b" +checksum = "7bab94bde396a3f7b4962e396fdad640e241ed797d4d8d77fc8c237d14c58fc0" dependencies = [ "futures-core", "futures-io", @@ -286,6 +283,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-lock" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-tempfile" version = "0.7.0" @@ -297,13 +305,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.77" +version = "0.1.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -322,18 +330,21 @@ dependencies = [ [[package]] name = "async-tungstenite" -version = "0.28.0" +version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90e661b6cb0a6eb34d02c520b052daa3aa9ac0cc02495c9d066bbce13ead132b" +checksum = "1c348fb0b6d132c596eca3dcd941df48fb597aafcb07a738ec41c004b087dc99" dependencies = [ + "atomic-waker", + "futures-core", "futures-io", + "futures-task", "futures-util", "log", "pin-project-lite", "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.1", + "tokio-rustls 0.26.2", "tungstenite 0.24.0", ] @@ -375,9 +386,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.1.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws_mapper_ext" @@ -409,9 +420,9 @@ dependencies = [ "bitflags 1.3.2", "bytes", "futures-util", - "http 0.2.11", + "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.28", + "hyper 0.14.32", "itoa", "matchit 0.7.3", "memchr", @@ -432,17 +443,17 @@ dependencies = [ [[package]] name = "axum" -version = "0.8.1" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d6fd624c75e18b3b4c6b9caf42b1afe24437daaee904069137d8bab077be8b8" +checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5" dependencies = [ - "axum-core 0.5.0", + "axum-core 0.5.2", "axum-macros", "base64 0.22.1", "bytes", "form_urlencoded", "futures-util", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "http-body-util", "hyper 1.6.0", @@ -477,7 +488,7 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 0.2.11", + "http 0.2.12", "http-body 0.4.6", "mime", "rustversion", @@ -487,13 +498,13 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1362f362fd16024ae199c1970ce98f9661bf5ef94b9808fee734bc3698b733" +checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6" dependencies = [ "bytes", - "futures-util", - "http 1.2.0", + "futures-core", + "http 1.3.1", "http-body 1.0.1", "http-body-util", "mime", @@ -507,20 +518,21 @@ dependencies = [ [[package]] name = "axum-extra" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fc6f625a1f7705c6cf62d0d070794e94668988b1c38111baeec177c715f7b" +checksum = "45bf463831f5131b7d3c756525b305d40f1185b688565648a92e1392ca35713d" dependencies = [ - "axum 0.8.1", - "axum-core 0.5.0", + "axum 0.8.4", + "axum-core 0.5.2", "bytes", "futures-util", "headers", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", + "rustversion", "serde", "tower 0.5.2", "tower-layer", @@ -535,30 +547,28 @@ checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] name = "axum-server" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56bac90848f6a9393ac03c63c640925c4b7c8ca21654de40d53f55964667c7d8" +checksum = "495c05f60d6df0093e8fb6e74aa5846a0ad06abaf96d76166283720bf740f8ab" dependencies = [ "arc-swap", "bytes", - "futures-util", - "http 1.2.0", + "fs-err", + "http 1.3.1", "http-body 1.0.1", - "http-body-util", "hyper 1.6.0", "hyper-util", "pin-project-lite", - "rustls 0.23.22", + "rustls 0.23.27", "rustls-pemfile 2.2.0", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.1", - "tower 0.4.13", + "tokio-rustls 0.26.2", "tower-service", ] @@ -568,7 +578,7 @@ version = "1.5.1" dependencies = [ "anyhow", "assert_matches", - "axum 0.8.1", + "axum 0.8.4", "axum-server", "camino", "futures", @@ -577,12 +587,12 @@ dependencies = [ "pin-project", "rcgen", "reqwest", - "rustls 0.23.22", + "rustls 0.23.27", "rustls-pemfile 2.2.0", "tedge_config", "tempfile", "tokio", - "tokio-rustls 0.26.1", + "tokio-rustls 0.26.2", "tower 0.4.13", "tracing", "x509-parser 0.16.0", @@ -615,26 +625,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" dependencies = [ "futures-core", - "getrandom 0.2.12", + "getrandom 0.2.16", "instant", "pin-project-lite", - "rand", + "rand 0.8.5", "tokio", ] [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", - "miniz_oxide 0.7.1", + "miniz_oxide", "object", "rustc-demangle", + "windows-targets 0.52.6", ] [[package]] @@ -676,18 +686,18 @@ dependencies = [ [[package]] name = "bit-set" -version = "0.5.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ "bit-vec", ] [[package]] name = "bit-vec" -version = "0.6.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" @@ -697,9 +707,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" dependencies = [ "serde", ] @@ -737,26 +747,26 @@ dependencies = [ [[package]] name = "bstr" -version = "1.9.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c48f0051a4b4c5e0b6d365cd04af53aeaa209e3cc15ec2cdb69e73cc87fbd0dc" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" dependencies = [ "memchr", - "regex-automata 0.4.5", + "regex-automata 0.4.9", "serde", ] [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "bytemuck" -version = "1.22.0" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540" +checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c" [[package]] name = "byteorder" @@ -766,9 +776,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.9.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" dependencies = [ "serde", ] @@ -796,8 +806,8 @@ version = "1.5.1" dependencies = [ "async-compat", "async-http-proxy", - "async-tungstenite 0.28.0", - "axum 0.8.1", + "async-tungstenite 0.28.2", + "axum 0.8.4", "base64 0.22.1", "bytes", "c8y_api", @@ -806,11 +816,11 @@ dependencies = [ "csv", "futures", "futures-util", - "http 1.2.0", + "http 1.3.1", "miette", - "rand", + "rand 0.8.5", "rstest", - "rustls 0.23.22", + "rustls 0.23.27", "serde", "sha1", "tedge_config", @@ -818,7 +828,7 @@ dependencies = [ "tempfile", "thiserror 1.0.69", "tokio", - "tokio-rustls 0.26.1", + "tokio-rustls 0.26.2", "url", "ws_stream_tungstenite 0.14.0", ] @@ -845,7 +855,7 @@ dependencies = [ "thiserror 1.0.69", "time", "tokio", - "toml 0.8.8", + "toml 0.8.22", "tracing", "url", ] @@ -856,7 +866,7 @@ version = "1.5.1" dependencies = [ "anyhow", "async-trait", - "axum 0.8.1", + "axum 0.8.4", "axum-extra", "axum-server", "axum_tls", @@ -872,7 +882,7 @@ dependencies = [ "pin-project", "rcgen", "reqwest", - "rustls 0.23.22", + "rustls 0.23.27", "tedge_actors", "tedge_config", "tedge_config_macros", @@ -912,7 +922,7 @@ version = "1.5.1" dependencies = [ "anyhow", "c8y_api", - "http 1.2.0", + "http 1.3.1", "mockito", "reqwest", "serde", @@ -961,16 +971,16 @@ dependencies = [ "thiserror 1.0.69", "time", "tokio", - "toml 0.8.8", + "toml 0.8.22", "tracing", "url", ] [[package]] name = "camino" -version = "1.1.6" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" dependencies = [ "serde", ] @@ -983,9 +993,9 @@ checksum = "6f125eb85b84a24c36b02ed1d22c9dd8632f53b3cde6e4d23512f94021030003" [[package]] name = "cc" -version = "1.2.10" +version = "1.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" +checksum = "32db95edf998450acc7881c932f94cd9b05c87b4b2599e8bab064753da4acfd1" dependencies = [ "shlex", ] @@ -995,13 +1005,13 @@ name = "certificate" version = "1.5.1" dependencies = [ "anyhow", - "asn1-rs 0.7.0", + "asn1-rs 0.7.1", "assert_matches", "base64 0.22.1", "camino", "rcgen", "reqwest", - "rustls 0.23.22", + "rustls 0.23.27", "rustls-native-certs", "rustls-pemfile 2.2.0", "sha1", @@ -1028,18 +1038,18 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.40" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" dependencies = [ "num-traits", ] [[package]] name = "clap" -version = "4.5.26" +version = "4.5.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8eb5e908ef3a6efbe1ed62520fb7287959888c88485abe072543190ecc66783" +checksum = "ed93b9805f8ba930df42c2590f05453d5ec36cbb85d018868a5b24d31f6ac000" dependencies = [ "clap_builder", "clap_derive", @@ -1047,9 +1057,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.26" +version = "4.5.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" +checksum = "379026ff283facf611b0ea629334361c4211d1b12ee01024eec1591133b04120" dependencies = [ "anstream", "anstyle", @@ -1059,9 +1069,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.42" +version = "4.5.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33a7e468e750fa4b6be660e8b5651ad47372e8fb114030b594c2d75d48c5ffd0" +checksum = "c91d3baa3bcd889d60e6ef28874126a0b384fd225ab83aa6d8a801c519194ce1" dependencies = [ "clap", "clap_lex", @@ -1071,14 +1081,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.24" +version = "4.5.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" +checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -1121,18 +1131,26 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "colored" -version = "2.1.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e" dependencies = [ - "lazy_static", - "windows-sys 0.48.0", + "windows-sys 0.59.0", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", ] [[package]] @@ -1178,9 +1196,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] @@ -1230,9 +1248,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crypto-common" @@ -1269,9 +1287,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" dependencies = [ "csv-core", "itoa", @@ -1281,9 +1299,9 @@ dependencies = [ [[package]] name = "csv-core" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" dependencies = [ "memchr", ] @@ -1300,12 +1318,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core 0.20.10", - "darling_macro 0.20.10", + "darling_core 0.20.11", + "darling_macro 0.20.11", ] [[package]] @@ -1324,16 +1342,16 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -1349,20 +1367,20 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core 0.20.10", + "darling_core 0.20.11", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] name = "data-encoding" -version = "2.5.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" +checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" [[package]] name = "der-parser" @@ -1394,9 +1412,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.11" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", "serde", @@ -1426,13 +1444,13 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -1481,7 +1499,7 @@ name = "download" version = "1.5.1" dependencies = [ "anyhow", - "axum 0.8.1", + "axum 0.8.4", "axum_tls", "backoff", "certificate", @@ -1491,7 +1509,7 @@ dependencies = [ "nix", "rcgen", "reqwest", - "rustls 0.23.22", + "rustls 0.23.27", "serde", "tedge_utils", "tempfile", @@ -1507,14 +1525,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f5ce6d7f6b0c1a6330fb8450f49a8423b78e30d04132146938c35baab3877eb" dependencies = [ "fnv", - "rand", + "rand 0.8.5", ] [[package]] name = "either" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7914353092ddf589ad78f25c5c1c21b7f80b0ff8621e7c814c3485b5306da9d" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "embedded-io" @@ -1543,9 +1561,9 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" @@ -1557,47 +1575,68 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "event-listener" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fastrand" -version = "2.0.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "figment" -version = "0.10.14" +version = "0.10.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b6e5bc7bd59d60d0d45a6ccab6cf0f4ce28698fb4e81e750ddf229c9b824026" +checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" dependencies = [ "atomic", "parking_lot", "pear", "serde", "tempfile", - "toml 0.8.8", + "toml 0.8.22", "uncased", "version_check", ] [[package]] name = "file-id" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6584280525fb2059cba3db2c04abf947a1a29a45ddae89f3870f8281704fafc9" +checksum = "6bc904b9bbefcadbd8e3a9fb0d464a9b979de6324c03b3c663e8994f46a5be36" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "filetime" -version = "0.2.23" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" dependencies = [ "cfg-if", "libc", - "redox_syscall", - "windows-sys 0.52.0", + "libredox", + "windows-sys 0.59.0", ] [[package]] @@ -1613,7 +1652,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", - "miniz_oxide 0.8.8", + "miniz_oxide", ] [[package]] @@ -1638,9 +1677,9 @@ dependencies = [ [[package]] name = "flume" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", @@ -1664,9 +1703,9 @@ dependencies = [ [[package]] name = "fragile" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" +checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619" [[package]] name = "freedesktop_entry_parser" @@ -1678,6 +1717,16 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "fs-err" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f89bda4c2a21204059a977ed3bfe746677dfd137b83c339e702b0ac91d482aa" +dependencies = [ + "autocfg", + "tokio", +] + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -1695,9 +1744,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -1726,9 +1775,9 @@ checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -1749,7 +1798,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -1766,9 +1815,9 @@ checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-timer" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" @@ -1800,9 +1849,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.12" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "js-sys", @@ -1813,41 +1862,43 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", + "js-sys", "libc", - "wasi 0.13.3+wasi-0.2.2", - "windows-targets 0.52.6", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", ] [[package]] name = "gimli" -version = "0.28.1" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "h2" -version = "0.4.6" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http 1.2.0", - "indexmap 2.2.1", + "http 1.3.1", + "indexmap 2.9.0", "slab", "tokio", "tokio-util", @@ -1878,7 +1929,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.12", ] [[package]] @@ -1887,9 +1938,15 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.12", ] +[[package]] +name = "hashbrown" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3" + [[package]] name = "headers" version = "0.4.0" @@ -1899,7 +1956,7 @@ dependencies = [ "base64 0.21.7", "bytes", "headers-core", - "http 1.2.0", + "http 1.3.1", "httpdate", "mime", "sha1", @@ -1911,7 +1968,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http 1.2.0", + "http 1.3.1", ] [[package]] @@ -1942,9 +1999,15 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.3.4" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hermit-abi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08" [[package]] name = "hex" @@ -1954,18 +2017,18 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "home" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "http" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", @@ -1974,9 +2037,9 @@ dependencies = [ [[package]] name = "http" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" dependencies = [ "bytes", "fnv", @@ -1990,7 +2053,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http 0.2.11", + "http 0.2.12", "pin-project-lite", ] @@ -2001,27 +2064,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.2.0", + "http 1.3.1", ] [[package]] name = "http-body-util" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", - "futures-util", - "http 1.2.0", + "futures-core", + "http 1.3.1", "http-body 1.0.1", "pin-project-lite", ] [[package]] name = "httparse" -version = "1.9.4" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] name = "httpdate" @@ -2031,21 +2094,21 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" [[package]] name = "hyper" -version = "0.14.28" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes", "futures-channel", "futures-core", "futures-util", - "http 0.2.11", + "http 0.2.12", "http-body 0.4.6", "httparse", "httpdate", @@ -2068,7 +2131,7 @@ dependencies = [ "futures-channel", "futures-util", "h2", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "httparse", "httpdate", @@ -2086,29 +2149,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", - "http 1.2.0", + "http 1.3.1", "hyper 1.6.0", "hyper-util", - "rustls 0.23.22", + "rustls 0.23.27", "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.1", + "tokio-rustls 0.26.2", "tower-service", ] [[package]] name = "hyper-util" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2" dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "hyper 1.6.0", + "libc", "pin-project-lite", "socket2", "tokio", @@ -2118,21 +2182,22 @@ dependencies = [ [[package]] name = "icu_collections" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" dependencies = [ "displaydoc", + "potential_utf", "yoke", "zerofrom", "zerovec", ] [[package]] -name = "icu_locid" -version = "1.5.0" +name = "icu_locale_core" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" dependencies = [ "displaydoc", "litemap", @@ -2141,31 +2206,11 @@ dependencies = [ "zerovec", ] -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" - [[package]] name = "icu_normalizer" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" dependencies = [ "displaydoc", "icu_collections", @@ -2173,67 +2218,54 @@ dependencies = [ "icu_properties", "icu_provider", "smallvec", - "utf16_iter", - "utf8_iter", - "write16", "zerovec", ] [[package]] name = "icu_normalizer_data" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" [[package]] name = "icu_properties" -version = "1.5.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +checksum = "2549ca8c7241c82f59c80ba2a6f415d931c5b58d24fb8412caa1a1f02c49139a" dependencies = [ "displaydoc", "icu_collections", - "icu_locid_transform", + "icu_locale_core", "icu_properties_data", "icu_provider", - "tinystr", + "potential_utf", + "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" +checksum = "8197e866e47b68f8f7d95249e172903bec06004b18b2937f1095d40a0c57de04" [[package]] name = "icu_provider" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" dependencies = [ "displaydoc", - "icu_locid", - "icu_provider_macros", + "icu_locale_core", "stable_deref_trait", "tinystr", "writeable", "yoke", "zerofrom", + "zerotrie", "zerovec", ] -[[package]] -name = "icu_provider_macros" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.96", -] - [[package]] name = "ident_case" version = "1.0.1" @@ -2253,9 +2285,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" dependencies = [ "icu_normalizer", "icu_properties", @@ -2273,12 +2305,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.1" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433de089bd45971eecf4668ee0ee8f4cec17db4f8bd8f7bc3197a6ce37aa7d9b" +checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.3", ] [[package]] @@ -2309,35 +2341,35 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", ] [[package]] name = "ipnet" -version = "2.9.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "is-terminal" -version = "0.4.10" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" dependencies = [ - "hermit-abi", - "rustix 0.38.34", - "windows-sys 0.52.0", + "hermit-abi 0.5.1", + "libc", + "windows-sys 0.59.0", ] [[package]] name = "is_ci" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616cde7c720bb2bb5824a224687d8f77bfd38922027f01d825cd7453be5099fb" +checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" [[package]] name = "is_executable" @@ -2348,6 +2380,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -2368,16 +2406,17 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.10" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "js-sys" -version = "0.3.67" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -2423,9 +2462,9 @@ dependencies = [ [[package]] name = "kqueue" -version = "1.0.8" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c" +checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" dependencies = [ "kqueue-sys", "libc", @@ -2443,15 +2482,15 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.169" +version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] name = "libloading" @@ -2463,10 +2502,15 @@ dependencies = [ ] [[package]] -name = "libm" -version = "0.2.8" +name = "libredox" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.9.0", + "libc", + "redox_syscall", +] [[package]] name = "linked-hash-map" @@ -2476,9 +2520,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.13" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "linux-raw-sys" @@ -2488,15 +2532,15 @@ checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" [[package]] name = "litemap" -version = "0.7.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -2504,9 +2548,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "mach2" @@ -2546,9 +2596,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "memchr" -version = "2.7.1" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memoffset" @@ -2565,7 +2615,7 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.12", "metrics-macros", "portable-atomic", ] @@ -2577,7 +2627,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d4fa7ce7c4862db464a37b0b31d89bca874562f034bd7993895572783d02950" dependencies = [ "base64 0.21.7", - "hyper 0.14.28", + "hyper 0.14.32", "indexmap 1.9.3", "ipnet", "metrics", @@ -2596,7 +2646,7 @@ checksum = "38b4faf00617defe497754acde3024865bc143d44a86799b24e191ecff91354f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -2643,7 +2693,7 @@ checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -2654,9 +2704,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mime_guess" -version = "2.0.4" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" dependencies = [ "mime", "unicase", @@ -2668,15 +2718,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" -dependencies = [ - "adler", -] - [[package]] name = "miniz_oxide" version = "0.8.8" @@ -2738,21 +2779,21 @@ dependencies = [ [[package]] name = "mockito" -version = "1.5.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b34bd91b9e5c5b06338d392463e1318d683cf82ec3d3af4014609be6e2108d" +checksum = "7760e0e418d9b7e5777c0374009ca4c93861b9066f18cb334a20ce50ab63aa48" dependencies = [ "assert-json-diff", "bytes", "colored", "futures-util", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "http-body-util", "hyper 1.6.0", "hyper-util", "log", - "rand", + "rand 0.9.1", "regex", "serde_json", "serde_urlencoded", @@ -2806,7 +2847,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ffa00dec017b5b1a8b7cf5e2c008bfda1aa7e0697ac1508b491fdf2622fb4d8" dependencies = [ - "rand", + "rand 0.8.5", ] [[package]] @@ -2844,7 +2885,7 @@ version = "6.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "crossbeam-channel", "filetime", "fsevent-sys", @@ -2912,7 +2953,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", - "libm", ] [[package]] @@ -2921,24 +2961,24 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", ] [[package]] name = "num_threads" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" dependencies = [ "libc", ] [[package]] name = "object" -version = "0.32.2" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] @@ -2963,15 +3003,15 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.20.3" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "openssl-probe" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "ordered-multimap" @@ -3004,11 +3044,17 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -3016,15 +3062,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -3041,15 +3087,15 @@ checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef" [[package]] name = "pathdiff" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" [[package]] name = "pear" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ccca0f6c17acc81df8e242ed473ec144cbf5c98037e69aa6d144780aad103c8" +checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" dependencies = [ "inlinable_string", "pear_codegen", @@ -3058,14 +3104,14 @@ dependencies = [ [[package]] name = "pear_codegen" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e22670e8eb757cff11d6c199ca7b987f352f0346e0be4dd23869ec72cb53c77" +checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -3086,20 +3132,20 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.6" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f200d8d83c44a45b21764d1916299752ca035d15ecd46faca3e9a2a2bf6ad06" +checksum = "198db74531d58c70a361c42201efde7e2591e976d518caf7662a47dc5720e7b6" dependencies = [ "memchr", - "thiserror 1.0.69", + "thiserror 2.0.12", "ucd-trie", ] [[package]] name = "pest_derive" -version = "2.7.6" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcd6ab1236bbdb3a49027e920e693192ebfe8913f6d60e294de57463a493cfde" +checksum = "d725d9cfd79e87dccc9341a2ef39d1b6f6353d68c4b33c177febbe1a402c97c5" dependencies = [ "pest", "pest_generator", @@ -3107,22 +3153,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.6" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a31940305ffc96863a735bef7c7994a00b325a7138fdbc5bda0f1a0476d3275" +checksum = "db7d01726be8ab66ab32f9df467ae8b1148906685bbe75c82d1e65d7f5b3f841" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] name = "pest_meta" -version = "2.7.6" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7ff62f5259e53b78d1af898941cdcdccfae7385cf7d793a6e55de5d05bb4b7d" +checksum = "7f9f832470494906d1fca5329f8ab5791cc60beb230c74815dff541cbd2b5ca0" dependencies = [ "once_cell", "pest", @@ -3141,29 +3187,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.4" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.4" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -3197,9 +3243,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.6.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" [[package]] name = "postcard" @@ -3214,6 +3260,15 @@ dependencies = [ "serde", ] +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -3222,9 +3277,12 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] [[package]] name = "predicates" @@ -3242,9 +3300,9 @@ dependencies = [ [[package]] name = "predicates" -version = "3.1.0" +version = "3.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b87bfd4605926cdfefc1c3b5f8fe560e3feca9d5552cf68c466d3d8236c7e8" +checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573" dependencies = [ "anstyle", "difflib", @@ -3253,15 +3311,15 @@ dependencies = [ [[package]] name = "predicates-core" -version = "1.0.6" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" +checksum = "727e462b119fe9c93fd0eb1429a5f7647394014cf3c04ab2c0350eeb09095ffa" [[package]] name = "predicates-tree" -version = "1.0.9" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +checksum = "72dd2d6d381dfb73a193c7fca536518d7caee39fc8503f74e7dc0be0531b425c" dependencies = [ "predicates-core", "termtree", @@ -3279,19 +3337,19 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.22" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" +checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" dependencies = [ "proc-macro2", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] name = "proc-macro2" -version = "1.0.93" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] @@ -3304,26 +3362,26 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", "version_check", "yansi", ] [[package]] name = "proptest" -version = "1.4.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf" +checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.8.0", + "bitflags 2.9.0", "lazy_static", "num-traits", - "rand", - "rand_chacha", + "rand 0.8.5", + "rand_chacha 0.3.1", "rand_xorshift", - "regex-syntax 0.8.2", + "regex-syntax 0.8.5", "rusty-fork", "tempfile", "unarray", @@ -3353,37 +3411,40 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quinn" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" +checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" dependencies = [ "bytes", + "cfg_aliases", "pin-project-lite", "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.22", + "rustls 0.23.27", "socket2", - "thiserror 2.0.11", + "thiserror 2.0.12", "tokio", "tracing", + "web-time", ] [[package]] name = "quinn-proto" -version = "0.11.9" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" +checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" dependencies = [ "bytes", - "getrandom 0.2.12", - "rand", + "getrandom 0.3.3", + "lru-slab", + "rand 0.9.1", "ring", "rustc-hash", - "rustls 0.23.22", + "rustls 0.23.27", "rustls-pki-types", "slab", - "thiserror 2.0.11", + "thiserror 2.0.12", "tinyvec", "tracing", "web-time", @@ -3391,9 +3452,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.9" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" +checksum = "ee4e529991f949c5e25755532370b8af5d114acae52326361d68d47af64aa842" dependencies = [ "cfg_aliases", "libc", @@ -3405,13 +3466,19 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.39" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1f1914ce909e1658d9907913b4b91947430c7d9be598b15a1912935b8c04801" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + [[package]] name = "radium" version = "0.7.0" @@ -3425,8 +3492,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", ] [[package]] @@ -3436,7 +3513,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", ] [[package]] @@ -3445,7 +3532,16 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.12", + "getrandom 0.2.16", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.3", ] [[package]] @@ -3454,7 +3550,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" dependencies = [ - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -3559,23 +3655,23 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", ] [[package]] name = "regex" -version = "1.10.3" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.5", - "regex-syntax 0.8.2", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", ] [[package]] @@ -3589,13 +3685,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.5" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.2", + "regex-syntax 0.8.5", ] [[package]] @@ -3606,21 +3702,21 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.2" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.9" +version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" +checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" dependencies = [ "base64 0.22.1", "bytes", "futures-core", "futures-util", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "http-body-util", "hyper 1.6.0", @@ -3635,7 +3731,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.22", + "rustls 0.23.27", "rustls-native-certs", "rustls-pemfile 2.2.0", "rustls-pki-types", @@ -3644,8 +3740,9 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.2", "tokio", - "tokio-rustls 0.26.1", + "tokio-rustls 0.26.2", "tokio-util", + "tower 0.5.2", "tower-service", "url", "wasm-bindgen", @@ -3657,13 +3754,13 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.13" +version = "0.17.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.12", + "getrandom 0.2.16", "libc", "untrusted", "windows-sys 0.52.0", @@ -3687,7 +3784,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" dependencies = [ "base64 0.21.7", - "bitflags 2.8.0", + "bitflags 2.9.0", "serde", "serde_derive", ] @@ -3702,6 +3799,34 @@ dependencies = [ "winapi", ] +[[package]] +name = "rquickjs" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5227859c4dfc83f428e58f9569bf439e628c8d139020e7faff437e6f5abaa0" +dependencies = [ + "rquickjs-core", +] + +[[package]] +name = "rquickjs-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e82e0ca83028ad5b533b53b96c395bbaab905a5774de4aaf1004eeacafa3d85d" +dependencies = [ + "async-lock", + "rquickjs-sys", +] + +[[package]] +name = "rquickjs-sys" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fed0097b0b4fbb2a87f6dd3b995a7c64ca56de30007eb7e867dfdfc78324ba5" +dependencies = [ + "cc", +] + [[package]] name = "rstest" version = "0.16.0" @@ -3742,9 +3867,9 @@ dependencies = [ "rustls-native-certs", "rustls-pemfile 2.2.0", "rustls-webpki 0.102.8", - "thiserror 2.0.11", + "thiserror 2.0.12", "tokio", - "tokio-rustls 0.26.1", + "tokio-rustls 0.26.2", "tokio-stream", "tokio-util", ] @@ -3765,7 +3890,7 @@ dependencies = [ "metrics", "metrics-exporter-prometheus", "parking_lot", - "rand", + "rand 0.8.5", "rustls-pemfile 1.0.4", "rustls-webpki 0.101.7", "serde", @@ -3793,21 +3918,21 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] @@ -3823,15 +3948,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "errno", "libc", - "linux-raw-sys 0.4.13", - "windows-sys 0.52.0", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", ] [[package]] @@ -3840,7 +3965,7 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "errno", "libc", "linux-raw-sys 0.9.4", @@ -3849,9 +3974,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.11" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring", @@ -3861,14 +3986,14 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.22" +version = "0.23.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9263ab4eb695e42321db096e3b8fbd715a59b154d5c88d82db2175b681ba7" +checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.102.8", + "rustls-webpki 0.103.3", "subtle", "zeroize", ] @@ -3934,11 +4059,22 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustls-webpki" +version = "0.103.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" [[package]] name = "rusty-fork" @@ -3954,9 +4090,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.16" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" [[package]] name = "same-file" @@ -3969,11 +4105,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4013,7 +4149,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "core-foundation", "core-foundation-sys", "libc", @@ -4032,46 +4168,47 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.21" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" [[package]] name = "serde" -version = "1.0.196" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.196" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] name = "serde_json" -version = "1.0.113" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] [[package]] name = "serde_path_to_error" -version = "0.1.15" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd154a240de39fdebcf5775d2675c204d7c13cf39a4c697be6493c8e734337c" +checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a" dependencies = [ "itoa", "serde", @@ -4079,9 +4216,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.5" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" dependencies = [ "serde", ] @@ -4111,9 +4248,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -4122,9 +4259,9 @@ dependencies = [ [[package]] name = "sha256" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18278f6a914fa3070aa316493f7d2ddfb9ac86ebc06fa3b83bffda487e9065b0" +checksum = "f880fc8562bdeb709793f00eb42a2ad0e672c4f883bbe59122b926eca935c8f6" dependencies = [ "async-trait", "bytes", @@ -4156,24 +4293,24 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" -version = "1.4.1" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" dependencies = [ "libc", ] [[package]] name = "similar" -version = "2.4.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32fea41aca09ee824cc9724996433064c89f7777e60762749a4170a14abbfa21" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" [[package]] name = "sketches-ddsketch" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a406c1882ed7f29cd5e248c9848a80e7cb6ae0fea82346d2746f2f941c07e1" +checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" [[package]] name = "slab" @@ -4186,9 +4323,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.13.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" [[package]] name = "smawk" @@ -4220,12 +4357,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.5" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -4324,9 +4461,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.96" +version = "2.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" +checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" dependencies = [ "proc-macro2", "quote", @@ -4362,13 +4499,13 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -4443,7 +4580,7 @@ dependencies = [ "thiserror 1.0.69", "time", "tokio", - "toml 0.8.8", + "toml 0.8.22", "tracing", "tracing-subscriber", "url", @@ -4460,7 +4597,7 @@ dependencies = [ "anyhow", "assert-json-diff", "async-trait", - "axum 0.8.1", + "axum 0.8.4", "axum-server", "axum_tls", "camino", @@ -4478,7 +4615,7 @@ dependencies = [ "rcgen", "reqwest", "ron 0.8.1", - "rustls 0.23.22", + "rustls 0.23.27", "serde", "serde_json", "sha256", @@ -4502,7 +4639,7 @@ dependencies = [ "time", "tokio", "tokio-util", - "toml 0.8.8", + "toml 0.8.22", "tower 0.4.13", "tower-http", "tracing", @@ -4561,18 +4698,18 @@ name = "tedge-p11-server" version = "1.5.1" dependencies = [ "anyhow", - "asn1-rs 0.7.0", + "asn1-rs 0.7.1", "camino", "clap", "cryptoki", "percent-encoding", "postcard", - "rustls 0.23.22", + "rustls 0.23.27", "sd-listen-fds", "serde", "tempfile", "tokio", - "toml 0.8.8", + "toml 0.8.22", "tracing", "tracing-subscriber", ] @@ -4651,7 +4788,7 @@ dependencies = [ "thiserror 1.0.69", "time", "tokio", - "toml 0.8.8", + "toml 0.8.22", "walkdir", ] @@ -4673,7 +4810,7 @@ dependencies = [ "path-clean", "regex", "reqwest", - "rustls 0.23.22", + "rustls 0.23.27", "serde", "strum", "strum_macros", @@ -4684,7 +4821,7 @@ dependencies = [ "test-case", "thiserror 1.0.69", "tokio", - "toml 0.8.8", + "toml 0.8.22", "tracing", "tracing-subscriber", "url", @@ -4707,7 +4844,7 @@ dependencies = [ "serde_json", "tedge_config_macros-macro", "thiserror 1.0.69", - "toml 0.8.8", + "toml 0.8.22", "tracing", "url", ] @@ -4716,7 +4853,7 @@ dependencies = [ name = "tedge_config_macros-impl" version = "1.5.1" dependencies = [ - "darling 0.20.10", + "darling 0.20.11", "heck 0.4.1", "itertools 0.13.0", "pretty_assertions", @@ -4724,7 +4861,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.96", + "syn 2.0.101", "test-case", ] @@ -4734,7 +4871,7 @@ version = "1.5.1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", "tedge_config_macros-impl", ] @@ -4761,7 +4898,7 @@ dependencies = [ "tempfile", "thiserror 1.0.69", "tokio", - "toml 0.8.8", + "toml 0.8.22", "uzers", ] @@ -4800,13 +4937,15 @@ version = "1.5.1" dependencies = [ "async-trait", "camino", + "rquickjs", "serde", + "serde_json", "tedge_actors", "tedge_mqtt_ext", "thiserror 1.0.69", "time", "tokio", - "toml 0.8.8", + "toml 0.8.22", "tracing", ] @@ -4830,13 +4969,13 @@ name = "tedge_http_ext" version = "1.5.1" dependencies = [ "async-trait", - "http 1.2.0", + "http 1.3.1", "http-body-util", "hyper 1.6.0", "hyper-rustls", "hyper-util", "mockito", - "rustls 0.23.22", + "rustls 0.23.27", "serde", "serde_json", "tedge_actors", @@ -4855,7 +4994,7 @@ dependencies = [ "filetime", "glob", "log", - "rand", + "rand 0.8.5", "regex", "serde", "serde_json", @@ -4870,7 +5009,7 @@ dependencies = [ "thiserror 1.0.69", "time", "tokio", - "toml 0.8.8", + "toml 0.8.22", ] [[package]] @@ -4943,7 +5082,7 @@ dependencies = [ "anyhow", "camino", "tempfile", - "toml 0.8.8", + "toml 0.8.22", ] [[package]] @@ -5001,14 +5140,14 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.12.0" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" dependencies = [ - "cfg-if", "fastrand", + "getrandom 0.3.3", "once_cell", - "rustix 0.38.34", + "rustix 1.0.7", "windows-sys 0.59.0", ] @@ -5033,9 +5172,9 @@ dependencies = [ [[package]] name = "termtree" -version = "0.4.1" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" [[package]] name = "test-case" @@ -5055,7 +5194,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -5066,7 +5205,7 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", "test-case-core", ] @@ -5092,11 +5231,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" dependencies = [ - "thiserror-impl 2.0.11", + "thiserror-impl 2.0.12", ] [[package]] @@ -5107,25 +5246,25 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] name = "thiserror-impl" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] name = "thread_local" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ "cfg-if", "once_cell", @@ -5133,9 +5272,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" dependencies = [ "deranged", "itoa", @@ -5150,15 +5289,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" dependencies = [ "num-conv", "time-core", @@ -5166,9 +5305,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.7.6" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" dependencies = [ "displaydoc", "zerovec", @@ -5176,9 +5315,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" dependencies = [ "tinyvec_macros", ] @@ -5191,9 +5330,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.44.2" +version = "1.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" +checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165" dependencies = [ "backtrace", "bytes", @@ -5215,7 +5354,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -5224,17 +5363,17 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.11", + "rustls 0.21.12", "tokio", ] [[package]] name = "tokio-rustls" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.22", + "rustls 0.23.27", "tokio", ] @@ -5251,32 +5390,31 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4bf6fecd69fcdede0ec680aaf474cdab988f9de6bc73d3758f0160e3b7025a" +checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" dependencies = [ "futures-util", "log", - "rustls 0.23.22", + "rustls 0.23.27", "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.1", - "tungstenite 0.26.1", + "tokio-rustls 0.26.2", + "tungstenite 0.26.2", ] [[package]] name = "tokio-util" -version = "0.7.10" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes", "futures-core", "futures-sink", "pin-project-lite", "tokio", - "tracing", ] [[package]] @@ -5290,9 +5428,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.8" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" +checksum = "05ae329d1f08c4d17a59bed7ff5b5a769d062e64a62d34a3261b219e62cd5aae" dependencies = [ "serde", "serde_spanned", @@ -5302,26 +5440,33 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.5" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.21.0" +version = "0.22.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" +checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e" dependencies = [ - "indexmap 2.2.1", + "indexmap 2.9.0", "serde", "serde_spanned", "toml_datetime", + "toml_write", "winnow", ] +[[package]] +name = "toml_write" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfb942dfe1d8e29a7ee7fcbde5bd2b9a25fb89aa70caea2eba3bee836ff41076" + [[package]] name = "tower" version = "0.4.13" @@ -5360,9 +5505,9 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "bytes", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "http-body-util", "pin-project-lite", @@ -5384,9 +5529,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "log", "pin-project-lite", @@ -5396,20 +5541,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", @@ -5428,9 +5573,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", "nu-ansi-term", @@ -5466,10 +5611,10 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http 0.2.11", + "http 0.2.12", "httparse", "log", - "rand", + "rand 0.8.5", "sha1", "thiserror 1.0.69", "url", @@ -5485,11 +5630,11 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http 1.2.0", + "http 1.3.1", "httparse", "log", - "rand", - "rustls 0.23.22", + "rand 0.8.5", + "rustls 0.23.27", "rustls-pki-types", "sha1", "thiserror 1.0.69", @@ -5498,29 +5643,28 @@ dependencies = [ [[package]] name = "tungstenite" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413083a99c579593656008130e29255e54dcaae495be556cc26888f211648c24" +checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" dependencies = [ - "byteorder", "bytes", "data-encoding", - "http 1.2.0", + "http 1.3.1", "httparse", "log", - "rand", - "rustls 0.23.22", + "rand 0.9.1", + "rustls 0.23.27", "rustls-pki-types", "sha1", - "thiserror 2.0.11", + "thiserror 2.0.12", "utf-8", ] [[package]] name = "typenum" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "typewit" @@ -5539,9 +5683,9 @@ checksum = "e36a83ea2b3c704935a01b4642946aadd445cea40b10935e3f8bd8052b8193d6" [[package]] name = "ucd-trie" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" [[package]] name = "unarray" @@ -5560,18 +5704,15 @@ dependencies = [ [[package]] name = "unicase" -version = "2.7.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] name = "unicode-linebreak" @@ -5581,15 +5722,15 @@ checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" [[package]] name = "unicode-width" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-xid" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "untrusted" @@ -5602,7 +5743,7 @@ name = "upload" version = "1.5.1" dependencies = [ "anyhow", - "axum 0.8.1", + "axum 0.8.4", "axum_tls", "backoff", "camino", @@ -5637,12 +5778,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" -[[package]] -name = "utf16_iter" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -5651,17 +5786,17 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "utf8parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.15.1" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587" +checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" dependencies = [ - "getrandom 0.3.1", + "getrandom 0.3.3", ] [[package]] @@ -5676,30 +5811,30 @@ dependencies = [ [[package]] name = "valuable" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "wait-timeout" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" dependencies = [ "libc", ] [[package]] name = "walkdir" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", @@ -5722,9 +5857,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasi" -version = "0.13.3+wasi-0.2.2" +version = "0.14.2+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" dependencies = [ "wit-bindgen-rt", ] @@ -5737,46 +5872,48 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.90" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", + "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.90" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.40" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" dependencies = [ "cfg-if", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.90" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5784,28 +5921,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.90" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.90" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "wasm-streams" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" dependencies = [ "futures-util", "js-sys", @@ -5816,9 +5956,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.67" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -5843,14 +5983,14 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.34", + "rustix 0.38.44", ] [[package]] name = "whoami" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fec781d48b41f8163426ed18e8fc2864c12937df9ce54c88ede7bd47270893e" +checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7" dependencies = [ "redox_syscall", "wasite", @@ -5875,11 +6015,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "winapi", + "windows-sys 0.59.0", ] [[package]] @@ -5888,34 +6028,39 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + [[package]] name = "windows-registry" -version = "0.2.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ "windows-result", "windows-strings", - "windows-targets 0.52.6", + "windows-targets 0.53.0", ] [[package]] name = "windows-result" -version = "0.2.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" dependencies = [ - "windows-targets 0.52.6", + "windows-link", ] [[package]] name = "windows-strings" -version = "0.1.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" dependencies = [ - "windows-result", - "windows-targets 0.52.6", + "windows-link", ] [[package]] @@ -5969,13 +6114,29 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -5988,6 +6149,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -6000,6 +6167,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -6012,12 +6185,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -6030,6 +6215,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -6042,6 +6233,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -6054,6 +6251,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -6066,35 +6269,35 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "winnow" -version = "0.5.35" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1931d78a9c73861da0134f453bb1f790ce49b2e30eba8410b4b79bac72b46a2d" +checksum = "c06928c8748d81b05c9be96aad92e1b6ff01833332f281e8cfca3be4b35fc9ec" dependencies = [ "memchr", ] [[package]] name = "wit-bindgen-rt" -version = "0.33.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", ] -[[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" - [[package]] name = "writeable" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" [[package]] name = "ws_stream_tungstenite" @@ -6104,7 +6307,7 @@ checksum = "e283cc794a890f5bdc01e358ad7c34535025f79ba83c1b5c7e01e5d6c60b336d" dependencies = [ "async-tungstenite 0.23.0", "async_io_stream", - "bitflags 2.8.0", + "bitflags 2.9.0", "futures-core", "futures-io", "futures-sink", @@ -6122,9 +6325,9 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed39ff9f8b2eda91bf6390f9f49eee93d655489e15708e3bb638c1c4f07cecb4" dependencies = [ - "async-tungstenite 0.28.0", + "async-tungstenite 0.28.2", "async_io_stream", - "bitflags 2.8.0", + "bitflags 2.9.0", "futures-core", "futures-io", "futures-sink", @@ -6214,9 +6417,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" dependencies = [ "serde", "stable_deref_trait", @@ -6226,55 +6429,55 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", - "synstructure 0.13.1", + "syn 2.0.101", + "synstructure 0.13.2", ] [[package]] name = "zerocopy" -version = "0.7.32" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.32" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] name = "zerofrom" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", - "synstructure 0.13.1", + "syn 2.0.101", + "synstructure 0.13.2", ] [[package]] @@ -6283,11 +6486,22 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + [[package]] name = "zerovec" -version = "0.10.4" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" dependencies = [ "yoke", "zerofrom", @@ -6296,11 +6510,11 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.10.3" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] diff --git a/Cargo.toml b/Cargo.toml index 059536d342a..7c7b41d9704 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -164,7 +164,7 @@ regex = "1.4" reqwest = { version = "0.12", default-features = false } ron = "0.8" rpassword = "5.0" -rquickjs = { version = "0.9", default-features = false} +rquickjs = { version = "0.9", default-features = false } rstest = "0.16.0" rumqttc = { git = "https://github.com/jarhodes314/rumqtt", rev = "8c489faf6af910956c97b55587ff3ecb2ac4e96f" } rumqttd = "0.19" diff --git a/crates/core/tedge_mapper/src/gen/mod.rs b/crates/core/tedge_mapper/src/gen/mod.rs index ceb8b1acb6c..091fb47542b 100644 --- a/crates/core/tedge_mapper/src/gen/mod.rs +++ b/crates/core/tedge_mapper/src/gen/mod.rs @@ -15,8 +15,8 @@ impl TEdgeComponent for GenMapper { let (mut runtime, mut mqtt_actor) = start_basic_actors("tedge-gen-mapper", &tedge_config).await?; - let mut gen_mapper = GenMapperBuilder::default(); - gen_mapper.load("/etc/tedge/gen-mapper").await; + let mut gen_mapper = GenMapperBuilder::try_new("/etc/tedge/gen-mapper").await?; + gen_mapper.load().await; gen_mapper.connect(&mut mqtt_actor); runtime.spawn(gen_mapper).await?; diff --git a/crates/extensions/tedge_gen_mapper/Cargo.toml b/crates/extensions/tedge_gen_mapper/Cargo.toml index ff51e439ed7..32b302357e3 100644 --- a/crates/extensions/tedge_gen_mapper/Cargo.toml +++ b/crates/extensions/tedge_gen_mapper/Cargo.toml @@ -10,8 +10,10 @@ repository.workspace = true [dependencies] async-trait = { workspace = true } -camino = { workspace = true } +camino = { workspace = true, features = ["serde1"] } +rquickjs = { workspace = true, features = ["futures","parallel"] } serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } tedge_actors = { workspace = true } tedge_mqtt_ext = { workspace = true } thiserror = { workspace = true } diff --git a/crates/extensions/tedge_gen_mapper/pipelines/add_timestamp.js b/crates/extensions/tedge_gen_mapper/pipelines/add_timestamp.js new file mode 100644 index 00000000000..2c58927b104 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/add_timestamp.js @@ -0,0 +1,9 @@ +export function process (timestamp, message) { + let payload = JSON.parse(message.payload) + payload.time = Number(timestamp.seconds) + (timestamp.nanoseconds / 1e9) + + return [{ + topic: message.topic, + payload: JSON.stringify(payload) + }] +} diff --git a/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml b/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml new file mode 100644 index 00000000000..2501140d5e0 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml @@ -0,0 +1,6 @@ +input_topics = ["collectd/+/+/+"] + +stages = [ + { filter = "collectd-to-te.js" }, + { filter = "group_by_timestamp.js" } +] diff --git a/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml b/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml new file mode 100644 index 00000000000..0d5e06d55a0 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml @@ -0,0 +1,6 @@ +input_topics = ["te/+/+/+/+/m/+"] + +stages = [ + { filter = "add_timestamp.js" }, + { filter = "te_to_c8y.js" } +] diff --git a/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js b/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js new file mode 100644 index 00000000000..9c7583a8bd3 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js @@ -0,0 +1,6 @@ +export function process(t,msg) { + msg.topic = "te/error" + return [msg] +} + + diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index ac72cd562d7..69cc6c9b77f 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -1,3 +1,4 @@ +use crate::js_filter::JsRuntime; use crate::pipeline::Pipeline; use async_trait::async_trait; use std::collections::HashMap; @@ -15,6 +16,7 @@ use tracing::error; pub struct GenMapper { pub(super) mqtt: SimpleMessageBox, pub(super) pipelines: HashMap, + pub(super) js_runtime: JsRuntime, } #[async_trait] @@ -47,7 +49,10 @@ impl GenMapper { async fn filter(&mut self, message: MqttMessage) -> Result<(), RuntimeError> { let timestamp = OffsetDateTime::now_utc(); for (pipeline_id, pipeline) in self.pipelines.iter_mut() { - match pipeline.process(timestamp, &message) { + match pipeline + .process(&self.js_runtime, timestamp, &message) + .await + { Ok(messages) => { for message in messages { self.mqtt.send(message).await?; @@ -65,7 +70,7 @@ impl GenMapper { async fn tick(&mut self) -> Result<(), RuntimeError> { let timestamp = OffsetDateTime::now_utc(); for (pipeline_id, pipeline) in self.pipelines.iter_mut() { - match pipeline.tick(timestamp) { + match pipeline.tick(&self.js_runtime, timestamp).await { Ok(messages) => { for message in messages { self.mqtt.send(message).await?; diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index a445729f86f..694fb901451 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -1,8 +1,10 @@ +use crate::js_filter::JsRuntime; use crate::pipeline::Pipeline; use crate::pipeline::Stage; -use crate::gen_filter::GenFilter; +use crate::LoadError; +use camino::Utf8PathBuf; use serde::Deserialize; -use std::path::PathBuf; +use std::path::Path; use tedge_mqtt_ext::TopicFilter; #[derive(Deserialize)] @@ -22,40 +24,48 @@ pub struct StageConfig { #[derive(Deserialize)] #[serde(untagged)] pub enum FilterSpec { - JavaScript(PathBuf), + JavaScript(Utf8PathBuf), } #[derive(thiserror::Error, Debug)] pub enum ConfigError { #[error("Not a valid MQTT topic filter: {0}")] IncorrectTopicFilter(String), -} -impl TryFrom for Pipeline { - type Error = ConfigError; + #[error(transparent)] + LoadError(#[from] LoadError), +} - fn try_from(config: PipelineConfig) -> Result { - let input = topic_filters(&config.input_topics)?; - let stages = config +impl PipelineConfig { + pub fn compile( + self, + js_runtime: &JsRuntime, + config_dir: &Path, + ) -> Result { + let input = topic_filters(&self.input_topics)?; + let stages = self .stages .into_iter() - .map(Stage::try_from) + .map(|stage| stage.compile(js_runtime, config_dir)) .collect::, _>>()?; - Ok(Pipeline { input_topics: input, stages }) + Ok(Pipeline { + input_topics: input, + stages, + }) } } -impl TryFrom for Stage { - type Error = ConfigError; - - fn try_from(config: StageConfig) -> Result { - let filter = match config.filter { - FilterSpec::JavaScript(path) => GenFilter::new(path), +impl StageConfig { + pub fn compile(self, js_runtime: &JsRuntime, config_dir: &Path) -> Result { + let path = match self.filter { + FilterSpec::JavaScript(path) if path.is_absolute() => path.into(), + FilterSpec::JavaScript(path) => config_dir.join(path), }; - let config = topic_filters(&config.config_topics)?; + let filter = js_runtime.loaded_module(path)?; + let config_topics = topic_filters(&self.config_topics)?; Ok(Stage { - filter: Box::new(filter), - config_topics: config, + filter, + config_topics, }) } } diff --git a/crates/extensions/tedge_gen_mapper/src/gen_filter.rs b/crates/extensions/tedge_gen_mapper/src/gen_filter.rs deleted file mode 100644 index 038543c8340..00000000000 --- a/crates/extensions/tedge_gen_mapper/src/gen_filter.rs +++ /dev/null @@ -1,38 +0,0 @@ -use crate::pipeline::Filter; -use crate::pipeline::FilterError; -use std::path::PathBuf; -use tedge_mqtt_ext::MqttMessage; -use time::OffsetDateTime; -use tracing::debug; - -/// User-defined filter -pub struct GenFilter {} - -impl GenFilter { - pub fn new(path: impl Into) -> Self { - let path = path.into(); - debug!(target: "MAPPING", "new({path:?})"); - GenFilter {} - } -} - -impl Filter for GenFilter { - fn process( - &mut self, - timestamp: OffsetDateTime, - message: &MqttMessage, - ) -> Result, FilterError> { - debug!(target: "MAPPING", "process({timestamp}, {message:?})"); - Ok(vec![message.clone()]) - } - - fn update_config(&mut self, config: &MqttMessage) -> Result<(), FilterError> { - debug!(target: "MAPPING", "update_config({config:?})"); - Ok(()) - } - - fn tick(&mut self, timestamp: OffsetDateTime) -> Result, FilterError> { - debug!(target: "MAPPING", "tick({timestamp})"); - Ok(vec![]) - } -} diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs new file mode 100644 index 00000000000..906f3287068 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -0,0 +1,250 @@ +use crate::pipeline; +use crate::pipeline::FilterError; +use crate::LoadError; +use rquickjs::Ctx; +use rquickjs::FromJs; +use rquickjs::IntoJs; +use rquickjs::Object; +use rquickjs::Value; +use std::collections::HashMap; +use std::path::Path; +use std::path::PathBuf; +use tedge_mqtt_ext::MqttMessage; +use time::OffsetDateTime; +use tracing::debug; + +#[derive(serde::Deserialize, serde::Serialize)] +pub struct DateTime { + seconds: u64, + nanoseconds: u32, +} + +#[derive(serde::Deserialize, serde::Serialize)] +pub struct Message { + topic: String, + payload: String, +} + +#[derive(Clone)] +pub struct JsFilter { + path: PathBuf, +} + +impl JsFilter { + pub async fn process( + &self, + js: &JsRuntime, + timestamp: OffsetDateTime, + message: &MqttMessage, + ) -> Result, FilterError> { + debug!(target: "MAPPING", "{}: process({timestamp}, {message:?})", self.path.display()); + let timestamp = DateTime::try_from(timestamp)?; + let message = Message::try_from(message)?; + let input = (timestamp, message); + let output: Vec = js + .call_function(&self, "process", input) + .await + .map_err(error_from_js)?; + output.into_iter().map(MqttMessage::try_from).collect() + } + + pub fn update_config(&self, _js: &JsRuntime, config: &MqttMessage) -> Result<(), FilterError> { + debug!(target: "MAPPING", "{}: update_config({config:?})", self.path.display()); + Ok(()) + } + + pub fn tick( + &self, + _js: &JsRuntime, + timestamp: OffsetDateTime, + ) -> Result, FilterError> { + debug!(target: "MAPPING", "{}: tick({timestamp})", self.path.display()); + Ok(vec![]) + } +} + +pub struct JsRuntime { + context: rquickjs::AsyncContext, + modules: HashMap>, +} + +impl JsRuntime { + pub async fn try_new() -> Result { + let runtime = rquickjs::AsyncRuntime::new()?; + let context = rquickjs::AsyncContext::full(&runtime).await?; + let modules = HashMap::new(); + Ok(JsRuntime { context, modules }) + } + + pub async fn load_file(&mut self, path: impl AsRef) -> Result { + let path = path.as_ref(); + let source = tokio::fs::read_to_string(path).await?; + self.load_js(path, source) + } + + pub fn load_js( + &mut self, + path: impl AsRef, + source: impl Into>, + ) -> Result { + let path = path.as_ref().to_path_buf(); + self.modules.insert(path.clone(), source.into()); + Ok(JsFilter { path }) + } + + pub fn loaded_module(&self, path: PathBuf) -> Result { + match self.modules.get(&path) { + None => Err(LoadError::ScriptNotLoaded { path }), + Some(_) => Ok(JsFilter { path }), + } + } + + pub async fn call_function( + &self, + module: &JsFilter, + function: &str, + args: Args, + ) -> Result + where + for<'a> Args: rquickjs::function::IntoArgs<'a> + Send + 'a, + for<'a> Ret: rquickjs::FromJs<'a> + Send + 'a, + { + let Some(source) = self.modules.get(&module.path) else { + return Err(LoadError::ScriptNotLoaded { + path: module.path.clone(), + }); + }; + + let name = module.path.display().to_string(); + + rquickjs::async_with!(self.context => |ctx| { + let m = rquickjs::Module::declare(ctx, name, source.clone())?; + let (m,p) = m.eval()?; + let () = p.finish()?; + + let f: rquickjs::Value = m.get(function)?; + let f = rquickjs::Function::from_value(f)?; + let r = f.call(args)?; + Ok(r) + }) + .await + } +} + +impl TryFrom for DateTime { + type Error = FilterError; + + fn try_from(value: OffsetDateTime) -> Result { + let seconds = u64::try_from(value.unix_timestamp()).map_err(|err| { + FilterError::UnsupportedMessage(format!("failed to convert timestamp: {}", err)) + })?; + + Ok(DateTime { + seconds, + nanoseconds: value.nanosecond(), + }) + } +} + +impl TryFrom<&MqttMessage> for Message { + type Error = FilterError; + + fn try_from(message: &MqttMessage) -> Result { + let topic = message.topic.to_string(); + let payload = message + .payload_str() + .map_err(|_| { + pipeline::FilterError::UnsupportedMessage("Not an UTF8 payload".to_string()) + })? + .to_string(); + Ok(Message { topic, payload }) + } +} + +impl TryFrom for MqttMessage { + type Error = FilterError; + + fn try_from(message: Message) -> Result { + let topic = message.topic.as_str().try_into().map_err(|_| { + FilterError::UnsupportedMessage(format!("invalid topic {}", message.topic)) + })?; + Ok(MqttMessage::new(&topic, message.payload)) + } +} + +impl<'js> FromJs<'js> for Message { + fn from_js(_ctx: &Ctx<'js>, value: Value<'js>) -> rquickjs::Result { + match value.as_object() { + None => Ok(Message { + topic: "".to_string(), + payload: "".to_string(), + }), + Some(object) => Ok(Message { + topic: object.get("topic")?, + payload: object.get("payload")?, + }), + } + } +} + +impl<'js> IntoJs<'js> for Message { + fn into_js(self, ctx: &Ctx<'js>) -> rquickjs::Result> { + let msg = Object::new(ctx.clone())?; + msg.set("topic", self.topic)?; + msg.set("payload", self.payload)?; + Ok(Value::from_object(msg)) + } +} + +impl<'js> IntoJs<'js> for DateTime { + fn into_js(self, ctx: &Ctx<'js>) -> rquickjs::Result> { + let msg = Object::new(ctx.clone())?; + msg.set("topic", self.seconds)?; + msg.set("payload", self.nanoseconds)?; + Ok(Value::from_object(msg)) + } +} + +fn error_from_js(err: LoadError) -> FilterError { + FilterError::IncorrectSetting(format!("{}", err)) +} + +#[cfg(test)] +mod tests { + use super::*; + use tedge_mqtt_ext::Topic; + + #[tokio::test] + async fn identity_filter() { + let script = "export function process(t,msg) { return [msg]; };"; + let mut runtime = JsRuntime::try_new().await.unwrap(); + let filter = runtime.load_js("id.js", script).unwrap(); + + let topic = Topic::new_unchecked("te/main/device///m/"); + let input = MqttMessage::new(&topic, "hello world"); + let output = input.clone(); + assert_eq!( + filter + .process(&runtime, OffsetDateTime::now_utc(), &input) + .await + .unwrap(), + vec![output] + ); + } + + #[tokio::test] + async fn error_filter() { + let script = r#"export function process(t,msg) { throw new Error("Cannot process that message"); };"#; + let mut runtime = JsRuntime::try_new().await.unwrap(); + let filter = runtime.load_js("err.js", script).unwrap(); + + let topic = Topic::new_unchecked("te/main/device///m/"); + let input = MqttMessage::new(&topic, "hello world"); + let error = filter + .process(&runtime, OffsetDateTime::now_utc(), &input) + .await + .unwrap_err(); + eprintln!("{:?}", error); + assert!(error.to_string().contains("Exception generated by QuickJS")); + } +} diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index b5aee325715..8019c489ae2 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -1,14 +1,17 @@ mod actor; mod config; +mod js_filter; mod pipeline; -mod gen_filter; use crate::actor::GenMapper; +use crate::config::PipelineConfig; +use crate::js_filter::JsRuntime; use crate::pipeline::Pipeline; use camino::Utf8Path; use std::collections::HashMap; use std::convert::Infallible; use std::path::Path; +use std::path::PathBuf; use tedge_actors::Builder; use tedge_actors::DynSender; use tedge_actors::MessageSink; @@ -25,24 +28,29 @@ use tracing::error; use tracing::info; pub struct GenMapperBuilder { + config_dir: PathBuf, message_box: SimpleMessageBoxBuilder, pipelines: HashMap, + pipeline_specs: HashMap, + js_runtime: JsRuntime, } -impl Default for GenMapperBuilder { - fn default() -> Self { - GenMapperBuilder { +impl GenMapperBuilder { + pub async fn try_new(config_dir: impl AsRef) -> Result { + let config_dir = config_dir.as_ref().to_owned(); + let js_runtime = JsRuntime::try_new().await?; + Ok(GenMapperBuilder { + config_dir, message_box: SimpleMessageBoxBuilder::new("GenMapper", 16), pipelines: HashMap::default(), - } + pipeline_specs: HashMap::default(), + js_runtime, + }) } -} -impl GenMapperBuilder { - pub async fn load(&mut self, config_dir: impl AsRef) { - let config_dir = config_dir.as_ref(); - let Ok(mut entries) = read_dir(config_dir).await.map_err(|err| - error!(target: "MAPPING", "Failed to read filters from {}: {err}", config_dir.display()) + pub async fn load(&mut self) { + let Ok(mut entries) = read_dir(&self.config_dir).await.map_err(|err| + error!(target: "MAPPING", "Failed to read filters from {}: {err}", self.config_dir.display()) ) else { return; }; @@ -53,26 +61,61 @@ impl GenMapperBuilder { continue; }; if let Ok(file_type) = entry.file_type().await { - if file_type.is_file() && path.extension() == Some("toml") { - info!(target: "MAPPING", "Loading pipeline: {path}"); - if let Err(err) = self.load_pipeline(path).await { - error!(target: "MAPPING", "Failed to load pipeline: {err}"); + if file_type.is_file() { + match path.extension() { + Some("toml") => { + info!(target: "MAPPING", "Loading pipeline: {path}"); + if let Err(err) = self.load_pipeline(path).await { + error!(target: "MAPPING", "Failed to load pipeline: {err}"); + } + } + Some("js") | Some("ts") => { + info!(target: "MAPPING", "Loading filter: {path}"); + if let Err(err) = self.load_filter(path).await { + error!(target: "MAPPING", "Failed to load filter: {err}"); + } + } + _ => { + info!(target: "MAPPING", "Skipping file which type is unknown: {path}"); + } } } } } + + // Done here to ease the computation of the topics to subscribe to + // as these topics have to be known when connect is called + self.compile() } async fn load_pipeline(&mut self, file: impl AsRef) -> Result<(), LoadError> { if let Some(name) = file.as_ref().file_name() { let specs = read_to_string(file.as_ref()).await?; - let pipeline: Pipeline = toml::from_str(&specs)?; - self.pipelines.insert(name.to_string(), pipeline); + let pipeline: PipelineConfig = toml::from_str(&specs)?; + self.pipeline_specs.insert(name.to_string(), pipeline); } Ok(()) } + async fn load_filter(&mut self, file: impl AsRef) -> Result<(), LoadError> { + self.js_runtime.load_file(file.as_ref()).await?; + Ok(()) + } + + fn compile(&mut self) { + for (name, specs) in self.pipeline_specs.drain() { + match specs.compile(&self.js_runtime, &self.config_dir) { + Ok(pipeline) => { + let _ = self.pipelines.insert(name, pipeline); + } + Err(err) => { + error!(target: "MAPPING", "Failed to compile pipeline {name}: {err}") + } + } + } + } + pub fn connect( &mut self, mqtt: &mut (impl MessageSource + MessageSink), @@ -107,15 +150,22 @@ impl Builder for GenMapperBuilder { GenMapper { mqtt: self.message_box.build(), pipelines: self.pipelines, + js_runtime: self.js_runtime, } } } #[derive(thiserror::Error, Debug)] pub enum LoadError { + #[error("Script not loaded: {path}")] + ScriptNotLoaded { path: PathBuf }, + #[error(transparent)] IoError(#[from] std::io::Error), #[error(transparent)] TomlError(#[from] toml::de::Error), + + #[error(transparent)] + JsError(#[from] rquickjs::Error), } diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index 03adc975276..782fd577429 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -1,11 +1,10 @@ -use serde::Deserialize; +use crate::js_filter::JsFilter; +use crate::js_filter::JsRuntime; use tedge_mqtt_ext::MqttMessage; use tedge_mqtt_ext::TopicFilter; use time::OffsetDateTime; /// A chain of transformation of MQTT messages -#[derive(Deserialize)] -#[serde(try_from = "crate::config::PipelineConfig")] pub struct Pipeline { /// The source topics pub input_topics: TopicFilter, @@ -16,34 +15,10 @@ pub struct Pipeline { /// A message transformation stage pub struct Stage { - pub filter: Box, + pub filter: JsFilter, pub config_topics: TopicFilter, } -/// A filter process a stream of messages, producing a stream of transformed messages -/// -/// Filters are chained along pipelines, consuming MQTT messages as input -/// and producing MQTT messages as output. -/// -/// The behavior of a filter can be time related and -/// -/// Filters are dynamically configured. New partial configuration updates are sent overtime, -/// giving the opportunity for a filter to adapt its behavior. -pub trait Filter: 'static + Send + Sync { - /// Process a single message; producing zero, one or more transformed messages - fn process( - &mut self, - timestamp: OffsetDateTime, - message: &MqttMessage, - ) -> Result, FilterError>; - - /// Update the filter configuration - fn update_config(&mut self, config: &MqttMessage) -> Result<(), FilterError>; - - /// Close the current time-window; producing zero, one or more accumulated messages - fn tick(&mut self, timestamp: OffsetDateTime) -> Result, FilterError>; -} - #[derive(thiserror::Error, Debug)] pub enum FilterError { #[error("Input message cannot be processed: {0}")] @@ -62,21 +37,26 @@ impl Pipeline { topics } - pub fn update_config(&mut self, message: &MqttMessage) -> Result<(), FilterError> { + pub fn update_config( + &mut self, + js_runtime: &JsRuntime, + message: &MqttMessage, + ) -> Result<(), FilterError> { for stage in self.stages.iter_mut() { if stage.config_topics.accept(message) { - stage.filter.update_config(message)? + stage.filter.update_config(js_runtime, message)? } } Ok(()) } - pub fn process( + pub async fn process( &mut self, + js_runtime: &JsRuntime, timestamp: OffsetDateTime, message: &MqttMessage, ) -> Result, FilterError> { - self.update_config(message)?; + self.update_config(js_runtime, message)?; if !self.input_topics.accept(message) { return Ok(vec![]); } @@ -84,10 +64,8 @@ impl Pipeline { let mut messages = vec![message.clone()]; for stage in self.stages.iter_mut() { let mut transformed_messages = vec![]; - for filter_output in messages - .iter() - .map(|message| stage.filter.process(timestamp, message)) - { + for message in messages.iter() { + let filter_output = stage.filter.process(js_runtime, timestamp, message).await; transformed_messages.extend(filter_output?); } messages = transformed_messages; @@ -95,20 +73,22 @@ impl Pipeline { Ok(messages) } - pub fn tick(&mut self, timestamp: OffsetDateTime) -> Result, FilterError> { + pub async fn tick( + &mut self, + js_runtime: &JsRuntime, + timestamp: OffsetDateTime, + ) -> Result, FilterError> { let mut messages = vec![]; for stage in self.stages.iter_mut() { // Process first the messages triggered upstream by the tick let mut transformed_messages = vec![]; - for filter_output in messages - .iter() - .map(|message| stage.filter.process(timestamp, message)) - { + for message in messages.iter() { + let filter_output = stage.filter.process(js_runtime, timestamp, message).await; transformed_messages.extend(filter_output?); } // Only then process the tick - transformed_messages.extend(stage.filter.tick(timestamp)?); + transformed_messages.extend(stage.filter.tick(js_runtime, timestamp)?); // Iterate with all the messages collected at this stage messages = transformed_messages; From 97c46a8c3e19bcc69eb5ca154971a2a368ce6dfc Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 16 May 2025 17:10:48 +0200 Subject: [PATCH 04/50] Update deprecated call tempfile into_path -> keep Signed-off-by: Didier Wenzek --- crates/core/plugin_sm/src/operation_logs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core/plugin_sm/src/operation_logs.rs b/crates/core/plugin_sm/src/operation_logs.rs index 0f5ad92383d..424dfab61ba 100644 --- a/crates/core/plugin_sm/src/operation_logs.rs +++ b/crates/core/plugin_sm/src/operation_logs.rs @@ -152,7 +152,7 @@ mod tests { let unrelated_2 = create_file(log_dir.path(), "bar"); // Open the log dir - let _operation_logs = OperationLogs::try_new(log_dir.into_path().try_into().unwrap())?; + let _operation_logs = OperationLogs::try_new(log_dir.keep().try_into().unwrap())?; // Outdated logs are removed assert!(!update_log_1.exists()); From 1b59365fd3d0883ee316a47f3863e7f6495afd95 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Mon, 19 May 2025 12:25:44 +0200 Subject: [PATCH 05/50] Avoid to deserialize MQTT at each pipeline stage. Signed-off-by: Didier Wenzek --- crates/common/mqtt_channel/src/topics.rs | 9 +- .../extensions/tedge_gen_mapper/src/actor.rs | 34 ++++-- .../tedge_gen_mapper/src/js_filter.rs | 102 +++--------------- .../tedge_gen_mapper/src/pipeline.rs | 102 +++++++++++++++--- 4 files changed, 138 insertions(+), 109 deletions(-) diff --git a/crates/common/mqtt_channel/src/topics.rs b/crates/common/mqtt_channel/src/topics.rs index 35885660eee..cad07cc2e6e 100644 --- a/crates/common/mqtt_channel/src/topics.rs +++ b/crates/common/mqtt_channel/src/topics.rs @@ -117,10 +117,15 @@ impl TopicFilter { } /// Check if the given topic matches this filter pattern. - pub fn accept_topic(&self, topic: &Topic) -> bool { + pub fn accept_topic_name(&self, topic: &str) -> bool { self.patterns .iter() - .any(|pattern| rumqttc::matches(&topic.name, pattern)) + .any(|pattern| rumqttc::matches(topic, pattern)) + } + + /// Check if the given topic matches this filter pattern. + pub fn accept_topic(&self, topic: &Topic) -> bool { + self.accept_topic_name(&topic.name) } /// Check if the given message matches this filter pattern. diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index 69cc6c9b77f..084ef835573 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -1,4 +1,6 @@ use crate::js_filter::JsRuntime; +use crate::pipeline::DateTime; +use crate::pipeline::Message; use crate::pipeline::Pipeline; use async_trait::async_trait; use std::collections::HashMap; @@ -8,7 +10,6 @@ use tedge_actors::RuntimeError; use tedge_actors::Sender; use tedge_actors::SimpleMessageBox; use tedge_mqtt_ext::MqttMessage; -use time::OffsetDateTime; use tokio::time::interval; use tokio::time::Duration; use tracing::error; @@ -34,8 +35,11 @@ impl Actor for GenMapper { self.tick().await?; } message = self.mqtt.recv() => { - match message { - Some(message) => self.filter(message).await?, + match message.map(Message::try_from) { + Some(Ok(message)) => self.filter(message).await?, + Some(Err(err)) => { + error!(target: "gen-mapper", "Cannot process message: {err}"); + }, None => break, } } @@ -46,16 +50,21 @@ impl Actor for GenMapper { } impl GenMapper { - async fn filter(&mut self, message: MqttMessage) -> Result<(), RuntimeError> { - let timestamp = OffsetDateTime::now_utc(); + async fn filter(&mut self, message: Message) -> Result<(), RuntimeError> { + let timestamp = DateTime::now(); for (pipeline_id, pipeline) in self.pipelines.iter_mut() { match pipeline - .process(&self.js_runtime, timestamp, &message) + .process(&self.js_runtime, ×tamp, &message) .await { Ok(messages) => { for message in messages { - self.mqtt.send(message).await?; + match MqttMessage::try_from(message) { + Ok(message) => self.mqtt.send(message).await?, + Err(err) => { + error!(target: "gen-mapper", "{pipeline_id}: cannot send transformed message: {err}") + } + } } } Err(err) => { @@ -68,12 +77,17 @@ impl GenMapper { } async fn tick(&mut self) -> Result<(), RuntimeError> { - let timestamp = OffsetDateTime::now_utc(); + let timestamp = DateTime::now(); for (pipeline_id, pipeline) in self.pipelines.iter_mut() { - match pipeline.tick(&self.js_runtime, timestamp).await { + match pipeline.tick(&self.js_runtime, ×tamp).await { Ok(messages) => { for message in messages { - self.mqtt.send(message).await?; + match MqttMessage::try_from(message) { + Ok(message) => self.mqtt.send(message).await?, + Err(err) => { + error!(target: "gen-mapper", "{pipeline_id}: cannot send transformed message: {err}") + } + } } } Err(err) => { diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 906f3287068..1691e2d352a 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -1,5 +1,7 @@ use crate::pipeline; +use crate::pipeline::DateTime; use crate::pipeline::FilterError; +use crate::pipeline::Message; use crate::LoadError; use rquickjs::Ctx; use rquickjs::FromJs; @@ -9,22 +11,8 @@ use rquickjs::Value; use std::collections::HashMap; use std::path::Path; use std::path::PathBuf; -use tedge_mqtt_ext::MqttMessage; -use time::OffsetDateTime; use tracing::debug; -#[derive(serde::Deserialize, serde::Serialize)] -pub struct DateTime { - seconds: u64, - nanoseconds: u32, -} - -#[derive(serde::Deserialize, serde::Serialize)] -pub struct Message { - topic: String, - payload: String, -} - #[derive(Clone)] pub struct JsFilter { path: PathBuf, @@ -34,31 +22,23 @@ impl JsFilter { pub async fn process( &self, js: &JsRuntime, - timestamp: OffsetDateTime, - message: &MqttMessage, - ) -> Result, FilterError> { - debug!(target: "MAPPING", "{}: process({timestamp}, {message:?})", self.path.display()); - let timestamp = DateTime::try_from(timestamp)?; - let message = Message::try_from(message)?; - let input = (timestamp, message); - let output: Vec = js - .call_function(&self, "process", input) + timestamp: &DateTime, + message: &Message, + ) -> Result, FilterError> { + debug!(target: "MAPPING", "{}: process({timestamp:?}, {message:?})", self.path.display()); + let input = (timestamp.clone(), message.clone()); + js.call_function(&self, "process", input) .await - .map_err(error_from_js)?; - output.into_iter().map(MqttMessage::try_from).collect() + .map_err(pipeline::error_from_js) } - pub fn update_config(&self, _js: &JsRuntime, config: &MqttMessage) -> Result<(), FilterError> { + pub fn update_config(&self, _js: &JsRuntime, config: &Message) -> Result<(), FilterError> { debug!(target: "MAPPING", "{}: update_config({config:?})", self.path.display()); Ok(()) } - pub fn tick( - &self, - _js: &JsRuntime, - timestamp: OffsetDateTime, - ) -> Result, FilterError> { - debug!(target: "MAPPING", "{}: tick({timestamp})", self.path.display()); + pub fn tick(&self, _js: &JsRuntime, timestamp: &DateTime) -> Result, FilterError> { + debug!(target: "MAPPING", "{}: tick({timestamp:?})", self.path.display()); Ok(vec![]) } } @@ -107,7 +87,7 @@ impl JsRuntime { ) -> Result where for<'a> Args: rquickjs::function::IntoArgs<'a> + Send + 'a, - for<'a> Ret: rquickjs::FromJs<'a> + Send + 'a, + for<'a> Ret: FromJs<'a> + Send + 'a, { let Some(source) = self.modules.get(&module.path) else { return Err(LoadError::ScriptNotLoaded { @@ -131,47 +111,6 @@ impl JsRuntime { } } -impl TryFrom for DateTime { - type Error = FilterError; - - fn try_from(value: OffsetDateTime) -> Result { - let seconds = u64::try_from(value.unix_timestamp()).map_err(|err| { - FilterError::UnsupportedMessage(format!("failed to convert timestamp: {}", err)) - })?; - - Ok(DateTime { - seconds, - nanoseconds: value.nanosecond(), - }) - } -} - -impl TryFrom<&MqttMessage> for Message { - type Error = FilterError; - - fn try_from(message: &MqttMessage) -> Result { - let topic = message.topic.to_string(); - let payload = message - .payload_str() - .map_err(|_| { - pipeline::FilterError::UnsupportedMessage("Not an UTF8 payload".to_string()) - })? - .to_string(); - Ok(Message { topic, payload }) - } -} - -impl TryFrom for MqttMessage { - type Error = FilterError; - - fn try_from(message: Message) -> Result { - let topic = message.topic.as_str().try_into().map_err(|_| { - FilterError::UnsupportedMessage(format!("invalid topic {}", message.topic)) - })?; - Ok(MqttMessage::new(&topic, message.payload)) - } -} - impl<'js> FromJs<'js> for Message { fn from_js(_ctx: &Ctx<'js>, value: Value<'js>) -> rquickjs::Result { match value.as_object() { @@ -205,14 +144,9 @@ impl<'js> IntoJs<'js> for DateTime { } } -fn error_from_js(err: LoadError) -> FilterError { - FilterError::IncorrectSetting(format!("{}", err)) -} - #[cfg(test)] mod tests { use super::*; - use tedge_mqtt_ext::Topic; #[tokio::test] async fn identity_filter() { @@ -220,12 +154,11 @@ mod tests { let mut runtime = JsRuntime::try_new().await.unwrap(); let filter = runtime.load_js("id.js", script).unwrap(); - let topic = Topic::new_unchecked("te/main/device///m/"); - let input = MqttMessage::new(&topic, "hello world"); + let input = Message::new("te/main/device///m/", "hello world"); let output = input.clone(); assert_eq!( filter - .process(&runtime, OffsetDateTime::now_utc(), &input) + .process(&runtime, &DateTime::now(), &input) .await .unwrap(), vec![output] @@ -238,10 +171,9 @@ mod tests { let mut runtime = JsRuntime::try_new().await.unwrap(); let filter = runtime.load_js("err.js", script).unwrap(); - let topic = Topic::new_unchecked("te/main/device///m/"); - let input = MqttMessage::new(&topic, "hello world"); + let input = Message::new("te/main/device///m/", "hello world"); let error = filter - .process(&runtime, OffsetDateTime::now_utc(), &input) + .process(&runtime, &DateTime::now(), &input) .await .unwrap_err(); eprintln!("{:?}", error); diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index 782fd577429..90f756a2e09 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -1,5 +1,8 @@ use crate::js_filter::JsFilter; use crate::js_filter::JsRuntime; +use crate::LoadError; +use serde_json::json; +use serde_json::Value; use tedge_mqtt_ext::MqttMessage; use tedge_mqtt_ext::TopicFilter; use time::OffsetDateTime; @@ -19,6 +22,18 @@ pub struct Stage { pub config_topics: TopicFilter, } +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Eq, PartialEq)] +pub struct DateTime { + pub(crate) seconds: u64, + pub(crate) nanoseconds: u32, +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Eq, PartialEq)] +pub struct Message { + pub(crate) topic: String, + pub(crate) payload: String, +} + #[derive(thiserror::Error, Debug)] pub enum FilterError { #[error("Input message cannot be processed: {0}")] @@ -38,12 +53,12 @@ impl Pipeline { } pub fn update_config( - &mut self, + &self, js_runtime: &JsRuntime, - message: &MqttMessage, + message: &Message, ) -> Result<(), FilterError> { - for stage in self.stages.iter_mut() { - if stage.config_topics.accept(message) { + for stage in self.stages.iter() { + if stage.config_topics.accept_topic_name(&message.topic) { stage.filter.update_config(js_runtime, message)? } } @@ -53,16 +68,16 @@ impl Pipeline { pub async fn process( &mut self, js_runtime: &JsRuntime, - timestamp: OffsetDateTime, - message: &MqttMessage, - ) -> Result, FilterError> { + timestamp: &DateTime, + message: &Message, + ) -> Result, FilterError> { self.update_config(js_runtime, message)?; - if !self.input_topics.accept(message) { + if !self.input_topics.accept_topic_name(&message.topic) { return Ok(vec![]); } let mut messages = vec![message.clone()]; - for stage in self.stages.iter_mut() { + for stage in self.stages.iter() { let mut transformed_messages = vec![]; for message in messages.iter() { let filter_output = stage.filter.process(js_runtime, timestamp, message).await; @@ -76,10 +91,10 @@ impl Pipeline { pub async fn tick( &mut self, js_runtime: &JsRuntime, - timestamp: OffsetDateTime, - ) -> Result, FilterError> { + timestamp: &DateTime, + ) -> Result, FilterError> { let mut messages = vec![]; - for stage in self.stages.iter_mut() { + for stage in self.stages.iter() { // Process first the messages triggered upstream by the tick let mut transformed_messages = vec![]; for message in messages.iter() { @@ -96,3 +111,66 @@ impl Pipeline { Ok(messages) } } + +impl DateTime { + pub fn now() -> Self { + DateTime::try_from(OffsetDateTime::now_utc()).unwrap() + } +} + +impl TryFrom for DateTime { + type Error = FilterError; + + fn try_from(value: OffsetDateTime) -> Result { + let seconds = u64::try_from(value.unix_timestamp()).map_err(|err| { + FilterError::UnsupportedMessage(format!("failed to convert timestamp: {}", err)) + })?; + + Ok(DateTime { + seconds, + nanoseconds: value.nanosecond(), + }) + } +} + +impl Message { + #[cfg(test)] + pub(crate) fn new(topic: &str, payload: &str) -> Self { + Message { + topic: topic.to_string(), + payload: payload.to_string(), + } + } + + pub fn json(&self) -> Value { + json!({"topic": self.topic, "payload": self.payload}) + } +} + +impl TryFrom for Message { + type Error = FilterError; + + fn try_from(message: MqttMessage) -> Result { + let topic = message.topic.to_string(); + let payload = message + .payload_str() + .map_err(|_| FilterError::UnsupportedMessage("Not an UTF8 payload".to_string()))? + .to_string(); + Ok(Message { topic, payload }) + } +} + +impl TryFrom for MqttMessage { + type Error = FilterError; + + fn try_from(message: Message) -> Result { + let topic = message.topic.as_str().try_into().map_err(|_| { + FilterError::UnsupportedMessage(format!("invalid topic {}", message.topic)) + })?; + Ok(MqttMessage::new(&topic, message.payload)) + } +} + +pub fn error_from_js(err: LoadError) -> FilterError { + FilterError::IncorrectSetting(format!("{}", err)) +} From 046db7ef43fe8abad7d6fd31aad0a8fd14497c65 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Mon, 19 May 2025 16:17:10 +0200 Subject: [PATCH 06/50] Add JS example: drop stragglers Signed-off-by: Didier Wenzek --- .../pipelines/add_timestamp.js | 4 ++- .../pipelines/drop_stragglers.js | 26 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js diff --git a/crates/extensions/tedge_gen_mapper/pipelines/add_timestamp.js b/crates/extensions/tedge_gen_mapper/pipelines/add_timestamp.js index 2c58927b104..f387141bfe3 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/add_timestamp.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/add_timestamp.js @@ -1,6 +1,8 @@ export function process (timestamp, message) { let payload = JSON.parse(message.payload) - payload.time = Number(timestamp.seconds) + (timestamp.nanoseconds / 1e9) + if (!payload.time) { + payload.time = timestamp.seconds + (timestamp.nanoseconds / 1e9) + } return [{ topic: message.topic, diff --git a/crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js b/crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js new file mode 100644 index 00000000000..0b50afcfb46 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js @@ -0,0 +1,26 @@ +// Reject any message that is too old, too new or with no timestamp +export function process (timestamp, message, config) { + let payload = JSON.parse(message.payload) + let msg_time = payload.time + if (!msg_time) { + return [] + } + if (!config) { + config = {} + } + + let msg_timestamp = msg_time + if (typeof(msg_time) === "string") { + msg_timestamp = Date.parse(msg_time) / 1e3 + } + + let time = timestamp.seconds + (timestamp.nanoseconds / 1e9) + let max = time + (config?.max_advance || 1); + let min = time - (config?.max_delay || 10); + + if (min <= msg_timestamp && msg_timestamp <= max) { + return [message] + } else { + return [{"topic":" te/error", "payload":`straggler rejected on ${message.topic} with time=${msg_timestamp}`}] + } +} From cbc345a8f2aa565c0fe309d0b1909a22236f4d0e Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Mon, 19 May 2025 17:42:20 +0200 Subject: [PATCH 07/50] Add ability to configure each stages Signed-off-by: Didier Wenzek --- .../pipelines/drop_stragglers.js | 5 +- .../pipelines/measurements.toml | 1 + .../extensions/tedge_gen_mapper/src/config.rs | 6 +- .../tedge_gen_mapper/src/js_filter.rs | 67 ++++++++++++++++++- 4 files changed, 71 insertions(+), 8 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js b/crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js index 0b50afcfb46..5ac019db494 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js @@ -5,9 +5,6 @@ export function process (timestamp, message, config) { if (!msg_time) { return [] } - if (!config) { - config = {} - } let msg_timestamp = msg_time if (typeof(msg_time) === "string") { @@ -21,6 +18,6 @@ export function process (timestamp, message, config) { if (min <= msg_timestamp && msg_timestamp <= max) { return [message] } else { - return [{"topic":" te/error", "payload":`straggler rejected on ${message.topic} with time=${msg_timestamp}`}] + return [{"topic":" te/error", "payload":`straggler rejected on ${message.topic} with time=${msg_timestamp} at ${time}`}] } } diff --git a/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml b/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml index 0d5e06d55a0..fea27e58da2 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml +++ b/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml @@ -2,5 +2,6 @@ input_topics = ["te/+/+/+/+/m/+"] stages = [ { filter = "add_timestamp.js" }, + { filter = "drop_stragglers.js", config = { max_delay = 60 } }, { filter = "te_to_c8y.js" } ] diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index 694fb901451..cdba92cf6a3 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -4,6 +4,7 @@ use crate::pipeline::Stage; use crate::LoadError; use camino::Utf8PathBuf; use serde::Deserialize; +use serde_json::Value; use std::path::Path; use tedge_mqtt_ext::TopicFilter; @@ -17,6 +18,9 @@ pub struct PipelineConfig { pub struct StageConfig { filter: FilterSpec, + #[serde(default)] + config: Option, + #[serde(default)] config_topics: Vec, } @@ -61,7 +65,7 @@ impl StageConfig { FilterSpec::JavaScript(path) if path.is_absolute() => path.into(), FilterSpec::JavaScript(path) => config_dir.join(path), }; - let filter = js_runtime.loaded_module(path)?; + let filter = js_runtime.loaded_module(path)?.with_config(self.config); let config_topics = topic_filters(&self.config_topics)?; Ok(Stage { filter, diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 1691e2d352a..de0edafcf0e 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -16,9 +16,31 @@ use tracing::debug; #[derive(Clone)] pub struct JsFilter { path: PathBuf, + config: JsonValue, } +#[derive(Clone, Default)] +pub struct JsonValue(serde_json::Value); + impl JsFilter { + pub fn new(path: PathBuf) -> Self { + JsFilter { + path, + config: JsonValue::default(), + } + } + + pub fn with_config(self, config: Option) -> Self { + if let Some(config) = config { + Self { + config: JsonValue(config), + ..self + } + } else { + self + } + } + pub async fn process( &self, js: &JsRuntime, @@ -26,7 +48,7 @@ impl JsFilter { message: &Message, ) -> Result, FilterError> { debug!(target: "MAPPING", "{}: process({timestamp:?}, {message:?})", self.path.display()); - let input = (timestamp.clone(), message.clone()); + let input = (timestamp.clone(), message.clone(), self.config.clone()); js.call_function(&self, "process", input) .await .map_err(pipeline::error_from_js) @@ -69,13 +91,13 @@ impl JsRuntime { ) -> Result { let path = path.as_ref().to_path_buf(); self.modules.insert(path.clone(), source.into()); - Ok(JsFilter { path }) + Ok(JsFilter::new(path)) } pub fn loaded_module(&self, path: PathBuf) -> Result { match self.modules.get(&path) { None => Err(LoadError::ScriptNotLoaded { path }), - Some(_) => Ok(JsFilter { path }), + Some(_) => Ok(JsFilter::new(path)), } } @@ -144,6 +166,45 @@ impl<'js> IntoJs<'js> for DateTime { } } +impl<'js> IntoJs<'js> for JsonValue { + fn into_js(self, ctx: &Ctx<'js>) -> rquickjs::Result> { + match self.0 { + serde_json::Value::Null => Ok(Value::new_null(ctx.clone())), + serde_json::Value::Bool(value) => Ok(Value::new_bool(ctx.clone(), value)), + serde_json::Value::Number(value) => { + if let Some(n) = value.as_i64() { + if let Ok(n) = i32::try_from(n) { + return Ok(Value::new_int(ctx.clone(), n)); + } + } + if let Some(f) = value.as_f64() { + return Ok(Value::new_float(ctx.clone(), f)); + } + let nan = rquickjs::String::from_str(ctx.clone(), "NaN")?; + Ok(nan.into_value()) + } + serde_json::Value::String(value) => { + let string = rquickjs::String::from_str(ctx.clone(), &value)?; + Ok(string.into_value()) + } + serde_json::Value::Array(values) => { + let array = rquickjs::Array::new(ctx.clone())?; + for (i, value) in values.into_iter().enumerate() { + array.set(i, JsonValue(value))?; + } + Ok(array.into_value()) + } + serde_json::Value::Object(values) => { + let object = rquickjs::Object::new(ctx.clone())?; + for (key, value) in values.into_iter() { + object.set(key, JsonValue(value))?; + } + Ok(object.into_value()) + } + } + } +} + #[cfg(test)] mod tests { use super::*; From c86ee6cb16e70b54cea4ae0061b0d175c7c3454d Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 20 May 2025 10:47:15 +0200 Subject: [PATCH 08/50] Add JS example: collectd pipeline Signed-off-by: Didier Wenzek --- .../tedge_gen_mapper/pipelines/collectd-to-te.js | 14 ++++++++++++++ .../pipelines/group_by_timestamp.js | 16 ++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/group_by_timestamp.js diff --git a/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js b/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js new file mode 100644 index 00000000000..e5aa71c0540 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js @@ -0,0 +1,14 @@ +export function process (timestamp, message, config) { + let groups = message.topic.split( '/') + let data = message.payload.split(':') + + let group = groups[2] + let measurement = groups[3] + let time = data[0] + let value = data[1] + + return [ { + topic: config.topic || "te/device/main///m/collectd", + payload: `{"time": ${time}, "${group}": {"${measurement}": ${value}}}` + }] +} \ No newline at end of file diff --git a/crates/extensions/tedge_gen_mapper/pipelines/group_by_timestamp.js b/crates/extensions/tedge_gen_mapper/pipelines/group_by_timestamp.js new file mode 100644 index 00000000000..ffa282ff475 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/group_by_timestamp.js @@ -0,0 +1,16 @@ +// Demonstrate that messages can be delayed +export function process (timestamp, message, config) { + if ( typeof process.batch == 'undefined' ) { + process.batch = []; + } + + let len = process.batch.push(message) + let batch_len = config.batch_len || 4 + if (len < batch_len) { + return [] + } + + let batch = process.batch + process.batch = [] + return batch +} \ No newline at end of file From c56c290ee95baa371983347e010fc2d510c549f3 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 20 May 2025 14:00:03 +0200 Subject: [PATCH 09/50] Add JS example: thin-edge JSON to c8y JSON Signed-off-by: Didier Wenzek --- .../tedge_gen_mapper/pipelines/te_to_c8y.js | 77 ++++++++++++++++++- 1 file changed, 73 insertions(+), 4 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js b/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js index 9c7583a8bd3..fc2968db926 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js @@ -1,6 +1,75 @@ -export function process(t,msg) { - msg.topic = "te/error" - return [msg] -} +/// Transform: +/// +/// ``` +/// [te/device/main///m/example] { +/// "time": "2020-10-15T05:30:47+00:00", +/// "temperature": 25, +/// "location": { +/// "latitude": 32.54, +/// "longitude": -117.67, +/// "altitude": 98.6 +/// }, +/// "pressure": 98 +/// } +/// ``` +/// +/// into +/// +/// ``` +/// [c8y/measurement/measurements/create] { +/// "time": "2020-10-15T05:30:47Z", +/// "type": "example", +/// "temperature": { +/// "temperature": { +/// "value": 25 +/// } +/// }, +/// "location": { +/// "latitude": { +/// "value": 32.54 +/// }, +/// "longitude": { +/// "value": -117.67 +/// }, +/// "altitude": { +/// "value": 98.6 +/// } +/// }, +/// "pressure": { +/// "pressure": { +/// "value": 98 +/// } +/// } +/// } +/// ``` +export function process(t,message) { + let topic_parts = message.topic.split( '/') + let type = topic_parts[6] + let payload = JSON.parse(message.payload) + + let c8y_msg = { + type: type + } + for (let [k, v] of Object.entries(payload)) { + if (k === "time") { + let fragment = { time: v } + Object.assign(c8y_msg, fragment) + } + else if (typeof(v) === "number") { + let fragment = { [k]: { [k]: v } } + Object.assign(c8y_msg, fragment) + } else for (let [sub_k, sub_v] of Object.entries(v)) { + if (typeof(sub_v) === "number") { + let fragment = { [k]: { [sub_k]: sub_v } } + Object.assign(c8y_msg, fragment) + } + } + } + + return [{ + topic: "c8y/measurement/measurements/create", + payload: JSON.stringify(c8y_msg) + }] +} From 5fa5cdd34446f334f7cd1ce0a85cfbc4a3e3f342 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 20 May 2025 15:47:59 +0200 Subject: [PATCH 10/50] Fix tests broken by cargo update Signed-off-by: Didier Wenzek --- .../fixtures/invalid/reject_invalid_timestamp.expected_error | 2 +- .../fixtures/invalid/reject_partial_timestamp.expected_error | 2 +- .../reject_timestamp_missing_time_separator.expected_error | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/core/tedge_api/tests/fixtures/invalid/reject_invalid_timestamp.expected_error b/crates/core/tedge_api/tests/fixtures/invalid/reject_invalid_timestamp.expected_error index 081cbde607d..422d9e8e533 100644 --- a/crates/core/tedge_api/tests/fixtures/invalid/reject_invalid_timestamp.expected_error +++ b/crates/core/tedge_api/tests/fixtures/invalid/reject_invalid_timestamp.expected_error @@ -1,4 +1,4 @@ -Invalid JSON: Invalid ISO8601 timestamp (expected YYYY-MM-DDThh:mm:ss.sss.±hh:mm): "2013-06-22 3am": a character literal was not valid at line 2 column 27: `", +Invalid JSON: Invalid ISO8601 timestamp (expected YYYY-MM-DDThh:mm:ss.sss.±hh:mm): "2013-06-22 3am": the 'hour' component could not be parsed at line 2 column 27: `", "pressure": 220 } ` \ No newline at end of file diff --git a/crates/core/tedge_api/tests/fixtures/invalid/reject_partial_timestamp.expected_error b/crates/core/tedge_api/tests/fixtures/invalid/reject_partial_timestamp.expected_error index 54a5bc5a042..faac58c9fa6 100644 --- a/crates/core/tedge_api/tests/fixtures/invalid/reject_partial_timestamp.expected_error +++ b/crates/core/tedge_api/tests/fixtures/invalid/reject_partial_timestamp.expected_error @@ -1,4 +1,4 @@ -Invalid JSON: Invalid ISO8601 timestamp (expected YYYY-MM-DDThh:mm:ss.sss.±hh:mm): "2013-06-22": a character literal was not valid at line 2 column 23: `", +Invalid JSON: Invalid ISO8601 timestamp (expected YYYY-MM-DDThh:mm:ss.sss.±hh:mm): "2013-06-22": the 'separator' component could not be parsed at line 2 column 23: `", "pressure": 220 } ` \ No newline at end of file diff --git a/crates/core/tedge_api/tests/fixtures/invalid/reject_timestamp_missing_time_separator.expected_error b/crates/core/tedge_api/tests/fixtures/invalid/reject_timestamp_missing_time_separator.expected_error index 7a92d95256a..bf76a12e79a 100644 --- a/crates/core/tedge_api/tests/fixtures/invalid/reject_timestamp_missing_time_separator.expected_error +++ b/crates/core/tedge_api/tests/fixtures/invalid/reject_timestamp_missing_time_separator.expected_error @@ -1,3 +1,3 @@ -Invalid JSON: Invalid ISO8601 timestamp (expected YYYY-MM-DDThh:mm:ss.sss.±hh:mm): "2013-06-2217:03:14.000658767+02:00": a character literal was not valid at line 2 column 47: `" +Invalid JSON: Invalid ISO8601 timestamp (expected YYYY-MM-DDThh:mm:ss.sss.±hh:mm): "2013-06-2217:03:14.000658767+02:00": the 'hour' component could not be parsed at line 2 column 47: `" } ` \ No newline at end of file From 09887c53b86aa43ffea0c1fde1e98ebd16af8a39 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 20 May 2025 17:56:25 +0200 Subject: [PATCH 11/50] Update JS filter config using MQTT Signed-off-by: Didier Wenzek --- .../pipelines/measurements.toml | 2 +- .../tedge_gen_mapper/pipelines/te_to_c8y.js | 53 ++++++++++++++- .../extensions/tedge_gen_mapper/src/config.rs | 4 +- .../tedge_gen_mapper/src/js_filter.rs | 65 ++++++++++++++++++- .../tedge_gen_mapper/src/pipeline.rs | 10 +-- 5 files changed, 122 insertions(+), 12 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml b/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml index fea27e58da2..1edde2696e6 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml +++ b/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml @@ -3,5 +3,5 @@ input_topics = ["te/+/+/+/+/m/+"] stages = [ { filter = "add_timestamp.js" }, { filter = "drop_stragglers.js", config = { max_delay = 60 } }, - { filter = "te_to_c8y.js" } + { filter = "te_to_c8y.js", meta_topics = ["te/+/+/+/+/m/+/meta"] } ] diff --git a/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js b/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js index fc2968db926..ae2d8f157c1 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js @@ -42,7 +42,7 @@ /// } /// } /// ``` -export function process(t,message) { +export function process(t, message, config) { let topic_parts = message.topic.split( '/') let type = topic_parts[6] let payload = JSON.parse(message.payload) @@ -51,17 +51,26 @@ export function process(t,message) { type: type } + let meta = (config || {})[`${message.topic}/meta`] || {} + for (let [k, v] of Object.entries(payload)) { + let k_meta = (meta || {})[k] || {} if (k === "time") { let fragment = { time: v } Object.assign(c8y_msg, fragment) - } else if (typeof(v) === "number") { + if (Object.keys(k_meta).length>0) { + v = { value: v, ...k_meta } + } let fragment = { [k]: { [k]: v } } Object.assign(c8y_msg, fragment) } else for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_k_meta = k_meta[sub_k] if (typeof(sub_v) === "number") { + if (sub_k_meta) { + sub_v = { value: sub_v, ...sub_k_meta } + } let fragment = { [k]: { [sub_k]: sub_v } } Object.assign(c8y_msg, fragment) } @@ -73,3 +82,43 @@ export function process(t,message) { payload: JSON.stringify(c8y_msg) }] } + +/// Update the config with measurement metadata. +/// +/// These metadata are expected to have the same shape of the actual values. +/// +/// ``` +/// [te/device/main///m/example/meta] { "temperature": { "unit": "°C" }} +/// ``` +/// +/// and: +/// ``` +/// [te/device/main///m/example] { "temperature": { "unit": 23 }} +/// ``` +/// +/// will be merged by the process function into: +/// ``` +/// [c8y/measurement/measurements/create] { +/// "type": "example", +/// "temperature": { +/// "temperature": { +/// "value": 23, +/// "unit": "°C" +/// } +/// } +/// } +/// ``` +export function update_config(message, config) { + let type = message.topic + let metadata = JSON.parse(message.payload) + + let fragment = { + [type]: metadata + } + if (!config) { + config = {} + } + Object.assign(config, fragment) + + return config +} diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index cdba92cf6a3..fd6db236715 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -22,7 +22,7 @@ pub struct StageConfig { config: Option, #[serde(default)] - config_topics: Vec, + meta_topics: Vec, } #[derive(Deserialize)] @@ -66,7 +66,7 @@ impl StageConfig { FilterSpec::JavaScript(path) => config_dir.join(path), }; let filter = js_runtime.loaded_module(path)?.with_config(self.config); - let config_topics = topic_filters(&self.config_topics)?; + let config_topics = topic_filters(&self.meta_topics)?; Ok(Stage { filter, config_topics, diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index de0edafcf0e..c2fa1f71876 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -41,6 +41,14 @@ impl JsFilter { } } + /// Process a message returning zero, one or more messages + /// + /// The "process" function of the JS module is passed 3 arguments + /// - the current timestamp + /// - the message to be transformed + /// - the filter config (as configured for the pipeline stage, possibly updated by update_config messages) + /// + /// The returned value is expected to be an array of messages. pub async fn process( &self, js: &JsRuntime, @@ -54,8 +62,25 @@ impl JsFilter { .map_err(pipeline::error_from_js) } - pub fn update_config(&self, _js: &JsRuntime, config: &Message) -> Result<(), FilterError> { - debug!(target: "MAPPING", "{}: update_config({config:?})", self.path.display()); + /// Update the filter config using a metadata message + /// + /// The "update_config" function of the JS module is passed 2 arguments + /// - the message + /// - the current filter config + /// + /// The value returned by this function is used as the updated filter config + pub async fn update_config( + &mut self, + js: &JsRuntime, + message: &Message, + ) -> Result<(), FilterError> { + debug!(target: "MAPPING", "{}: update_config({message:?})", self.path.display()); + let input = (message.clone(), self.config.clone()); + let config = js + .call_function(&self, "update_config", input) + .await + .map_err(pipeline::error_from_js)?; + self.config = config; Ok(()) } @@ -205,6 +230,42 @@ impl<'js> IntoJs<'js> for JsonValue { } } +impl<'js> FromJs<'js> for JsonValue { + fn from_js(_ctx: &Ctx<'js>, value: Value<'js>) -> rquickjs::Result { + if let Some(b) = value.as_bool() { + return Ok(JsonValue(serde_json::Value::Bool(b))); + } + if let Some(n) = value.as_int() { + return Ok(JsonValue(serde_json::Value::Number(n.into()))); + } + if let Some(n) = value.as_float() { + let js_n = serde_json::Number::from_f64(n) + .map(serde_json::Value::Number) + .unwrap_or(serde_json::Value::Null); + return Ok(JsonValue(js_n)); + } + if let Some(string) = value.as_string() { + return Ok(JsonValue(serde_json::Value::String(string.to_string()?))); + } + if let Some(array) = value.as_array() { + let array: rquickjs::Result> = array.iter().collect(); + let array = array?.into_iter().map(|v| v.0).collect(); + return Ok(JsonValue(serde_json::Value::Array(array))); + } + if let Some(object) = value.as_object() { + let mut js_object = serde_json::Map::new(); + for key in object.keys::().flatten() { + if let Ok(JsonValue(v)) = object.get(&key) { + js_object.insert(key, v.clone()); + } + } + return Ok(JsonValue(serde_json::Value::Object(js_object))); + } + + Ok(JsonValue(serde_json::Value::Null)) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index 90f756a2e09..787f87207ca 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -52,14 +52,14 @@ impl Pipeline { topics } - pub fn update_config( - &self, + pub async fn update_config( + &mut self, js_runtime: &JsRuntime, message: &Message, ) -> Result<(), FilterError> { - for stage in self.stages.iter() { + for stage in self.stages.iter_mut() { if stage.config_topics.accept_topic_name(&message.topic) { - stage.filter.update_config(js_runtime, message)? + stage.filter.update_config(js_runtime, message).await? } } Ok(()) @@ -71,7 +71,7 @@ impl Pipeline { timestamp: &DateTime, message: &Message, ) -> Result, FilterError> { - self.update_config(js_runtime, message)?; + self.update_config(js_runtime, message).await?; if !self.input_topics.accept_topic_name(&message.topic) { return Ok(vec![]); } From 42a3fffe2498814226880df09b152d986f4171a3 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 20 May 2025 18:59:51 +0200 Subject: [PATCH 12/50] Filters can defer messages up to the end of a time window Signed-off-by: Didier Wenzek --- .../tedge_gen_mapper/pipelines/collectd.toml | 2 +- .../pipelines/group_by_timestamp.js | 22 +++++++------- .../extensions/tedge_gen_mapper/src/actor.rs | 2 +- .../extensions/tedge_gen_mapper/src/config.rs | 8 ++++- .../tedge_gen_mapper/src/js_filter.rs | 30 +++++++++++++++++-- .../tedge_gen_mapper/src/pipeline.rs | 6 +++- 6 files changed, 52 insertions(+), 18 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml b/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml index 2501140d5e0..44feb5b3323 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml +++ b/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml @@ -2,5 +2,5 @@ input_topics = ["collectd/+/+/+"] stages = [ { filter = "collectd-to-te.js" }, - { filter = "group_by_timestamp.js" } + { filter = "group_by_timestamp.js", tick_every_seconds = 3 } ] diff --git a/crates/extensions/tedge_gen_mapper/pipelines/group_by_timestamp.js b/crates/extensions/tedge_gen_mapper/pipelines/group_by_timestamp.js index ffa282ff475..77b061ee1e5 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/group_by_timestamp.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/group_by_timestamp.js @@ -1,16 +1,14 @@ -// Demonstrate that messages can be delayed -export function process (timestamp, message, config) { - if ( typeof process.batch == 'undefined' ) { - process.batch = []; - } +class State { + static batch = [] +} - let len = process.batch.push(message) - let batch_len = config.batch_len || 4 - if (len < batch_len) { - return [] - } +export function process (timestamp, message) { + State.batch.push(message) + return [] +} - let batch = process.batch - process.batch = [] +export function tick() { + let batch = State.batch + State.batch = [] return batch } \ No newline at end of file diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index 084ef835573..cb2584eb34f 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -27,7 +27,7 @@ impl Actor for GenMapper { } async fn run(mut self) -> Result<(), RuntimeError> { - let mut interval = interval(Duration::from_secs(5)); + let mut interval = interval(Duration::from_secs(1)); loop { tokio::select! { diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index fd6db236715..551e6707247 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -21,6 +21,9 @@ pub struct StageConfig { #[serde(default)] config: Option, + #[serde(default)] + tick_every_seconds: u64, + #[serde(default)] meta_topics: Vec, } @@ -65,7 +68,10 @@ impl StageConfig { FilterSpec::JavaScript(path) if path.is_absolute() => path.into(), FilterSpec::JavaScript(path) => config_dir.join(path), }; - let filter = js_runtime.loaded_module(path)?.with_config(self.config); + let filter = js_runtime + .loaded_module(path)? + .with_config(self.config) + .with_tick_every_seconds(self.tick_every_seconds); let config_topics = topic_filters(&self.meta_topics)?; Ok(Stage { filter, diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index c2fa1f71876..f0a78cf6934 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -17,6 +17,7 @@ use tracing::debug; pub struct JsFilter { path: PathBuf, config: JsonValue, + tick_every_seconds: u64, } #[derive(Clone, Default)] @@ -27,6 +28,7 @@ impl JsFilter { JsFilter { path, config: JsonValue::default(), + tick_every_seconds: 0, } } @@ -41,6 +43,13 @@ impl JsFilter { } } + pub fn with_tick_every_seconds(self, tick_every_seconds: u64) -> Self { + Self { + tick_every_seconds, + ..self + } + } + /// Process a message returning zero, one or more messages /// /// The "process" function of the JS module is passed 3 arguments @@ -84,9 +93,26 @@ impl JsFilter { Ok(()) } - pub fn tick(&self, _js: &JsRuntime, timestamp: &DateTime) -> Result, FilterError> { + /// Trigger the tick function of the JS module + /// + /// The "tick" function is passed 2 arguments + /// - the current timestamp + /// - the current filter config + /// + /// Return zero, one or more messages + pub async fn tick( + &self, + js: &JsRuntime, + timestamp: &DateTime, + ) -> Result, FilterError> { + if !timestamp.tick_now(self.tick_every_seconds) { + return Ok(vec![]); + } debug!(target: "MAPPING", "{}: tick({timestamp:?})", self.path.display()); - Ok(vec![]) + let input = (timestamp.clone(), self.config.clone()); + js.call_function(&self, "tick", input) + .await + .map_err(pipeline::error_from_js) } } diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index 787f87207ca..4f5c8bb84fd 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -103,7 +103,7 @@ impl Pipeline { } // Only then process the tick - transformed_messages.extend(stage.filter.tick(js_runtime, timestamp)?); + transformed_messages.extend(stage.filter.tick(js_runtime, timestamp).await?); // Iterate with all the messages collected at this stage messages = transformed_messages; @@ -116,6 +116,10 @@ impl DateTime { pub fn now() -> Self { DateTime::try_from(OffsetDateTime::now_utc()).unwrap() } + + pub fn tick_now(&self, tick_every_seconds: u64) -> bool { + tick_every_seconds != 0 && (self.seconds % tick_every_seconds == 0) + } } impl TryFrom for DateTime { From b61f9d09b72f10abb86a3a244070a09fd0558e71 Mon Sep 17 00:00:00 2001 From: James Rhodes Date: Wed, 21 May 2025 16:10:39 +0100 Subject: [PATCH 13/50] Tedge-gen-mapper dynamically reloads pipelines and filters Signed-off-by: James Rhodes --- Cargo.lock | 1 + crates/core/tedge_mapper/src/gen/mod.rs | 4 + crates/extensions/tedge_gen_mapper/Cargo.toml | 1 + .../extensions/tedge_gen_mapper/src/actor.rs | 93 +++++++++++++++++-- .../extensions/tedge_gen_mapper/src/config.rs | 2 + .../tedge_gen_mapper/src/js_filter.rs | 4 + crates/extensions/tedge_gen_mapper/src/lib.rs | 38 ++++++-- .../tedge_gen_mapper/src/pipeline.rs | 3 + 8 files changed, 129 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ca9cdc35845..19317544fef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4941,6 +4941,7 @@ dependencies = [ "serde", "serde_json", "tedge_actors", + "tedge_file_system_ext", "tedge_mqtt_ext", "thiserror 1.0.69", "time", diff --git a/crates/core/tedge_mapper/src/gen/mod.rs b/crates/core/tedge_mapper/src/gen/mod.rs index 091fb47542b..acdaa3a6961 100644 --- a/crates/core/tedge_mapper/src/gen/mod.rs +++ b/crates/core/tedge_mapper/src/gen/mod.rs @@ -1,6 +1,7 @@ use crate::core::mapper::start_basic_actors; use crate::TEdgeComponent; use tedge_config::TEdgeConfig; +use tedge_file_system_ext::FsWatchActorBuilder; use tedge_gen_mapper::GenMapperBuilder; pub struct GenMapper; @@ -15,12 +16,15 @@ impl TEdgeComponent for GenMapper { let (mut runtime, mut mqtt_actor) = start_basic_actors("tedge-gen-mapper", &tedge_config).await?; + let mut fs_actor = FsWatchActorBuilder::new(); let mut gen_mapper = GenMapperBuilder::try_new("/etc/tedge/gen-mapper").await?; gen_mapper.load().await; gen_mapper.connect(&mut mqtt_actor); + gen_mapper.connect_fs(&mut fs_actor); runtime.spawn(gen_mapper).await?; runtime.spawn(mqtt_actor).await?; + runtime.spawn(fs_actor).await?; runtime.run_to_completion().await?; Ok(()) } diff --git a/crates/extensions/tedge_gen_mapper/Cargo.toml b/crates/extensions/tedge_gen_mapper/Cargo.toml index 32b302357e3..80c9b9cbb9b 100644 --- a/crates/extensions/tedge_gen_mapper/Cargo.toml +++ b/crates/extensions/tedge_gen_mapper/Cargo.toml @@ -15,6 +15,7 @@ rquickjs = { workspace = true, features = ["futures","parallel"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } tedge_actors = { workspace = true } +tedge_file_system_ext = { workspace = true } tedge_mqtt_ext = { workspace = true } thiserror = { workspace = true } time = { workspace = true } diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index cb2584eb34f..8506cc2bc99 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -1,23 +1,31 @@ +use crate::config::PipelineConfig; use crate::js_filter::JsRuntime; use crate::pipeline::DateTime; use crate::pipeline::Message; use crate::pipeline::Pipeline; +use crate::InputMessage; +use crate::OutputMessage; use async_trait::async_trait; +use camino::Utf8PathBuf; use std::collections::HashMap; +use std::path::PathBuf; use tedge_actors::Actor; use tedge_actors::MessageReceiver; use tedge_actors::RuntimeError; use tedge_actors::Sender; use tedge_actors::SimpleMessageBox; +use tedge_file_system_ext::FsWatchEvent; use tedge_mqtt_ext::MqttMessage; use tokio::time::interval; use tokio::time::Duration; use tracing::error; +use tracing::info; pub struct GenMapper { - pub(super) mqtt: SimpleMessageBox, + pub(super) messages: SimpleMessageBox, pub(super) pipelines: HashMap, pub(super) js_runtime: JsRuntime, + pub(super) config_dir: PathBuf, } #[async_trait] @@ -34,11 +42,26 @@ impl Actor for GenMapper { _ = interval.tick() => { self.tick().await?; } - message = self.mqtt.recv() => { - match message.map(Message::try_from) { - Some(Ok(message)) => self.filter(message).await?, - Some(Err(err)) => { - error!(target: "gen-mapper", "Cannot process message: {err}"); + message = self.messages.recv() => { + match message { + Some(InputMessage::MqttMessage(message)) => match Message::try_from(message) { + Ok(message) => self.filter(message).await?, + Err(err) => { + error!(target: "gen-mapper", "Cannot process message: {err}"); + } + }, + Some(InputMessage::FsWatchEvent(FsWatchEvent::Modified(path))) => { + let Ok(path) = Utf8PathBuf::try_from(path) else { + continue; + }; + if matches!(path.extension(), Some("js" | "ts")) { + self.reload_filter(path).await; + } else if path.extension() == Some("toml") { + self.reload_pipeline(path).await; + } + }, + Some(InputMessage::FsWatchEvent(e)) => { + tracing::warn!("TODO do something with {e:?}") }, None => break, } @@ -50,6 +73,52 @@ impl Actor for GenMapper { } impl GenMapper { + async fn reload_filter(&mut self, path: Utf8PathBuf) { + for pipeline in self.pipelines.values_mut() { + for stage in &mut pipeline.stages { + if stage.filter.path() == path { + match self.js_runtime.load_file(&path).await { + Ok(filter) => { + info!("Reloaded filter {path}"); + stage.filter = filter + } + Err(e) => { + error!("Failed to reload filter {path}: {e}"); + return; + } + } + } + } + } + } + + async fn reload_pipeline(&mut self, path: Utf8PathBuf) { + for pipeline in self.pipelines.values_mut() { + if pipeline.source == path { + let Ok(source) = tokio::fs::read_to_string(&path).await else { + error!("Failed to read updated filter {path}"); + break; + }; + let config: PipelineConfig = match toml::from_str(&source) { + Ok(config) => config, + Err(e) => { + error!("Failed to parse toml for updated filter {path}: {e}"); + break; + } + }; + match config.compile(&self.js_runtime, &self.config_dir, path.clone()) { + Ok(p) => { + *pipeline = p; + info!("Reloaded pipeline {path}"); + } + Err(e) => { + error!("Failed to load updated pipeline {path}: {e}") + } + }; + } + } + } + async fn filter(&mut self, message: Message) -> Result<(), RuntimeError> { let timestamp = DateTime::now(); for (pipeline_id, pipeline) in self.pipelines.iter_mut() { @@ -60,7 +129,11 @@ impl GenMapper { Ok(messages) => { for message in messages { match MqttMessage::try_from(message) { - Ok(message) => self.mqtt.send(message).await?, + Ok(message) => { + self.messages + .send(OutputMessage::MqttMessage(message)) + .await? + } Err(err) => { error!(target: "gen-mapper", "{pipeline_id}: cannot send transformed message: {err}") } @@ -83,7 +156,11 @@ impl GenMapper { Ok(messages) => { for message in messages { match MqttMessage::try_from(message) { - Ok(message) => self.mqtt.send(message).await?, + Ok(message) => { + self.messages + .send(OutputMessage::MqttMessage(message)) + .await? + } Err(err) => { error!(target: "gen-mapper", "{pipeline_id}: cannot send transformed message: {err}") } diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index 551e6707247..b1af38c627e 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -48,6 +48,7 @@ impl PipelineConfig { self, js_runtime: &JsRuntime, config_dir: &Path, + source: Utf8PathBuf, ) -> Result { let input = topic_filters(&self.input_topics)?; let stages = self @@ -58,6 +59,7 @@ impl PipelineConfig { Ok(Pipeline { input_topics: input, stages, + source, }) } } diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index f0a78cf6934..5812b150f4b 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -50,6 +50,10 @@ impl JsFilter { } } + pub fn path(&self) -> &Path { + &self.path + } + /// Process a message returning zero, one or more messages /// /// The "process" function of the JS module is passed 3 arguments diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index 8019c489ae2..95cc27a878d 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -8,10 +8,12 @@ use crate::config::PipelineConfig; use crate::js_filter::JsRuntime; use crate::pipeline::Pipeline; use camino::Utf8Path; +use camino::Utf8PathBuf; use std::collections::HashMap; use std::convert::Infallible; use std::path::Path; use std::path::PathBuf; +use tedge_actors::fan_in_message_type; use tedge_actors::Builder; use tedge_actors::DynSender; use tedge_actors::MessageSink; @@ -20,18 +22,23 @@ use tedge_actors::NoConfig; use tedge_actors::RuntimeRequest; use tedge_actors::RuntimeRequestSink; use tedge_actors::SimpleMessageBoxBuilder; +use tedge_file_system_ext::FsWatchEvent; use tedge_mqtt_ext::MqttMessage; +use tedge_mqtt_ext::PublishOrSubscribe; use tedge_mqtt_ext::TopicFilter; use tokio::fs::read_dir; use tokio::fs::read_to_string; use tracing::error; use tracing::info; +fan_in_message_type!(InputMessage[MqttMessage, FsWatchEvent]: Clone, Debug, Eq, PartialEq); +fan_in_message_type!(OutputMessage[MqttMessage]: Clone, Debug, Eq, PartialEq); + pub struct GenMapperBuilder { config_dir: PathBuf, - message_box: SimpleMessageBoxBuilder, + message_box: SimpleMessageBoxBuilder, pipelines: HashMap, - pipeline_specs: HashMap, + pipeline_specs: HashMap, js_runtime: JsRuntime, } @@ -92,7 +99,8 @@ impl GenMapperBuilder { if let Some(name) = file.as_ref().file_name() { let specs = read_to_string(file.as_ref()).await?; let pipeline: PipelineConfig = toml::from_str(&specs)?; - self.pipeline_specs.insert(name.to_string(), pipeline); + self.pipeline_specs + .insert(name.to_string(), (file.as_ref().to_owned(), pipeline)); } Ok(()) @@ -104,8 +112,8 @@ impl GenMapperBuilder { } fn compile(&mut self) { - for (name, specs) in self.pipeline_specs.drain() { - match specs.compile(&self.js_runtime, &self.config_dir) { + for (name, (source, specs)) in self.pipeline_specs.drain() { + match specs.compile(&self.js_runtime, &self.config_dir, source) { Ok(pipeline) => { let _ = self.pipelines.insert(name, pipeline); } @@ -118,10 +126,21 @@ impl GenMapperBuilder { pub fn connect( &mut self, - mqtt: &mut (impl MessageSource + MessageSink), + mqtt: &mut (impl MessageSource + MessageSink), ) { - mqtt.connect_sink(self.topics(), &self.message_box); - self.message_box.connect_sink(NoConfig, mqtt); + mqtt.connect_mapped_sink(self.topics(), &self.message_box, |msg| { + Some(InputMessage::MqttMessage(msg)) + }); + self.message_box + .connect_mapped_sink(NoConfig, mqtt, move |msg| match msg { + OutputMessage::MqttMessage(mqtt) => Some(PublishOrSubscribe::Publish(mqtt)), + }); + } + + pub fn connect_fs(&mut self, fs: &mut impl MessageSource) { + fs.connect_mapped_sink(self.config_dir.clone(), &self.message_box, |msg| { + Some(InputMessage::FsWatchEvent(msg)) + }); } fn topics(&self) -> TopicFilter { @@ -148,9 +167,10 @@ impl Builder for GenMapperBuilder { fn build(self) -> GenMapper { GenMapper { - mqtt: self.message_box.build(), + messages: self.message_box.build(), pipelines: self.pipelines, js_runtime: self.js_runtime, + config_dir: self.config_dir, } } } diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index 4f5c8bb84fd..3cd9e92db81 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -1,6 +1,7 @@ use crate::js_filter::JsFilter; use crate::js_filter::JsRuntime; use crate::LoadError; +use camino::Utf8PathBuf; use serde_json::json; use serde_json::Value; use tedge_mqtt_ext::MqttMessage; @@ -14,6 +15,8 @@ pub struct Pipeline { /// Transformation stages to apply in order to the messages pub stages: Vec, + + pub source: Utf8PathBuf, } /// A message transformation stage From e0a367cb73aed6eabf25200c1f7e1d5182ad8583 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Mon, 26 May 2025 14:19:56 +0200 Subject: [PATCH 14/50] Tedge-gen-mapper dynamically subscribes to MQTT topics Signed-off-by: Didier Wenzek --- .../extensions/tedge_gen_mapper/src/actor.rs | 23 ++++++++ crates/extensions/tedge_gen_mapper/src/lib.rs | 18 +++++- crates/extensions/tedge_mqtt_ext/src/lib.rs | 58 ++++++++++++++++++- crates/extensions/tedge_mqtt_ext/src/trie.rs | 10 ++++ 4 files changed, 105 insertions(+), 4 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index 8506cc2bc99..b2e832e99fb 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -9,6 +9,8 @@ use async_trait::async_trait; use camino::Utf8PathBuf; use std::collections::HashMap; use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; use tedge_actors::Actor; use tedge_actors::MessageReceiver; use tedge_actors::RuntimeError; @@ -16,6 +18,8 @@ use tedge_actors::Sender; use tedge_actors::SimpleMessageBox; use tedge_file_system_ext::FsWatchEvent; use tedge_mqtt_ext::MqttMessage; +use tedge_mqtt_ext::SubscriptionDiff; +use tedge_mqtt_ext::TopicFilter; use tokio::time::interval; use tokio::time::Duration; use tracing::error; @@ -24,6 +28,7 @@ use tracing::info; pub struct GenMapper { pub(super) messages: SimpleMessageBox, pub(super) pipelines: HashMap, + pub(super) subscriptions: Arc>, pub(super) js_runtime: JsRuntime, pub(super) config_dir: PathBuf, } @@ -58,6 +63,7 @@ impl Actor for GenMapper { self.reload_filter(path).await; } else if path.extension() == Some("toml") { self.reload_pipeline(path).await; + self.send_updated_subscriptions().await?; } }, Some(InputMessage::FsWatchEvent(e)) => { @@ -119,6 +125,23 @@ impl GenMapper { } } + async fn send_updated_subscriptions(&mut self) -> Result<(), RuntimeError> { + let topics = self.update_subscriptions(); + let diff = SubscriptionDiff::new(&topics, &TopicFilter::empty()); + self.messages + .send(OutputMessage::SubscriptionDiff(diff)) + .await?; + Ok(()) + } + + fn update_subscriptions(&self) -> TopicFilter { + let mut topics = self.subscriptions.lock().unwrap(); + for pipeline in self.pipelines.values() { + topics.add_all(pipeline.topics()) + } + topics.clone() + } + async fn filter(&mut self, message: Message) -> Result<(), RuntimeError> { let timestamp = DateTime::now(); for (pipeline_id, pipeline) in self.pipelines.iter_mut() { diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index 95cc27a878d..7956be10d69 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -13,6 +13,8 @@ use std::collections::HashMap; use std::convert::Infallible; use std::path::Path; use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; use tedge_actors::fan_in_message_type; use tedge_actors::Builder; use tedge_actors::DynSender; @@ -23,8 +25,10 @@ use tedge_actors::RuntimeRequest; use tedge_actors::RuntimeRequestSink; use tedge_actors::SimpleMessageBoxBuilder; use tedge_file_system_ext::FsWatchEvent; +use tedge_mqtt_ext::DynSubscriptions; use tedge_mqtt_ext::MqttMessage; use tedge_mqtt_ext::PublishOrSubscribe; +use tedge_mqtt_ext::SubscriptionDiff; use tedge_mqtt_ext::TopicFilter; use tokio::fs::read_dir; use tokio::fs::read_to_string; @@ -32,13 +36,14 @@ use tracing::error; use tracing::info; fan_in_message_type!(InputMessage[MqttMessage, FsWatchEvent]: Clone, Debug, Eq, PartialEq); -fan_in_message_type!(OutputMessage[MqttMessage]: Clone, Debug, Eq, PartialEq); +fan_in_message_type!(OutputMessage[MqttMessage, SubscriptionDiff]: Clone, Debug, Eq, PartialEq); pub struct GenMapperBuilder { config_dir: PathBuf, message_box: SimpleMessageBoxBuilder, pipelines: HashMap, pipeline_specs: HashMap, + subscriptions: Arc>, js_runtime: JsRuntime, } @@ -51,6 +56,7 @@ impl GenMapperBuilder { message_box: SimpleMessageBoxBuilder::new("GenMapper", 16), pipelines: HashMap::default(), pipeline_specs: HashMap::default(), + subscriptions: Arc::new(Mutex::new(TopicFilter::empty())), js_runtime, }) } @@ -126,14 +132,19 @@ impl GenMapperBuilder { pub fn connect( &mut self, - mqtt: &mut (impl MessageSource + MessageSink), + mqtt: &mut (impl MessageSource + MessageSink), ) { - mqtt.connect_mapped_sink(self.topics(), &self.message_box, |msg| { + let dyn_subscriptions = DynSubscriptions::new(self.topics()); + mqtt.connect_mapped_sink(dyn_subscriptions.clone(), &self.message_box, |msg| { Some(InputMessage::MqttMessage(msg)) }); + let client_id = dyn_subscriptions.client_id(); self.message_box .connect_mapped_sink(NoConfig, mqtt, move |msg| match msg { OutputMessage::MqttMessage(mqtt) => Some(PublishOrSubscribe::Publish(mqtt)), + OutputMessage::SubscriptionDiff(diff) => { + Some(PublishOrSubscribe::subscribe(client_id, diff)) + } }); } @@ -169,6 +180,7 @@ impl Builder for GenMapperBuilder { GenMapper { messages: self.message_box.build(), pipelines: self.pipelines, + subscriptions: self.subscriptions, js_runtime: self.js_runtime, config_dir: self.config_dir, } diff --git a/crates/extensions/tedge_mqtt_ext/src/lib.rs b/crates/extensions/tedge_mqtt_ext/src/lib.rs index 2be5b2f3902..e0295a1d1cd 100644 --- a/crates/extensions/tedge_mqtt_ext/src/lib.rs +++ b/crates/extensions/tedge_mqtt_ext/src/lib.rs @@ -16,6 +16,8 @@ use mqtt_channel::SubscriberOps; pub use mqtt_channel::Topic; pub use mqtt_channel::TopicFilter; use std::convert::Infallible; +use std::sync::Arc; +use std::sync::Mutex; use tedge_actors::fan_in_message_type; use tedge_actors::futures::channel::mpsc; use tedge_actors::Actor; @@ -36,7 +38,7 @@ use tedge_actors::Server; use tedge_actors::ServerActorBuilder; use tedge_actors::ServerConfig; use trie::MqtTrie; -use trie::SubscriptionDiff; +pub use trie::SubscriptionDiff; pub type MqttConfig = mqtt_channel::Config; @@ -64,6 +66,12 @@ pub enum PublishOrSubscribe { Subscribe(SubscriptionRequest), } +impl PublishOrSubscribe { + pub fn subscribe(client_id: ClientId, diff: SubscriptionDiff) -> Self { + PublishOrSubscribe::Subscribe(SubscriptionRequest { diff, client_id }) + } +} + impl InputCombiner { pub fn close_input(&mut self) { self.publish_receiver.close(); @@ -161,6 +169,54 @@ impl MessageSource for MqttActorBuilder { } } +impl MessageSource for MqttActorBuilder { + fn connect_sink( + &mut self, + subscriptions: DynSubscriptions, + peer: &impl MessageSink, + ) { + let client_id = self.connect_id_sink(subscriptions.init_topics(), peer); + subscriptions.set_client_id(client_id); + } +} + +#[derive(Clone)] +pub struct DynSubscriptions { + inner: Arc>, +} +pub struct DynSubscriptionsInner { + init_topics: TopicFilter, + client_id: Option, +} + +impl DynSubscriptions { + pub fn new(init_topics: TopicFilter) -> Self { + let inner = DynSubscriptionsInner { + init_topics, + client_id: None, + }; + DynSubscriptions { + inner: Arc::new(Mutex::new(inner)), + } + } + + fn set_client_id(&self, client_id: ClientId) { + let mut inner = self.inner.lock().unwrap(); + inner.client_id = Some(client_id); + } + + fn init_topics(&self) -> TopicFilter { + self.inner.lock().unwrap().init_topics.clone() + } + + /// Return the client id + /// + /// Panic if not properly registered as a sink of the MqttActorBuilder + pub fn client_id(&self) -> ClientId { + self.inner.lock().unwrap().client_id.unwrap() + } +} + impl MqttActorBuilder { pub fn connect_id_sink( &mut self, diff --git a/crates/extensions/tedge_mqtt_ext/src/trie.rs b/crates/extensions/tedge_mqtt_ext/src/trie.rs index 0d4163a0955..b961ad35814 100644 --- a/crates/extensions/tedge_mqtt_ext/src/trie.rs +++ b/crates/extensions/tedge_mqtt_ext/src/trie.rs @@ -156,6 +156,16 @@ impl SubscriptionDiff { } } + pub fn new( + subscribe: &mqtt_channel::TopicFilter, + unsubscribe: &mqtt_channel::TopicFilter, + ) -> Self { + Self { + subscribe: subscribe.patterns().iter().cloned().collect(), + unsubscribe: unsubscribe.patterns().iter().cloned().collect(), + } + } + fn with_topic_prefix(self, prefix: &str) -> Self { Self { subscribe: self From 7d296145302ed9cfdfaa61b4b0a06ac3dadddefe Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Wed, 28 May 2025 16:19:19 +0200 Subject: [PATCH 15/50] Test JS example: collectd pipeline --- .../pipelines/collectd-to-te.js | 7 +- .../tedge_gen_mapper/src/js_filter.rs | 76 ++++++++++++++++--- 2 files changed, 72 insertions(+), 11 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js b/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js index e5aa71c0540..53a56e13aff 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js @@ -7,8 +7,13 @@ export function process (timestamp, message, config) { let time = data[0] let value = data[1] + var topic = "te/device/main///m/collectd" + if (config && config.topic) { + topic = config.topic + } + return [ { - topic: config.topic || "te/device/main///m/collectd", + topic: topic, payload: `{"time": ${time}, "${group}": {"${measurement}": ${value}}}` }] } \ No newline at end of file diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 5812b150f4b..3db0917c4dc 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -70,7 +70,7 @@ impl JsFilter { ) -> Result, FilterError> { debug!(target: "MAPPING", "{}: process({timestamp:?}, {message:?})", self.path.display()); let input = (timestamp.clone(), message.clone(), self.config.clone()); - js.call_function(&self, "process", input) + js.call_function(self, "process", input) .await .map_err(pipeline::error_from_js) } @@ -90,7 +90,7 @@ impl JsFilter { debug!(target: "MAPPING", "{}: update_config({message:?})", self.path.display()); let input = (message.clone(), self.config.clone()); let config = js - .call_function(&self, "update_config", input) + .call_function(self, "update_config", input) .await .map_err(pipeline::error_from_js)?; self.config = config; @@ -114,7 +114,7 @@ impl JsFilter { } debug!(target: "MAPPING", "{}: tick({timestamp:?})", self.path.display()); let input = (timestamp.clone(), self.config.clone()); - js.call_function(&self, "tick", input) + js.call_function(self, "tick", input) .await .map_err(pipeline::error_from_js) } @@ -175,14 +175,24 @@ impl JsRuntime { let name = module.path.display().to_string(); rquickjs::async_with!(self.context => |ctx| { - let m = rquickjs::Module::declare(ctx, name, source.clone())?; + debug!(target: "MAPPING", "compile({name})"); + let m = rquickjs::Module::declare(ctx, name.clone(), source.clone())?; let (m,p) = m.eval()?; let () = p.finish()?; + debug!(target: "MAPPING", "link({name})"); let f: rquickjs::Value = m.get(function)?; let f = rquickjs::Function::from_value(f)?; - let r = f.call(args)?; - Ok(r) + + debug!(target: "MAPPING", "execute({name})"); + let r = f.call(args); + if r.is_err() { + let err = r.err().unwrap(); + debug!(target: "MAPPING", "execute({name}) => {err:?}"); + Err(err.into()) + } else { + Ok(r.unwrap()) + } }) .await } @@ -190,21 +200,28 @@ impl JsRuntime { impl<'js> FromJs<'js> for Message { fn from_js(_ctx: &Ctx<'js>, value: Value<'js>) -> rquickjs::Result { + debug!(target: "MAPPING", "from_js(...)"); match value.as_object() { None => Ok(Message { topic: "".to_string(), payload: "".to_string(), }), - Some(object) => Ok(Message { - topic: object.get("topic")?, - payload: object.get("payload")?, - }), + Some(object) => { + let topic = object.get("topic"); + let payload = object.get("payload"); + debug!(target: "MAPPING", "from_js(...) -> topic = {:?}, payload = {:?}", topic, payload); + Ok(Message { + topic: topic?, + payload: payload?, + }) + }, } } } impl<'js> IntoJs<'js> for Message { fn into_js(self, ctx: &Ctx<'js>) -> rquickjs::Result> { + debug!(target: "MAPPING", "into_js({self:?})"); let msg = Object::new(ctx.clone())?; msg.set("topic", self.topic)?; msg.set("payload", self.payload)?; @@ -214,6 +231,7 @@ impl<'js> IntoJs<'js> for Message { impl<'js> IntoJs<'js> for DateTime { fn into_js(self, ctx: &Ctx<'js>) -> rquickjs::Result> { + debug!(target: "MAPPING", "into_js({self:?})"); let msg = Object::new(ctx.clone())?; msg.set("topic", self.seconds)?; msg.set("payload", self.nanoseconds)?; @@ -331,4 +349,42 @@ mod tests { eprintln!("{:?}", error); assert!(error.to_string().contains("Exception generated by QuickJS")); } + + #[tokio::test] + async fn collectd_filter() { + let script = r#" +export function process (timestamp, message, config) { + let groups = message.topic.split( '/') + let data = message.payload.split(':') + + let group = groups[2] + let measurement = groups[3] + let time = data[0] + let value = data[1] + + var topic = "te/device/main///m/collectd" + if (config && config.topic) { + topic = config.topic + } + + return [ { + topic: topic, + payload: `{"time": ${time}, "${group}": {"${measurement}": ${value}}}` + }] +} + "#; + let mut runtime = JsRuntime::try_new().await.unwrap(); + let filter = runtime.load_js("collectd.js", script).unwrap(); + + let input = Message::new("collectd/h/memory/percent-used", "1748440192.104:19.9289468288182"); + let output = Message::new("te/device/main///m/collectd", r#"{"time": 1748440192.104, "memory": {"percent-used": 19.9289468288182}}"#); + assert_eq!( + filter + .process(&runtime, &DateTime::now(), &input) + .await + .unwrap(), + vec![output] + ); + } + } From edb5e0ff6fb82cae2e4353e3d3d34d5ea7a6b296 Mon Sep 17 00:00:00 2001 From: James Rhodes Date: Fri, 30 May 2025 16:24:02 +0100 Subject: [PATCH 16/50] Tidy JS example Signed-off-by: James Rhodes --- .../pipelines/collectd-to-te.js | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js b/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js index 53a56e13aff..31edd135589 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js @@ -1,19 +1,14 @@ -export function process (timestamp, message, config) { - let groups = message.topic.split( '/') +export function process(_timestamp, message, config) { + let groups = message.topic.split('/') let data = message.payload.split(':') let group = groups[2] - let measurement = groups[3] - let time = data[0] - let value = data[1] + let measurement = groups[3] + let time = data[0] + let value = data[1] - var topic = "te/device/main///m/collectd" - if (config && config.topic) { - topic = config.topic - } - - return [ { - topic: topic, + return [{ + topic: config?.topic || "te/device/main///m/collectd", payload: `{"time": ${time}, "${group}": {"${measurement}": ${value}}}` }] } \ No newline at end of file From 35510ee1f7d89f3ad673e935e38ae9ef5b23b462 Mon Sep 17 00:00:00 2001 From: James Rhodes Date: Fri, 30 May 2025 16:25:48 +0100 Subject: [PATCH 17/50] Relay error messages from Javascript to the generic mapper logs Signed-off-by: James Rhodes --- Cargo.lock | 1 + crates/extensions/tedge_gen_mapper/Cargo.toml | 1 + crates/extensions/tedge_gen_mapper/src/js_filter.rs | 11 ++++++++--- crates/extensions/tedge_gen_mapper/src/lib.rs | 3 +++ crates/extensions/tedge_gen_mapper/src/pipeline.rs | 2 +- 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 19317544fef..ff69243a0da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4935,6 +4935,7 @@ dependencies = [ name = "tedge_gen_mapper" version = "1.5.1" dependencies = [ + "anyhow", "async-trait", "camino", "rquickjs", diff --git a/crates/extensions/tedge_gen_mapper/Cargo.toml b/crates/extensions/tedge_gen_mapper/Cargo.toml index 80c9b9cbb9b..726dda5737f 100644 --- a/crates/extensions/tedge_gen_mapper/Cargo.toml +++ b/crates/extensions/tedge_gen_mapper/Cargo.toml @@ -9,6 +9,7 @@ homepage.workspace = true repository.workspace = true [dependencies] +anyhow = { workspace = true } async-trait = { workspace = true } camino = { workspace = true, features = ["serde1"] } rquickjs = { workspace = true, features = ["futures","parallel"] } diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 3db0917c4dc..ce900cbfd3e 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -176,7 +176,7 @@ impl JsRuntime { rquickjs::async_with!(self.context => |ctx| { debug!(target: "MAPPING", "compile({name})"); - let m = rquickjs::Module::declare(ctx, name.clone(), source.clone())?; + let m = rquickjs::Module::declare(ctx.clone(), name.clone(), source.clone())?; let (m,p) = m.eval()?; let () = p.finish()?; @@ -187,9 +187,14 @@ impl JsRuntime { debug!(target: "MAPPING", "execute({name})"); let r = f.call(args); if r.is_err() { + if let Some(ex) = ctx.catch().as_exception() { + let err = anyhow::anyhow!("{ex}"); + Err(err.context("JS raised exception").into()) + } else { let err = r.err().unwrap(); debug!(target: "MAPPING", "execute({name}) => {err:?}"); Err(err.into()) + } } else { Ok(r.unwrap()) } @@ -214,7 +219,7 @@ impl<'js> FromJs<'js> for Message { topic: topic?, payload: payload?, }) - }, + } } } } @@ -347,7 +352,7 @@ mod tests { .await .unwrap_err(); eprintln!("{:?}", error); - assert!(error.to_string().contains("Exception generated by QuickJS")); + assert!(error.to_string().contains("Cannot process that message")); } #[tokio::test] diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index 7956be10d69..ea8a7936e27 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -200,4 +200,7 @@ pub enum LoadError { #[error(transparent)] JsError(#[from] rquickjs::Error), + + #[error(transparent)] + Anyhow(#[from] anyhow::Error), } diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index 3cd9e92db81..73da0e0a6f1 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -179,5 +179,5 @@ impl TryFrom for MqttMessage { } pub fn error_from_js(err: LoadError) -> FilterError { - FilterError::IncorrectSetting(format!("{}", err)) + FilterError::IncorrectSetting(format!("{err:#}")) } From 04ec637572027b9db4fb4853a3ad35cfccd82e28 Mon Sep 17 00:00:00 2001 From: James Rhodes Date: Fri, 30 May 2025 16:26:40 +0100 Subject: [PATCH 18/50] fixup! fixup! Add JS example: collectd pipline Signed-off-by: James Rhodes --- crates/extensions/tedge_gen_mapper/src/js_filter.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index ce900cbfd3e..5d7c3374a05 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -238,8 +238,8 @@ impl<'js> IntoJs<'js> for DateTime { fn into_js(self, ctx: &Ctx<'js>) -> rquickjs::Result> { debug!(target: "MAPPING", "into_js({self:?})"); let msg = Object::new(ctx.clone())?; - msg.set("topic", self.seconds)?; - msg.set("payload", self.nanoseconds)?; + msg.set("seconds", self.seconds)?; + msg.set("nanoseconds", self.nanoseconds)?; Ok(Value::from_object(msg)) } } From d9b2ed517dbc8a8f1f161d981d46aa6fd7691389 Mon Sep 17 00:00:00 2001 From: James Rhodes Date: Fri, 30 May 2025 16:27:05 +0100 Subject: [PATCH 19/50] fixup! fixup! Add JS example: collectd pipline Signed-off-by: James Rhodes --- crates/extensions/tedge_gen_mapper/Cargo.toml | 2 +- crates/extensions/tedge_gen_mapper/src/js_filter.rs | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/Cargo.toml b/crates/extensions/tedge_gen_mapper/Cargo.toml index 726dda5737f..1c6349b02f4 100644 --- a/crates/extensions/tedge_gen_mapper/Cargo.toml +++ b/crates/extensions/tedge_gen_mapper/Cargo.toml @@ -12,7 +12,7 @@ repository.workspace = true anyhow = { workspace = true } async-trait = { workspace = true } camino = { workspace = true, features = ["serde1"] } -rquickjs = { workspace = true, features = ["futures","parallel"] } +rquickjs = { workspace = true, features = ["futures", "parallel"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } tedge_actors = { workspace = true } diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 5d7c3374a05..1809c6944de 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -381,8 +381,14 @@ export function process (timestamp, message, config) { let mut runtime = JsRuntime::try_new().await.unwrap(); let filter = runtime.load_js("collectd.js", script).unwrap(); - let input = Message::new("collectd/h/memory/percent-used", "1748440192.104:19.9289468288182"); - let output = Message::new("te/device/main///m/collectd", r#"{"time": 1748440192.104, "memory": {"percent-used": 19.9289468288182}}"#); + let input = Message::new( + "collectd/h/memory/percent-used", + "1748440192.104:19.9289468288182", + ); + let output = Message::new( + "te/device/main///m/collectd", + r#"{"time": 1748440192.104, "memory": {"percent-used": 19.9289468288182}}"#, + ); assert_eq!( filter .process(&runtime, &DateTime::now(), &input) @@ -391,5 +397,4 @@ export function process (timestamp, message, config) { vec![output] ); } - } From 2e797aba763f7596ff2017c07cc78de466d229ff Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Wed, 4 Jun 2025 19:12:08 +0200 Subject: [PATCH 20/50] Fix memory leak The first design was naive (i.e. loading the javascript module for each and every function call) and leading to a memory leak (loading a module with the same name keeps the previous version of the module in memory). The new design fixes the issue. The memory is stable while translating to c8y 600 tedge measurements per second during 20 minutes (mode debug) ```shell $ watch ps -p 1382888 -o args,%cpu,etimes,times,%mem,rss,vsz COMMAND %CPU ELAPSED TIME %MEM RSS VSZ tedge-mapper gen 47.4 1322 627 0.0 41440 1359476 $ tedge mqtt sub 'test/output' --duration 1000 | pv | wc -l 89.3MiB 0:16:40 [91.4KiB/s] [ <=> ] 586311 ``` Signed-off-by: Didier Wenzek --- crates/extensions/tedge_gen_mapper/Cargo.toml | 4 +- .../tedge_gen_mapper/pipelines/set_topic.js | 6 + .../extensions/tedge_gen_mapper/src/actor.rs | 8 +- .../extensions/tedge_gen_mapper/src/config.rs | 8 +- .../tedge_gen_mapper/src/js_filter.rs | 194 ++++++----------- .../tedge_gen_mapper/src/js_runtime.rs | 204 ++++++++++++++++++ crates/extensions/tedge_gen_mapper/src/lib.rs | 13 +- .../tedge_gen_mapper/src/pipeline.rs | 9 +- 8 files changed, 309 insertions(+), 137 deletions(-) create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/set_topic.js create mode 100644 crates/extensions/tedge_gen_mapper/src/js_runtime.rs diff --git a/crates/extensions/tedge_gen_mapper/Cargo.toml b/crates/extensions/tedge_gen_mapper/Cargo.toml index 1c6349b02f4..91d491c5b61 100644 --- a/crates/extensions/tedge_gen_mapper/Cargo.toml +++ b/crates/extensions/tedge_gen_mapper/Cargo.toml @@ -12,7 +12,7 @@ repository.workspace = true anyhow = { workspace = true } async-trait = { workspace = true } camino = { workspace = true, features = ["serde1"] } -rquickjs = { workspace = true, features = ["futures", "parallel"] } +rquickjs = { workspace = true, default-features = false, features = ["futures", "parallel"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } tedge_actors = { workspace = true } @@ -20,7 +20,7 @@ tedge_file_system_ext = { workspace = true } tedge_mqtt_ext = { workspace = true } thiserror = { workspace = true } time = { workspace = true } -tokio = { workspace = true, features = ["fs", "macros", "time"] } +tokio = { workspace = true, features = ["fs", "macros", "time", "sync"] } toml = { workspace = true, features = ["parse"] } tracing = { workspace = true } diff --git a/crates/extensions/tedge_gen_mapper/pipelines/set_topic.js b/crates/extensions/tedge_gen_mapper/pipelines/set_topic.js new file mode 100644 index 00000000000..16ce07f0589 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/set_topic.js @@ -0,0 +1,6 @@ +export function process (timestamp, message, config) { + return [{ + topic: config?.topic || "te/error", + payload: message.payload + }] +} diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index b2e832e99fb..121c6b3d38c 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -1,5 +1,5 @@ use crate::config::PipelineConfig; -use crate::js_filter::JsRuntime; +use crate::js_runtime::JsRuntime; use crate::pipeline::DateTime; use crate::pipeline::Message; use crate::pipeline::Pipeline; @@ -84,9 +84,8 @@ impl GenMapper { for stage in &mut pipeline.stages { if stage.filter.path() == path { match self.js_runtime.load_file(&path).await { - Ok(filter) => { + Ok(()) => { info!("Reloaded filter {path}"); - stage.filter = filter } Err(e) => { error!("Failed to reload filter {path}: {e}"); @@ -174,6 +173,9 @@ impl GenMapper { async fn tick(&mut self) -> Result<(), RuntimeError> { let timestamp = DateTime::now(); + if timestamp.seconds % 300 == 0 { + self.js_runtime.dump_memory_stats().await; + } for (pipeline_id, pipeline) in self.pipelines.iter_mut() { match pipeline.tick(&self.js_runtime, ×tamp).await { Ok(messages) => { diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index b1af38c627e..dd389bd2bae 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -1,4 +1,5 @@ -use crate::js_filter::JsRuntime; +use crate::js_filter::JsFilter; +use crate::js_runtime::JsRuntime; use crate::pipeline::Pipeline; use crate::pipeline::Stage; use crate::LoadError; @@ -65,13 +66,12 @@ impl PipelineConfig { } impl StageConfig { - pub fn compile(self, js_runtime: &JsRuntime, config_dir: &Path) -> Result { + pub fn compile(self, _js_runtime: &JsRuntime, config_dir: &Path) -> Result { let path = match self.filter { FilterSpec::JavaScript(path) if path.is_absolute() => path.into(), FilterSpec::JavaScript(path) => config_dir.join(path), }; - let filter = js_runtime - .loaded_module(path)? + let filter = JsFilter::new(path) .with_config(self.config) .with_tick_every_seconds(self.tick_every_seconds); let config_topics = topic_filters(&self.meta_topics)?; diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 1809c6944de..5e27b2b193d 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -1,14 +1,13 @@ +use crate::js_runtime::JsRuntime; use crate::pipeline; use crate::pipeline::DateTime; use crate::pipeline::FilterError; use crate::pipeline::Message; -use crate::LoadError; +use anyhow::Context; use rquickjs::Ctx; use rquickjs::FromJs; use rquickjs::IntoJs; -use rquickjs::Object; use rquickjs::Value; -use std::collections::HashMap; use std::path::Path; use std::path::PathBuf; use tracing::debug; @@ -20,7 +19,7 @@ pub struct JsFilter { tick_every_seconds: u64, } -#[derive(Clone, Default)] +#[derive(Clone, Debug, Default)] pub struct JsonValue(serde_json::Value); impl JsFilter { @@ -32,6 +31,10 @@ impl JsFilter { } } + pub fn module_name(&self) -> String { + self.path.display().to_string() + } + pub fn with_config(self, config: Option) -> Self { if let Some(config) = config { Self { @@ -68,11 +71,16 @@ impl JsFilter { timestamp: &DateTime, message: &Message, ) -> Result, FilterError> { - debug!(target: "MAPPING", "{}: process({timestamp:?}, {message:?})", self.path.display()); - let input = (timestamp.clone(), message.clone(), self.config.clone()); - js.call_function(self, "process", input) + debug!(target: "MAPPING", "{}: process({timestamp:?}, {message:?})", self.module_name()); + let input = vec![ + timestamp.clone().into(), + message.clone().into(), + self.config.clone(), + ]; + js.call_function(&self.path, "process", input) .await - .map_err(pipeline::error_from_js) + .map_err(pipeline::error_from_js)? + .try_into() } /// Update the filter config using a metadata message @@ -87,10 +95,10 @@ impl JsFilter { js: &JsRuntime, message: &Message, ) -> Result<(), FilterError> { - debug!(target: "MAPPING", "{}: update_config({message:?})", self.path.display()); - let input = (message.clone(), self.config.clone()); + debug!(target: "MAPPING", "{}: update_config({message:?})", self.module_name()); + let input = vec![message.clone().into(), self.config.clone()]; let config = js - .call_function(self, "update_config", input) + .call_function(&self.path, "update_config", input) .await .map_err(pipeline::error_from_js)?; self.config = config; @@ -112,143 +120,78 @@ impl JsFilter { if !timestamp.tick_now(self.tick_every_seconds) { return Ok(vec![]); } - debug!(target: "MAPPING", "{}: tick({timestamp:?})", self.path.display()); - let input = (timestamp.clone(), self.config.clone()); - js.call_function(self, "tick", input) + debug!(target: "MAPPING", "{}: tick({timestamp:?})", self.module_name()); + let input = vec![timestamp.clone().into(), self.config.clone()]; + js.call_function(&self.path, "tick", input) .await - .map_err(pipeline::error_from_js) + .map_err(pipeline::error_from_js)? + .try_into() } } -pub struct JsRuntime { - context: rquickjs::AsyncContext, - modules: HashMap>, +impl From for JsonValue { + fn from(value: Message) -> Self { + JsonValue(value.json()) + } } -impl JsRuntime { - pub async fn try_new() -> Result { - let runtime = rquickjs::AsyncRuntime::new()?; - let context = rquickjs::AsyncContext::full(&runtime).await?; - let modules = HashMap::new(); - Ok(JsRuntime { context, modules }) +impl From for JsonValue { + fn from(value: DateTime) -> Self { + JsonValue(value.json()) } +} - pub async fn load_file(&mut self, path: impl AsRef) -> Result { - let path = path.as_ref(); - let source = tokio::fs::read_to_string(path).await?; - self.load_js(path, source) - } +impl TryFrom for Message { + type Error = FilterError; - pub fn load_js( - &mut self, - path: impl AsRef, - source: impl Into>, - ) -> Result { - let path = path.as_ref().to_path_buf(); - self.modules.insert(path.clone(), source.into()); - Ok(JsFilter::new(path)) + fn try_from(value: serde_json::Value) -> Result { + let message = serde_json::from_value(value) + .with_context(|| "Couldn't extract message payload and topic")?; + Ok(message) } +} - pub fn loaded_module(&self, path: PathBuf) -> Result { - match self.modules.get(&path) { - None => Err(LoadError::ScriptNotLoaded { path }), - Some(_) => Ok(JsFilter::new(path)), - } - } +impl TryFrom for Message { + type Error = FilterError; - pub async fn call_function( - &self, - module: &JsFilter, - function: &str, - args: Args, - ) -> Result - where - for<'a> Args: rquickjs::function::IntoArgs<'a> + Send + 'a, - for<'a> Ret: FromJs<'a> + Send + 'a, - { - let Some(source) = self.modules.get(&module.path) else { - return Err(LoadError::ScriptNotLoaded { - path: module.path.clone(), - }); - }; - - let name = module.path.display().to_string(); - - rquickjs::async_with!(self.context => |ctx| { - debug!(target: "MAPPING", "compile({name})"); - let m = rquickjs::Module::declare(ctx.clone(), name.clone(), source.clone())?; - let (m,p) = m.eval()?; - let () = p.finish()?; - - debug!(target: "MAPPING", "link({name})"); - let f: rquickjs::Value = m.get(function)?; - let f = rquickjs::Function::from_value(f)?; - - debug!(target: "MAPPING", "execute({name})"); - let r = f.call(args); - if r.is_err() { - if let Some(ex) = ctx.catch().as_exception() { - let err = anyhow::anyhow!("{ex}"); - Err(err.context("JS raised exception").into()) - } else { - let err = r.err().unwrap(); - debug!(target: "MAPPING", "execute({name}) => {err:?}"); - Err(err.into()) - } - } else { - Ok(r.unwrap()) - } - }) - .await + fn try_from(value: JsonValue) -> Result { + Message::try_from(value.0) } } -impl<'js> FromJs<'js> for Message { - fn from_js(_ctx: &Ctx<'js>, value: Value<'js>) -> rquickjs::Result { - debug!(target: "MAPPING", "from_js(...)"); - match value.as_object() { - None => Ok(Message { - topic: "".to_string(), - payload: "".to_string(), - }), - Some(object) => { - let topic = object.get("topic"); - let payload = object.get("payload"); - debug!(target: "MAPPING", "from_js(...) -> topic = {:?}, payload = {:?}", topic, payload); - Ok(Message { - topic: topic?, - payload: payload?, - }) +impl TryFrom for Vec { + type Error = FilterError; + + fn try_from(value: JsonValue) -> Result { + match value.0 { + serde_json::Value::Array(array) => array.into_iter().map(Message::try_from).collect(), + serde_json::Value::Object(map) => { + Message::try_from(serde_json::Value::Object(map)).map(|message| vec![message]) } + _ => Err(anyhow::anyhow!("Filters are expected to return an array of messages").into()), } } } -impl<'js> IntoJs<'js> for Message { +struct JsonValueRef<'a>(&'a serde_json::Value); + +impl<'js> IntoJs<'js> for JsonValue { fn into_js(self, ctx: &Ctx<'js>) -> rquickjs::Result> { - debug!(target: "MAPPING", "into_js({self:?})"); - let msg = Object::new(ctx.clone())?; - msg.set("topic", self.topic)?; - msg.set("payload", self.payload)?; - Ok(Value::from_object(msg)) + JsonValueRef(&self.0).into_js(ctx) } } -impl<'js> IntoJs<'js> for DateTime { +impl<'js> IntoJs<'js> for &JsonValue { fn into_js(self, ctx: &Ctx<'js>) -> rquickjs::Result> { - debug!(target: "MAPPING", "into_js({self:?})"); - let msg = Object::new(ctx.clone())?; - msg.set("seconds", self.seconds)?; - msg.set("nanoseconds", self.nanoseconds)?; - Ok(Value::from_object(msg)) + JsonValueRef(&self.0).into_js(ctx) } } -impl<'js> IntoJs<'js> for JsonValue { +impl<'a, 'js> IntoJs<'js> for JsonValueRef<'a> { fn into_js(self, ctx: &Ctx<'js>) -> rquickjs::Result> { match self.0 { serde_json::Value::Null => Ok(Value::new_null(ctx.clone())), - serde_json::Value::Bool(value) => Ok(Value::new_bool(ctx.clone(), value)), + serde_json::Value::Bool(value) => Ok(Value::new_bool(ctx.clone(), *value)), serde_json::Value::Number(value) => { if let Some(n) = value.as_i64() { if let Ok(n) = i32::try_from(n) { @@ -262,20 +205,20 @@ impl<'js> IntoJs<'js> for JsonValue { Ok(nan.into_value()) } serde_json::Value::String(value) => { - let string = rquickjs::String::from_str(ctx.clone(), &value)?; + let string = rquickjs::String::from_str(ctx.clone(), value)?; Ok(string.into_value()) } serde_json::Value::Array(values) => { let array = rquickjs::Array::new(ctx.clone())?; - for (i, value) in values.into_iter().enumerate() { - array.set(i, JsonValue(value))?; + for (i, value) in values.iter().enumerate() { + array.set(i, JsonValueRef(value))?; } Ok(array.into_value()) } serde_json::Value::Object(values) => { let object = rquickjs::Object::new(ctx.clone())?; for (key, value) in values.into_iter() { - object.set(key, JsonValue(value))?; + object.set(key, JsonValueRef(value))?; } Ok(object.into_value()) } @@ -327,7 +270,8 @@ mod tests { async fn identity_filter() { let script = "export function process(t,msg) { return [msg]; };"; let mut runtime = JsRuntime::try_new().await.unwrap(); - let filter = runtime.load_js("id.js", script).unwrap(); + runtime.load_js("id.js", script).await.unwrap(); + let filter = JsFilter::new("id.js".into()); let input = Message::new("te/main/device///m/", "hello world"); let output = input.clone(); @@ -344,7 +288,8 @@ mod tests { async fn error_filter() { let script = r#"export function process(t,msg) { throw new Error("Cannot process that message"); };"#; let mut runtime = JsRuntime::try_new().await.unwrap(); - let filter = runtime.load_js("err.js", script).unwrap(); + runtime.load_js("err.js", script).await.unwrap(); + let filter = JsFilter::new("err.js".into()); let input = Message::new("te/main/device///m/", "hello world"); let error = filter @@ -379,7 +324,8 @@ export function process (timestamp, message, config) { } "#; let mut runtime = JsRuntime::try_new().await.unwrap(); - let filter = runtime.load_js("collectd.js", script).unwrap(); + runtime.load_js("collectd.js", script).await.unwrap(); + let filter = JsFilter::new("collectd.js".into()); let input = Message::new( "collectd/h/memory/percent-used", diff --git a/crates/extensions/tedge_gen_mapper/src/js_runtime.rs b/crates/extensions/tedge_gen_mapper/src/js_runtime.rs new file mode 100644 index 00000000000..892b9f3793d --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/src/js_runtime.rs @@ -0,0 +1,204 @@ +use crate::js_filter::JsonValue; +use crate::LoadError; +use anyhow::anyhow; +use rquickjs::module::Evaluated; +use rquickjs::Ctx; +use rquickjs::Module; +use std::collections::HashMap; +use std::path::Path; +use tokio::sync::mpsc; +use tokio::sync::oneshot; +use tracing::debug; + +pub struct JsRuntime { + runtime: rquickjs::AsyncRuntime, + worker: mpsc::Sender, +} + +impl JsRuntime { + pub async fn try_new() -> Result { + let runtime = rquickjs::AsyncRuntime::new()?; + let context = rquickjs::AsyncContext::full(&runtime).await?; + let worker = JsWorker::spawn(context).await; + Ok(JsRuntime { runtime, worker }) + } + + pub async fn load_file(&mut self, path: impl AsRef) -> Result<(), LoadError> { + let path = path.as_ref(); + let source = tokio::fs::read_to_string(path).await?; + self.load_js(path, source).await + } + + pub async fn load_js( + &mut self, + path: impl AsRef, + source: impl Into>, + ) -> Result<(), LoadError> { + let (sender, receiver) = oneshot::channel(); + let path = path.as_ref().to_path_buf(); + let name = path.display().to_string(); + let source = source.into(); + self.worker + .send(JsRequest::LoadModule { + name, + source, + sender, + }) + .await + .map_err(|err| anyhow!(err))?; + receiver.await.map_err(|err| anyhow!(err))? + } + + pub async fn call_function( + &self, + module: &Path, + function: &str, + args: Vec, + ) -> Result { + let (sender, receiver) = oneshot::channel(); + self.worker + .send(JsRequest::CallFunction { + module: module.display().to_string(), + function: function.to_string(), + args, + sender, + }) + .await + .map_err(|err| anyhow!(err))?; + receiver.await.map_err(|err| anyhow!(err))? + } + + pub async fn dump_memory_stats(&self) { + let usage = self.runtime.memory_usage().await; + tracing::info!(target: "gen-mapper", "Memory usage:"); + tracing::info!(target: "gen-mapper", " - malloc size: {}", usage.malloc_size); + tracing::info!(target: "gen-mapper", " - used memory size: {}", usage.memory_used_size); + tracing::info!(target: "gen-mapper", " - function count: {}", usage.js_func_count); + tracing::info!(target: "gen-mapper", " - object count: {}", usage.obj_count); + tracing::info!(target: "gen-mapper", " - array count: {}", usage.array_count); + tracing::info!(target: "gen-mapper", " - string count: {}", usage.str_count); + tracing::info!(target: "gen-mapper", " - atom count: {}", usage.atom_count); + } +} + +enum JsRequest { + LoadModule { + name: String, + source: Vec, + sender: oneshot::Sender>, + }, + CallFunction { + module: String, + function: String, + args: Vec, + sender: oneshot::Sender>, + }, +} + +struct JsWorker { + context: rquickjs::AsyncContext, + requests: mpsc::Receiver, +} + +impl JsWorker { + pub async fn spawn(context: rquickjs::AsyncContext) -> mpsc::Sender { + let (sender, requests) = mpsc::channel(100); + tokio::spawn(async move { + let worker = JsWorker { context, requests }; + worker.run().await + }); + sender + } + + async fn run(mut self) { + rquickjs::async_with!(self.context => |ctx| { + let mut modules = JsModules::new(); + while let Some(request) = self.requests.recv().await { + match request { + JsRequest::LoadModule{name, source, sender} => { + let result = modules.load_module(ctx.clone(), name, source).await; + let _ = sender.send(result); + } + JsRequest::CallFunction{module, function, args, sender} => { + let result = modules.call_function(ctx.clone(), module, function, args).await; + let _ = sender.send(result); + } + } + } + }) + .await + } +} + +struct JsModules<'js> { + modules: HashMap>, +} + +impl<'js> JsModules<'js> { + fn new() -> Self { + JsModules { + modules: HashMap::new(), + } + } + + async fn load_module( + &mut self, + ctx: Ctx<'js>, + name: String, + source: Vec, + ) -> Result<(), LoadError> { + debug!(target: "MAPPING", "compile({name})"); + let module = Module::declare(ctx, name.clone(), source)?; + let (module, p) = module.eval()?; + let () = p.finish()?; + self.modules.insert(name, module); + Ok(()) + } + + async fn call_function( + &mut self, + ctx: Ctx<'js>, + module_name: String, + function: String, + args: Vec, + ) -> Result { + debug!(target: "MAPPING", "link({module_name}.{function})"); + let module = self + .modules + .get(&module_name) + .ok_or_else(|| LoadError::UnknownModule { + module_name: module_name.clone(), + })?; + let f: rquickjs::Value = module + .get(&function) + .map_err(|_| LoadError::UnknownFunction { + module_name: module_name.clone(), + function: function.clone(), + })?; + let f = rquickjs::Function::from_value(f)?; + + debug!(target: "MAPPING", "execute({module_name}.{function})"); + let r = match &args[..] { + [] => f.call(()), + [v0] => f.call((v0,)), + [v0, v1] => f.call((v0, v1)), + [v0, v1, v2] => f.call((v0, v1, v2)), + [v0, v1, v2, v3] => f.call((v0, v1, v2, v3)), + [v0, v1, v2, v3, v4] => f.call((v0, v1, v2, v3, v4)), + [v0, v1, v2, v3, v4, v5] => f.call((v0, v1, v2, v3, v4, v5)), + [v0, v1, v2, v3, v4, v5, v6] => f.call((v0, v1, v2, v3, v4, v5, v6)), + _ => return Err(anyhow::anyhow!("Too many args").into()), + }; + + debug!(target: "MAPPING", "execute({module_name}.{function}) => {r:?}"); + r.map_err(|err| { + if let Some(ex) = ctx.catch().as_exception() { + let err = anyhow::anyhow!("{ex}"); + err.context("JS raised exception").into() + } else { + debug!(target: "MAPPING", "execute({module_name}.{function}) => {err:?}"); + err.into() + } + }) + } +} diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index ea8a7936e27..130094195a5 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -1,11 +1,12 @@ mod actor; mod config; mod js_filter; +mod js_runtime; mod pipeline; use crate::actor::GenMapper; use crate::config::PipelineConfig; -use crate::js_filter::JsRuntime; +use crate::js_runtime::JsRuntime; use crate::pipeline::Pipeline; use camino::Utf8Path; use camino::Utf8PathBuf; @@ -189,8 +190,14 @@ impl Builder for GenMapperBuilder { #[derive(thiserror::Error, Debug)] pub enum LoadError { - #[error("Script not loaded: {path}")] - ScriptNotLoaded { path: PathBuf }, + #[error("JavaScript module not found: {module_name}")] + UnknownModule { module_name: String }, + + #[error("JavaScript function not found: {function} in {module_name}")] + UnknownFunction { + module_name: String, + function: String, + }, #[error(transparent)] IoError(#[from] std::io::Error), diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index 73da0e0a6f1..e9e993a03f2 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -1,5 +1,5 @@ use crate::js_filter::JsFilter; -use crate::js_filter::JsRuntime; +use crate::js_runtime::JsRuntime; use crate::LoadError; use camino::Utf8PathBuf; use serde_json::json; @@ -44,6 +44,9 @@ pub enum FilterError { #[error("No messages can be processed due to an incorrect setting: {0}")] IncorrectSetting(String), + + #[error(transparent)] + Anyhow(#[from] anyhow::Error), } impl Pipeline { @@ -123,6 +126,10 @@ impl DateTime { pub fn tick_now(&self, tick_every_seconds: u64) -> bool { tick_every_seconds != 0 && (self.seconds % tick_every_seconds == 0) } + + pub fn json(&self) -> Value { + json!({"seconds": self.seconds, "nanoseconds": self.nanoseconds}) + } } impl TryFrom for DateTime { From e827a12aa4ecb406f8775ecf20040c5ad0dc6395 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Wed, 4 Jun 2025 19:20:23 +0200 Subject: [PATCH 21/50] Add benchmark options to tedge mqtt pub Signed-off-by: Didier Wenzek --- crates/core/tedge/src/cli/mqtt/cli.rs | 10 ++++++++++ crates/core/tedge/src/cli/mqtt/publish.rs | 12 +++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/crates/core/tedge/src/cli/mqtt/cli.rs b/crates/core/tedge/src/cli/mqtt/cli.rs index a99931e6415..5739d6ac541 100644 --- a/crates/core/tedge/src/cli/mqtt/cli.rs +++ b/crates/core/tedge/src/cli/mqtt/cli.rs @@ -30,6 +30,12 @@ pub enum TEdgeMqttCli { /// Retain flag #[clap(short, long = "retain")] retain: bool, + /// Repeat the message + #[clap(long)] + repeat: Option, + /// Pause between repeated messages (e.g., 60s, 1h) + #[clap(long, default_value = "1s")] + sleep: SecondsOrHumanTime, }, /// Subscribe a MQTT topic. @@ -69,6 +75,8 @@ impl BuildCommand for TEdgeMqttCli { message, qos, retain, + repeat, + sleep, } => MqttPublishCommand { host: config.mqtt.client.host.clone(), port: config.mqtt.client.port.into(), @@ -80,6 +88,8 @@ impl BuildCommand for TEdgeMqttCli { ca_file: auth_config.ca_file.clone(), ca_dir: auth_config.ca_dir, client_auth_config: auth_config.client, + count: repeat.unwrap_or(1), + sleep: sleep.duration(), } .into_boxed(), TEdgeMqttCli::Sub { diff --git a/crates/core/tedge/src/cli/mqtt/publish.rs b/crates/core/tedge/src/cli/mqtt/publish.rs index 48873ba61b1..9f19507ee94 100644 --- a/crates/core/tedge/src/cli/mqtt/publish.rs +++ b/crates/core/tedge/src/cli/mqtt/publish.rs @@ -22,6 +22,8 @@ pub struct MqttPublishCommand { pub ca_file: Option, pub ca_dir: Option, pub client_auth_config: Option, + pub count: u32, + pub sleep: std::time::Duration, } #[async_trait::async_trait] @@ -36,7 +38,15 @@ impl Command for MqttPublishCommand { } async fn execute(&self, _: TEdgeConfig) -> Result<(), MaybeFancy> { - Ok(publish(self).await?) + let mut i = 0; + loop { + publish(self).await?; + i += 1; + if i == self.count { + return Ok(()); + } + tokio::time::sleep(self.sleep).await; + } } } From c19623f18099c3368b65f0852702b3528a7c417e Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 6 Jun 2025 10:50:08 +0200 Subject: [PATCH 22/50] Add gen-mapper system tests Signed-off-by: Didier Wenzek --- ci/build_scripts/build.sh | 2 +- .../pipelines/add_timestamp.js | 11 ++ .../pipelines/measurements.toml | 7 + .../tedge_gen_mapper/pipelines/set_topic.js | 6 + .../tedge_gen_mapper/pipelines/te_to_c8y.js | 124 ++++++++++++++++++ .../tedge_gen_mapper/tedge_gen_mapper.robot | 27 ++++ 6 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/add_timestamp.js create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.toml create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/set_topic.js create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot diff --git a/ci/build_scripts/build.sh b/ci/build_scripts/build.sh index a9b66114c56..0e232293a37 100755 --- a/ci/build_scripts/build.sh +++ b/ci/build_scripts/build.sh @@ -101,7 +101,7 @@ BUILD_WITH="${BUILD_WITH:-zig}" COMMON_BUILD_OPTIONS=( "--release" ) -TOOLCHAIN="${TOOLCHAIN:-+1.78}" +TOOLCHAIN="${TOOLCHAIN:-+1.82}" # Note: Minimum version that is supported with riscv64gc-unknown-linux-gnu is 2.27 GLIBC_VERSION="${GLIBC_VERSION:-2.17}" RISCV_GLIBC_VERSION="${RISCV_GLIBC_VERSION:-2.27}" diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/add_timestamp.js b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/add_timestamp.js new file mode 100644 index 00000000000..f387141bfe3 --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/add_timestamp.js @@ -0,0 +1,11 @@ +export function process (timestamp, message) { + let payload = JSON.parse(message.payload) + if (!payload.time) { + payload.time = timestamp.seconds + (timestamp.nanoseconds / 1e9) + } + + return [{ + topic: message.topic, + payload: JSON.stringify(payload) + }] +} diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.toml b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.toml new file mode 100644 index 00000000000..a777177165b --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.toml @@ -0,0 +1,7 @@ +input_topics = ["te/+/+/+/+/m/+"] + +stages = [ + { filter = "add_timestamp.js" }, + { filter = "te_to_c8y.js", meta_topics = ["te/+/+/+/+/m/+/meta"] }, + { filter = "set_topic.js", config = { topic = "gen-mapper/c8y" } } +] diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/set_topic.js b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/set_topic.js new file mode 100644 index 00000000000..16ce07f0589 --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/set_topic.js @@ -0,0 +1,6 @@ +export function process (timestamp, message, config) { + return [{ + topic: config?.topic || "te/error", + payload: message.payload + }] +} diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js new file mode 100644 index 00000000000..ae2d8f157c1 --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js @@ -0,0 +1,124 @@ +/// Transform: +/// +/// ``` +/// [te/device/main///m/example] { +/// "time": "2020-10-15T05:30:47+00:00", +/// "temperature": 25, +/// "location": { +/// "latitude": 32.54, +/// "longitude": -117.67, +/// "altitude": 98.6 +/// }, +/// "pressure": 98 +/// } +/// ``` +/// +/// into +/// +/// ``` +/// [c8y/measurement/measurements/create] { +/// "time": "2020-10-15T05:30:47Z", +/// "type": "example", +/// "temperature": { +/// "temperature": { +/// "value": 25 +/// } +/// }, +/// "location": { +/// "latitude": { +/// "value": 32.54 +/// }, +/// "longitude": { +/// "value": -117.67 +/// }, +/// "altitude": { +/// "value": 98.6 +/// } +/// }, +/// "pressure": { +/// "pressure": { +/// "value": 98 +/// } +/// } +/// } +/// ``` +export function process(t, message, config) { + let topic_parts = message.topic.split( '/') + let type = topic_parts[6] + let payload = JSON.parse(message.payload) + + let c8y_msg = { + type: type + } + + let meta = (config || {})[`${message.topic}/meta`] || {} + + for (let [k, v] of Object.entries(payload)) { + let k_meta = (meta || {})[k] || {} + if (k === "time") { + let fragment = { time: v } + Object.assign(c8y_msg, fragment) + } + else if (typeof(v) === "number") { + if (Object.keys(k_meta).length>0) { + v = { value: v, ...k_meta } + } + let fragment = { [k]: { [k]: v } } + Object.assign(c8y_msg, fragment) + } else for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_k_meta = k_meta[sub_k] + if (typeof(sub_v) === "number") { + if (sub_k_meta) { + sub_v = { value: sub_v, ...sub_k_meta } + } + let fragment = { [k]: { [sub_k]: sub_v } } + Object.assign(c8y_msg, fragment) + } + } + } + + return [{ + topic: "c8y/measurement/measurements/create", + payload: JSON.stringify(c8y_msg) + }] +} + +/// Update the config with measurement metadata. +/// +/// These metadata are expected to have the same shape of the actual values. +/// +/// ``` +/// [te/device/main///m/example/meta] { "temperature": { "unit": "°C" }} +/// ``` +/// +/// and: +/// ``` +/// [te/device/main///m/example] { "temperature": { "unit": 23 }} +/// ``` +/// +/// will be merged by the process function into: +/// ``` +/// [c8y/measurement/measurements/create] { +/// "type": "example", +/// "temperature": { +/// "temperature": { +/// "value": 23, +/// "unit": "°C" +/// } +/// } +/// } +/// ``` +export function update_config(message, config) { + let type = message.topic + let metadata = JSON.parse(message.payload) + + let fragment = { + [type]: metadata + } + if (!config) { + config = {} + } + Object.assign(config, fragment) + + return config +} diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot new file mode 100644 index 00000000000..67ed106835e --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot @@ -0,0 +1,27 @@ +*** Settings *** +Library ThinEdgeIO + +Test Setup Custom Setup +Test Teardown Get Logs + +Test Tags theme:tedge_mapper + +*** Test Cases *** +Add missing timestamps + Execute Command tedge mqtt pub te/device/main// '{}' + ${transformed_msg} Should Have MQTT Messages gen-mapper/c8y + Should Contain ${transformed_msg} item=time + +*** Keywords *** +Custom Setup + ${DEVICE_SN}= Setup + Set Suite Variable $DEVICE_SN + Copy Configuration Files + Start Generic Mapper + +Copy Configuration Files + Execute Command mkdir /etc/tedge/gen-mapper/ + ThinEdgeIO.Transfer To Device ${CURDIR}/pipelines/* /etc/tedge/gen-mapper/ + +Start Generic Mapper + Execute Command nohup tedge run tedge-mapper gen & From 61270b4335aa50324bb9c0df68a4935728eed108 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 6 Jun 2025 19:42:00 +0200 Subject: [PATCH 23/50] Add a reference guide Signed-off-by: Didier Wenzek --- docs/src/references/mappers/gen-mapper.md | 168 ++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 docs/src/references/mappers/gen-mapper.md diff --git a/docs/src/references/mappers/gen-mapper.md b/docs/src/references/mappers/gen-mapper.md new file mode 100644 index 00000000000..2696ddf73e5 --- /dev/null +++ b/docs/src/references/mappers/gen-mapper.md @@ -0,0 +1,168 @@ +--- +title: Generic Mapper +tags: [Reference, Mappers, Cloud] +sidebar_position: 2 +draft: true +--- + +import ProposalBanner from '@site/src/components/ProposalBanner' + + + +:::note +This section is actually a design document. +It includes a reference guide for the POC, but also proposes a plan toward a generic mapper. +::: + +## Motivation + +In theory, %%te%% users can implement customized mappers to transform data published on the MQTT bus +or to interact with the cloud. In practice, they don't. +Implementing a mapper is costly while what is provided out-the-box by %%te%% already meets most requirements. +The need is not to write new mappers but to adapt existing ones. + +The aim of the generic mapper it to let users extend and adapt the mappers with their own filtering and mapping rules, +leveraging the core mapping rules and mapper mechanisms (bridge connections, HTTP proxies, operations). + +## Vision + +The %%te%% mappers for Cumulocity, Azure, AWS and Collectd are implemented on top of a so-called generic mapper +which is used to drive all MQTT message transformations. +- Transformations are implemented as pipelines consuming MQTT messages, feeding a chain of filters and producing MQTT messages. + - `MQTT sub| filter-1 | filter-2 | ... | filter-n | MQTT pub` +- A pipeline can combine builtin and user-provided filters. +- The user can configure all the transformations used by a mapper, + editing MQTT sources, pipelines, filters and MQTT sinks. +- By contrast with the current implementation, where the translation of measurements from %%te%% JSON to Cumulocity JSON + is fully hard-coded, with the generic mapper a user can re-use the core of this transformation while adding customized steps: + - consuming measurement from a non-standard topic + - filtering out part of the measurements + - normalizing units + - adding units read from some config + - producing transformed measurements on a non-standard topic. + +## POC reference + +- The generic mapper loads pipeline and filters stored in `/etc/tedge/gen-mapper/`. +- A pipeline is defined by a TOML file with `.toml` extension. +- A filter is defined by a Javascript file with `.js` extension. +- The definition of pipeline must provide a list of MQTT topics to subscribe to. + - The pipeline will be feed with all the messages received on these topics. +- A pipeline definition also provides a list of stages. + - Each stage is built from a javascript and is possibly given a config (arbitrary json that will be passed to the script) + - Each stage can also subscribe to a list of MQTT topics (which messages will be passed to the script to update its config) + +```toml +input_topics = ["te/+/+/+/+/m/+"] + +stages = [ + { filter = "add_timestamp.js" }, + { filter = "drop_stragglers.js", config = { max_delay = 60 } }, + { filter = "te_to_c8y.js", meta_topics = ["te/+/+/+/+/m/+/meta"] } +] +``` + +- A filter has to export at least one `process` function. + - `process(t: Timestamp, msg: Message, config: Json) -> Vec` + - This function is called for each message to be transformed + - The arguments passed to the function are: + - The current time as `{ seconds: u64, nanoseconds: u32 }` + - The message `{ topic: string, payload: string }` + - The config as read from the pipeline config or updated by the script + - The function is expected to return zero, one or many transformed messages `[{ topic: string, payload: string }]` + - An exception can be thrown if the input message cannot be transformed. +- A filter can also export an `update_config` function + - This function is called on each message received on the `meta_topics` as defined in the config. + - The arguments are: + - The message to be interpreted as a config update `{ topic: string, payload: string }` + - The current config + - The returned value (an arbitrary JSON value) is then used as the new config for the filter. +- A filter can also export a `tick` function + - This function is called at a regular pace with the current time and config. + - The filter can then return zero, one or many transformed messages + - By sharing an internal state between the `process` and `tick` functions, + the filter can implement aggregations over a time window. + When messages are received they are pushed by the `process` function into that state + and the final outcome is extracted by the `tick` function at the end of the time window. + +## First release + +While the POC provides a generic mapper that is fully independent of the legacy mappers, +the plan is not to abandon the latter in favor of the former +but to revisit the legacy mappers to include the ability for users to add their own mapping rules. + +To be lovable, the first release of an extensible mapper should at least: + +- be a drop-in replacement of the current mapper (for c8y, aws, az or collect) +- feature the ability to customize MEA processing by combining builtin filters with user-provided functions written in JavaScript +- provide tools to create, test, monitor and debug filters and pipelines +- be stable enough that user-defined filters will still work without changes with future releases. + +To keep things simple for the first release, the following questions are deferred: + +- Could a generic mapper let users define bridge rules as well as message transformation pipelines? +- Does it make sense to run such a mapper on child-devices? +- Could a pipeline send HTTP messages? Or could a filter tell the runtime to send messages over HTTP? +- How to handle binary payloads on the MQTT bus? +- Could operations be managed is a similar way with user-provided functions to transform commands? +- To handle operations, would the plugins be expanded to do more complex things like HTTP calls, file-system interactions, etc.? +- What are the pros and cons to persist filter states? +- Split a pipeline, forwarding transformed messages to different pipelines for further processing + +### API + +The POC expects the filter to implement a bunch of functions. This gives a quite expressive interface +(filtering, mapping, splitting, dynamic configuration, aggregation over time windows), but at the cost of some complexity. + +- `process(t: Timestamp, msg: Message, config: Json) -> Vec` +- `tick(t: Timestamp) -> Vec` +- `update_config(msg: Message, config: Json) -> Json` + +An alternative is to let the user implement more specific functions with simpler type signatures: + +- `filter(msg: Message, config: Json) -> bool` +- `map(msg: Message, config: Json) -> Message` +- `filter_map(msg: Message, config: Json) -> Option` +- `flat_map(msg: Message, config: Json) -> Vec` + +One can also rearrange the argument order for these functions, +making life easier when a transformation does need a config or the current time +leveraging that one can pass more arguments than declared to a javascript function: + +- `process(msg: Message, config: Json, t: Timestamp) -> Vec` +- `process(msg: Message, config: Json) -> Vec` +- `process(msg: Message) -> Vec` + +One can even use a bit further the flexibility of javascript, to let the process function freely return: +- An array of message objects +- A single message object +- A null value interpreted as no messages +- A boolean + +Other ideas to explore to make the API more flexible: + +- Interaction with the entity store and tedge config. +- Allow a pipeline to subscribe to topics related to the device/entity it is running on +- Feed filters with message excerpts as done for the workflows + +### Devops tools + +The flexibility to customize MQTT message processing with user-provided functions comes with risks: +- a filter might not behave as expected, +- pipelines might be overlapping or conflicting, possibly sending duplicate messages or creating infinite loops +- builtin pipelines might be accidentally disconnected or broken +- a filter might introduce a performance bottleneck. + +To help mitigating these risks, the `tedge mapping` sub-commands provide the tools to test, monitor and debug filters and pipelines. + +- `tedge mapping flow [topic]` displays pipelines and filters messages received on this topic will flow through + - can be used with a set of pipelines not configured yet for a mapper +- `tedge mapping test [filter]` feeds a filter or pipeline with input messages and produces the transformed output messages + - allow users to run an assertion based on the input/output of a filter + - ability to pipe `tedge mqtt sub` and `tedge mapping test` + - control of the timestamps + - test aggregation over ticks + - can be used with a set of pipelines not configured yet for a mapper +- `tedge mapping stats [pipeline]` returns statistics on the messages processed by a pipeline + - count message in, message out + - processing time min, median, max per filter From 4edc7750a4baff5eb77190d05c6664945cb70260 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 20 Jun 2025 10:41:04 +0200 Subject: [PATCH 24/50] Implement a circuit breaker filter Signed-off-by: Didier Wenzek --- .../pipelines/circuit-breaker.js | 65 +++++++++++++++++++ .../tedge_gen_mapper/pipelines/loop.toml | 7 ++ 2 files changed, 72 insertions(+) create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/circuit-breaker.js create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/loop.toml diff --git a/crates/extensions/tedge_gen_mapper/pipelines/circuit-breaker.js b/crates/extensions/tedge_gen_mapper/pipelines/circuit-breaker.js new file mode 100644 index 00000000000..3ecb09e4bf4 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/circuit-breaker.js @@ -0,0 +1,65 @@ +// A filter that let messages go through, unless too many messages are received within a given period +// +// This filter is configured by the following settings: +// - tick_every_seconds: the frequency at which the sliding window is moved +// - tick_count: size of the time windows +// - too_many: how many messages is too many (received during the last tick_count*tick_every_seconds seconds) +// - back_to_normal: how many messages is okay to reactivate the filter if bellow +// - message_on_too_many: message sent when the upper threshold is crossed +// - message_on_back_to_normal: message sent when the lower threshold is crossed +// - stats_topic: topic for statistic messages +class State { + static open = false + static total = 0 + static batch = [0] +} + + +export function process (timestamp, message, config) { + State.total += 1 + State.batch[0] += 1 + if (State.open) { + let back_to_normal = config?.back_to_normal || 100 + if (State.total < back_to_normal) { + State.open = false + if (config?.message_on_back_to_normal) { + return [config?.message_on_back_to_normal, message] + } else { + return [message] + } + } else { + return [] + } + } else { + let too_many = config?.too_many || 1000 + if (State.total < too_many) { + return [message] + } else { + State.open = true + if (config?.message_on_too_many) { + return [config?.message_on_too_many] + } else { + return [] + } + } + } +} + + +export function tick(timestamp, config) { + let max_batch_count = config?.tick_count || 10 + let new_batch_count = State.batch.unshift(0) + if (new_batch_count > max_batch_count) { + State.total -= State.batch.pop() + } + + if (config?.stats_topic) { + return [{ + topic: config?.stats_topic, + payload: `{"circuit-breaker-open": ${State.open}, "total": ${State.total}, "batch": ${State.batch}}` + }] + } else { + return [] + } + +} \ No newline at end of file diff --git a/crates/extensions/tedge_gen_mapper/pipelines/loop.toml b/crates/extensions/tedge_gen_mapper/pipelines/loop.toml new file mode 100644 index 00000000000..64304daf4a3 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/loop.toml @@ -0,0 +1,7 @@ +# This pipeline is on purpose looping: the messages are published to the same topic +input_topics = ["loopback/#"] + +stages = [ + { filter = "add_timestamp.js" }, + { filter = "circuit-breaker.js", tick_every_seconds = 1, config = { stats_topic = "te/error", too_many = 10000, message_on_too_many = { topic = "te/device/main///a/too-many-messages", payload = "too many messages" }, message_on_back_to_normal = { topic = "te/device/main///a/too-many-messages", payload = "back to normal" } } } +] From 96a056fb0c9ca2084d47b9ecea574e5147f6bd7c Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 20 Jun 2025 16:37:30 +0200 Subject: [PATCH 25/50] Add a filter to compute average over a time window Signed-off-by: Didier Wenzek --- .../tedge_gen_mapper/pipelines/average.js | 99 +++++++++++++++++++ .../tedge_gen_mapper/pipelines/collectd.toml | 2 +- 2 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/average.js diff --git a/crates/extensions/tedge_gen_mapper/pipelines/average.js b/crates/extensions/tedge_gen_mapper/pipelines/average.js new file mode 100644 index 00000000000..1a3f1c7e8dd --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/average.js @@ -0,0 +1,99 @@ +// Compute the average value of a series of measurements received during a time windows +// - Take care of the topic: messages received over different topics are not mixed +// - Ignore messages which are not formated as thin-edge JSON +// - Ignore values which are not numbers +// - Use the first timestamp as the timestamp for the aggregate +class State { + static agg_for_topic = {} +} + +export function process (timestamp, message) { + let topic = message.topic + let payload = JSON.parse(message.payload) + let agg_payload = State.agg_for_topic[topic] + + if (agg_payload) { + for (let [k, v] of Object.entries(payload)) { + let agg = agg_payload[k] + if (k === "time") { + if (!agg) { + let fragment = {time: v} + Object.assign(agg_payload, fragment) + } + } else if (typeof (v) === "number") { + if (!agg) { + let fragment = {k: {sum: v, count: 1}} + Object.assign(agg_payload, fragment) + } else { + agg.sum += v + agg.count += 1 + } + } else { + if (!agg) { + for (let [sub_k, sub_v] of Object.entries(v)) { + let fragment = { [k]: { [sub_k]: { sum: sub_v, count: 1 } } } + Object.assign(agg_payload, fragment) + } + } else { + for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_agg = agg_payload[sub_k] + if (!sub_agg) { + let fragment = {k: { [sub_k]: { sum: sub_v, count: 1 } } } + Object.assign(agg_payload, fragment) + } else { + sub_agg.sum += sub_v + sub_agg.count += 1 + } + } + } + } + } + } else { + let agg_payload = {} + for (let [k, v] of Object.entries(payload)) { + if (k === "time") { + let fragment = { time: v } + Object.assign(agg_payload, fragment) + } + else if (typeof(v) === "number") { + let fragment = { k: { sum: v, count: 1 } } + Object.assign(agg_payload, fragment) + } else for (let [sub_k, sub_v] of Object.entries(v)) { + let fragment = { [k]: { [sub_k]: { sum: sub_v, count: 1 } } } + Object.assign(agg_payload, fragment) + } + } + State.agg_for_topic[topic] = agg_payload + } + + return [] +} + +export function tick() { + let messages = [] + + for (let [topic, agg] of Object.entries(State.agg_for_topic)) { + let payload = {} + for (let [k, v] of Object.entries(agg)) { + if (k === "time") { + let fragment = { time: v } + Object.assign(payload, fragment) + } + else if (v.sum && v.count) { + let fragment = { k: v.sum / v.count } + Object.assign(payload, fragment) + } else for (let [sub_k, sub_v] of Object.entries(v)) { + let fragment = { [k]: { [sub_k]: sub_v.sum / sub_v.count } } + Object.assign(payload, fragment) + } + } + + messages.push ({ + topic: topic, + payload: JSON.stringify(payload) + }) + } + + State.agg_for_topic = {} + return messages +} \ No newline at end of file diff --git a/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml b/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml index 44feb5b3323..05330b870f8 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml +++ b/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml @@ -2,5 +2,5 @@ input_topics = ["collectd/+/+/+"] stages = [ { filter = "collectd-to-te.js" }, - { filter = "group_by_timestamp.js", tick_every_seconds = 3 } + { filter = "average.js", tick_every_seconds = 10 } ] From 44873ae860ccfea9f1b21b2141859870e10c421d Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Wed, 25 Jun 2025 19:35:04 +0200 Subject: [PATCH 26/50] Extract message processing from gen-mapper actor The goal is to be able to use the message processor from within the tedge cli command, notably for testing pipelines and filters. Signed-off-by: Didier Wenzek --- crates/core/tedge_mapper/src/gen/mod.rs | 1 - .../extensions/tedge_gen_mapper/src/actor.rs | 77 +------ crates/extensions/tedge_gen_mapper/src/lib.rs | 111 +--------- .../tedge_gen_mapper/src/runtime.rs | 205 ++++++++++++++++++ 4 files changed, 227 insertions(+), 167 deletions(-) create mode 100644 crates/extensions/tedge_gen_mapper/src/runtime.rs diff --git a/crates/core/tedge_mapper/src/gen/mod.rs b/crates/core/tedge_mapper/src/gen/mod.rs index acdaa3a6961..e89644506de 100644 --- a/crates/core/tedge_mapper/src/gen/mod.rs +++ b/crates/core/tedge_mapper/src/gen/mod.rs @@ -18,7 +18,6 @@ impl TEdgeComponent for GenMapper { let mut fs_actor = FsWatchActorBuilder::new(); let mut gen_mapper = GenMapperBuilder::try_new("/etc/tedge/gen-mapper").await?; - gen_mapper.load().await; gen_mapper.connect(&mut mqtt_actor); gen_mapper.connect_fs(&mut fs_actor); diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index 121c6b3d38c..3497e3f6c9e 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -1,14 +1,10 @@ -use crate::config::PipelineConfig; -use crate::js_runtime::JsRuntime; use crate::pipeline::DateTime; use crate::pipeline::Message; -use crate::pipeline::Pipeline; +use crate::runtime::MessageProcessor; use crate::InputMessage; use crate::OutputMessage; use async_trait::async_trait; use camino::Utf8PathBuf; -use std::collections::HashMap; -use std::path::PathBuf; use std::sync::Arc; use std::sync::Mutex; use tedge_actors::Actor; @@ -23,14 +19,11 @@ use tedge_mqtt_ext::TopicFilter; use tokio::time::interval; use tokio::time::Duration; use tracing::error; -use tracing::info; pub struct GenMapper { pub(super) messages: SimpleMessageBox, - pub(super) pipelines: HashMap, pub(super) subscriptions: Arc>, - pub(super) js_runtime: JsRuntime, - pub(super) config_dir: PathBuf, + pub(super) processor: MessageProcessor, } #[async_trait] @@ -60,9 +53,9 @@ impl Actor for GenMapper { continue; }; if matches!(path.extension(), Some("js" | "ts")) { - self.reload_filter(path).await; + self.processor.reload_filter(path).await; } else if path.extension() == Some("toml") { - self.reload_pipeline(path).await; + self.processor.reload_pipeline(path).await; self.send_updated_subscriptions().await?; } }, @@ -79,51 +72,6 @@ impl Actor for GenMapper { } impl GenMapper { - async fn reload_filter(&mut self, path: Utf8PathBuf) { - for pipeline in self.pipelines.values_mut() { - for stage in &mut pipeline.stages { - if stage.filter.path() == path { - match self.js_runtime.load_file(&path).await { - Ok(()) => { - info!("Reloaded filter {path}"); - } - Err(e) => { - error!("Failed to reload filter {path}: {e}"); - return; - } - } - } - } - } - } - - async fn reload_pipeline(&mut self, path: Utf8PathBuf) { - for pipeline in self.pipelines.values_mut() { - if pipeline.source == path { - let Ok(source) = tokio::fs::read_to_string(&path).await else { - error!("Failed to read updated filter {path}"); - break; - }; - let config: PipelineConfig = match toml::from_str(&source) { - Ok(config) => config, - Err(e) => { - error!("Failed to parse toml for updated filter {path}: {e}"); - break; - } - }; - match config.compile(&self.js_runtime, &self.config_dir, path.clone()) { - Ok(p) => { - *pipeline = p; - info!("Reloaded pipeline {path}"); - } - Err(e) => { - error!("Failed to load updated pipeline {path}: {e}") - } - }; - } - } - } - async fn send_updated_subscriptions(&mut self) -> Result<(), RuntimeError> { let topics = self.update_subscriptions(); let diff = SubscriptionDiff::new(&topics, &TopicFilter::empty()); @@ -135,19 +83,14 @@ impl GenMapper { fn update_subscriptions(&self) -> TopicFilter { let mut topics = self.subscriptions.lock().unwrap(); - for pipeline in self.pipelines.values() { - topics.add_all(pipeline.topics()) - } + topics.add_all(self.processor.subscriptions()); topics.clone() } async fn filter(&mut self, message: Message) -> Result<(), RuntimeError> { let timestamp = DateTime::now(); - for (pipeline_id, pipeline) in self.pipelines.iter_mut() { - match pipeline - .process(&self.js_runtime, ×tamp, &message) - .await - { + for (pipeline_id, pipeline_messages) in self.processor.process(×tamp, &message).await { + match pipeline_messages { Ok(messages) => { for message in messages { match MqttMessage::try_from(message) { @@ -174,10 +117,10 @@ impl GenMapper { async fn tick(&mut self) -> Result<(), RuntimeError> { let timestamp = DateTime::now(); if timestamp.seconds % 300 == 0 { - self.js_runtime.dump_memory_stats().await; + self.processor.dump_memory_stats().await; } - for (pipeline_id, pipeline) in self.pipelines.iter_mut() { - match pipeline.tick(&self.js_runtime, ×tamp).await { + for (pipeline_id, pipeline_messages) in self.processor.tick(×tamp).await { + match pipeline_messages { Ok(messages) => { for message in messages { match MqttMessage::try_from(message) { diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index 130094195a5..cad842c304d 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -3,14 +3,10 @@ mod config; mod js_filter; mod js_runtime; mod pipeline; +mod runtime; use crate::actor::GenMapper; -use crate::config::PipelineConfig; -use crate::js_runtime::JsRuntime; -use crate::pipeline::Pipeline; -use camino::Utf8Path; -use camino::Utf8PathBuf; -use std::collections::HashMap; +use crate::runtime::MessageProcessor; use std::convert::Infallible; use std::path::Path; use std::path::PathBuf; @@ -31,106 +27,27 @@ use tedge_mqtt_ext::MqttMessage; use tedge_mqtt_ext::PublishOrSubscribe; use tedge_mqtt_ext::SubscriptionDiff; use tedge_mqtt_ext::TopicFilter; -use tokio::fs::read_dir; -use tokio::fs::read_to_string; use tracing::error; -use tracing::info; fan_in_message_type!(InputMessage[MqttMessage, FsWatchEvent]: Clone, Debug, Eq, PartialEq); fan_in_message_type!(OutputMessage[MqttMessage, SubscriptionDiff]: Clone, Debug, Eq, PartialEq); pub struct GenMapperBuilder { - config_dir: PathBuf, message_box: SimpleMessageBoxBuilder, - pipelines: HashMap, - pipeline_specs: HashMap, subscriptions: Arc>, - js_runtime: JsRuntime, + processor: MessageProcessor, } impl GenMapperBuilder { pub async fn try_new(config_dir: impl AsRef) -> Result { - let config_dir = config_dir.as_ref().to_owned(); - let js_runtime = JsRuntime::try_new().await?; + let processor = MessageProcessor::try_new(config_dir).await?; Ok(GenMapperBuilder { - config_dir, message_box: SimpleMessageBoxBuilder::new("GenMapper", 16), - pipelines: HashMap::default(), - pipeline_specs: HashMap::default(), subscriptions: Arc::new(Mutex::new(TopicFilter::empty())), - js_runtime, + processor, }) } - pub async fn load(&mut self) { - let Ok(mut entries) = read_dir(&self.config_dir).await.map_err(|err| - error!(target: "MAPPING", "Failed to read filters from {}: {err}", self.config_dir.display()) - ) else { - return; - }; - - while let Ok(Some(entry)) = entries.next_entry().await { - let Some(path) = Utf8Path::from_path(&entry.path()).map(|p| p.to_path_buf()) else { - error!(target: "MAPPING", "Skipping non UTF8 path: {}", entry.path().display()); - continue; - }; - if let Ok(file_type) = entry.file_type().await { - if file_type.is_file() { - match path.extension() { - Some("toml") => { - info!(target: "MAPPING", "Loading pipeline: {path}"); - if let Err(err) = self.load_pipeline(path).await { - error!(target: "MAPPING", "Failed to load pipeline: {err}"); - } - } - Some("js") | Some("ts") => { - info!(target: "MAPPING", "Loading filter: {path}"); - if let Err(err) = self.load_filter(path).await { - error!(target: "MAPPING", "Failed to load filter: {err}"); - } - } - _ => { - info!(target: "MAPPING", "Skipping file which type is unknown: {path}"); - } - } - } - } - } - - // Done here to ease the computation of the topics to subscribe to - // as these topics have to be known when connect is called - self.compile() - } - - async fn load_pipeline(&mut self, file: impl AsRef) -> Result<(), LoadError> { - if let Some(name) = file.as_ref().file_name() { - let specs = read_to_string(file.as_ref()).await?; - let pipeline: PipelineConfig = toml::from_str(&specs)?; - self.pipeline_specs - .insert(name.to_string(), (file.as_ref().to_owned(), pipeline)); - } - - Ok(()) - } - - async fn load_filter(&mut self, file: impl AsRef) -> Result<(), LoadError> { - self.js_runtime.load_file(file.as_ref()).await?; - Ok(()) - } - - fn compile(&mut self) { - for (name, (source, specs)) in self.pipeline_specs.drain() { - match specs.compile(&self.js_runtime, &self.config_dir, source) { - Ok(pipeline) => { - let _ = self.pipelines.insert(name, pipeline); - } - Err(err) => { - error!(target: "MAPPING", "Failed to compile pipeline {name}: {err}") - } - } - } - } - pub fn connect( &mut self, mqtt: &mut (impl MessageSource + MessageSink), @@ -150,17 +67,15 @@ impl GenMapperBuilder { } pub fn connect_fs(&mut self, fs: &mut impl MessageSource) { - fs.connect_mapped_sink(self.config_dir.clone(), &self.message_box, |msg| { - Some(InputMessage::FsWatchEvent(msg)) - }); + fs.connect_mapped_sink( + self.processor.config_dir.clone(), + &self.message_box, + |msg| Some(InputMessage::FsWatchEvent(msg)), + ); } fn topics(&self) -> TopicFilter { - let mut topics = TopicFilter::empty(); - for pipeline in self.pipelines.values() { - topics.add_all(pipeline.topics()) - } - topics + self.processor.subscriptions() } } @@ -180,10 +95,8 @@ impl Builder for GenMapperBuilder { fn build(self) -> GenMapper { GenMapper { messages: self.message_box.build(), - pipelines: self.pipelines, subscriptions: self.subscriptions, - js_runtime: self.js_runtime, - config_dir: self.config_dir, + processor: self.processor, } } } diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs new file mode 100644 index 00000000000..31417716f9d --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -0,0 +1,205 @@ +use crate::config::PipelineConfig; +use crate::js_runtime::JsRuntime; +use crate::pipeline::DateTime; +use crate::pipeline::FilterError; +use crate::pipeline::Message; +use crate::pipeline::Pipeline; +use crate::LoadError; +use camino::Utf8Path; +use camino::Utf8PathBuf; +use std::collections::HashMap; +use std::path::Path; +use std::path::PathBuf; +use tedge_mqtt_ext::TopicFilter; +use tokio::fs::read_dir; +use tokio::fs::read_to_string; +use tracing::error; +use tracing::info; + +pub struct MessageProcessor { + pub(super) config_dir: PathBuf, + pub(super) pipelines: HashMap, + pub(super) js_runtime: JsRuntime, +} + +impl MessageProcessor { + pub async fn try_new(config_dir: impl AsRef) -> Result { + let config_dir = config_dir.as_ref().to_owned(); + let mut js_runtime = JsRuntime::try_new().await?; + let mut pipeline_specs = PipelineSpecs::default(); + pipeline_specs.load(&mut js_runtime, &config_dir).await; + let pipelines = pipeline_specs.compile(&mut js_runtime, &config_dir); + + Ok(MessageProcessor { + config_dir, + pipelines, + js_runtime, + }) + } + + pub fn subscriptions(&self) -> TopicFilter { + let mut topics = TopicFilter::empty(); + for pipeline in self.pipelines.values() { + topics.add_all(pipeline.topics()) + } + topics + } + + pub async fn process( + &mut self, + timestamp: &DateTime, + message: &Message, + ) -> Vec<(String, Result, FilterError>)> { + let mut out_messages = vec![]; + for (pipeline_id, pipeline) in self.pipelines.iter_mut() { + let pipeline_output = pipeline + .process(&self.js_runtime, ×tamp, &message) + .await; + out_messages.push((pipeline_id.clone(), pipeline_output)); + } + out_messages + } + + pub async fn tick( + &mut self, + timestamp: &DateTime, + ) -> Vec<(String, Result, FilterError>)> { + let mut out_messages = vec![]; + for (pipeline_id, pipeline) in self.pipelines.iter_mut() { + let pipeline_output = pipeline.tick(&self.js_runtime, ×tamp).await; + out_messages.push((pipeline_id.clone(), pipeline_output)); + } + out_messages + } + + pub async fn dump_memory_stats(&self) { + self.js_runtime.dump_memory_stats().await; + } + + pub async fn reload_filter(&mut self, path: Utf8PathBuf) { + for pipeline in self.pipelines.values_mut() { + for stage in &mut pipeline.stages { + if stage.filter.path() == path { + match self.js_runtime.load_file(&path).await { + Ok(()) => { + info!("Reloaded filter {path}"); + } + Err(e) => { + error!("Failed to reload filter {path}: {e}"); + return; + } + } + } + } + } + } + + pub async fn reload_pipeline(&mut self, path: Utf8PathBuf) { + for pipeline in self.pipelines.values_mut() { + if pipeline.source == path { + let Ok(source) = tokio::fs::read_to_string(&path).await else { + error!("Failed to read updated filter {path}"); + break; + }; + let config: PipelineConfig = match toml::from_str(&source) { + Ok(config) => config, + Err(e) => { + error!("Failed to parse toml for updated filter {path}: {e}"); + break; + } + }; + match config.compile(&self.js_runtime, &self.config_dir, path.clone()) { + Ok(p) => { + *pipeline = p; + info!("Reloaded pipeline {path}"); + } + Err(e) => { + error!("Failed to load updated pipeline {path}: {e}") + } + }; + } + } + } +} + +#[derive(Default)] +struct PipelineSpecs { + pipeline_specs: HashMap, +} + +impl PipelineSpecs { + pub async fn load(&mut self, js_runtime: &mut JsRuntime, config_dir: &PathBuf) { + let Ok(mut entries) = read_dir(config_dir).await.map_err(|err| + error!(target: "MAPPING", "Failed to read filters from {}: {err}", config_dir.display()) + ) else { + return; + }; + + while let Ok(Some(entry)) = entries.next_entry().await { + let Some(path) = Utf8Path::from_path(&entry.path()).map(|p| p.to_path_buf()) else { + error!(target: "MAPPING", "Skipping non UTF8 path: {}", entry.path().display()); + continue; + }; + if let Ok(file_type) = entry.file_type().await { + if file_type.is_file() { + match path.extension() { + Some("toml") => { + info!(target: "MAPPING", "Loading pipeline: {path}"); + if let Err(err) = self.load_pipeline(path).await { + error!(target: "MAPPING", "Failed to load pipeline: {err}"); + } + } + Some("js") | Some("ts") => { + info!(target: "MAPPING", "Loading filter: {path}"); + if let Err(err) = self.load_filter(js_runtime, path).await { + error!(target: "MAPPING", "Failed to load filter: {err}"); + } + } + _ => { + info!(target: "MAPPING", "Skipping file which type is unknown: {path}"); + } + } + } + } + } + } + + async fn load_pipeline(&mut self, file: impl AsRef) -> Result<(), LoadError> { + if let Some(name) = file.as_ref().file_name() { + let specs = read_to_string(file.as_ref()).await?; + let pipeline: PipelineConfig = toml::from_str(&specs)?; + self.pipeline_specs + .insert(name.to_string(), (file.as_ref().to_owned(), pipeline)); + } + + Ok(()) + } + + async fn load_filter( + &mut self, + js_runtime: &mut JsRuntime, + file: impl AsRef, + ) -> Result<(), LoadError> { + js_runtime.load_file(file.as_ref()).await?; + Ok(()) + } + + fn compile( + mut self, + js_runtime: &JsRuntime, + config_dir: &PathBuf, + ) -> HashMap { + let mut pipelines = HashMap::new(); + for (name, (source, specs)) in self.pipeline_specs.drain() { + match specs.compile(js_runtime, config_dir, source) { + Ok(pipeline) => { + let _ = pipelines.insert(name, pipeline); + } + Err(err) => { + error!(target: "MAPPING", "Failed to compile pipeline {name}: {err}") + } + } + } + pipelines + } +} From 819e44c1b0e588fc878d88d6439c42e50f407fed Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Thu, 26 Jun 2025 13:18:17 +0200 Subject: [PATCH 27/50] Add cli commands to test user-defined mappings Signed-off-by: Didier Wenzek --- Cargo.lock | 1 + crates/core/tedge/Cargo.toml | 1 + crates/core/tedge/src/cli/mapping/cli.rs | 104 ++++++++++++++++++ crates/core/tedge/src/cli/mapping/list.rs | 47 ++++++++ crates/core/tedge/src/cli/mapping/mod.rs | 5 + crates/core/tedge/src/cli/mapping/test.rs | 61 ++++++++++ crates/core/tedge/src/cli/mod.rs | 6 + .../tedge_gen_mapper/src/js_filter.rs | 6 +- crates/extensions/tedge_gen_mapper/src/lib.rs | 4 +- .../tedge_gen_mapper/src/pipeline.rs | 8 +- .../tedge_gen_mapper/src/runtime.rs | 31 ++++-- 11 files changed, 253 insertions(+), 21 deletions(-) create mode 100644 crates/core/tedge/src/cli/mapping/cli.rs create mode 100644 crates/core/tedge/src/cli/mapping/list.rs create mode 100644 crates/core/tedge/src/cli/mapping/mod.rs create mode 100644 crates/core/tedge/src/cli/mapping/test.rs diff --git a/Cargo.lock b/Cargo.lock index ff69243a0da..50a061f5293 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4573,6 +4573,7 @@ dependencies = [ "tedge-write", "tedge_api", "tedge_config", + "tedge_gen_mapper", "tedge_test_utils", "tedge_utils", "tempfile", diff --git a/crates/core/tedge/Cargo.toml b/crates/core/tedge/Cargo.toml index fd930cae168..83315dc2197 100644 --- a/crates/core/tedge/Cargo.toml +++ b/crates/core/tedge/Cargo.toml @@ -49,6 +49,7 @@ strum_macros = { workspace = true } tar = { workspace = true } tedge-agent = { workspace = true } tedge-apt-plugin = { workspace = true } +tedge_gen_mapper = { workspace = true } tedge-mapper = { workspace = true, default-features = false } tedge-watchdog = { workspace = true } tedge-write = { workspace = true } diff --git a/crates/core/tedge/src/cli/mapping/cli.rs b/crates/core/tedge/src/cli/mapping/cli.rs new file mode 100644 index 00000000000..729bf0ededb --- /dev/null +++ b/crates/core/tedge/src/cli/mapping/cli.rs @@ -0,0 +1,104 @@ +use crate::cli::mapping::list::ListCommand; +use crate::cli::mapping::test::TestCommand; +use crate::command::BuildCommand; +use crate::command::Command; +use crate::ConfigError; +use anyhow::anyhow; +use anyhow::Context; +use anyhow::Error; +use std::path::PathBuf; +use tedge_config::TEdgeConfig; +use tedge_gen_mapper::pipeline::Message; +use tedge_gen_mapper::MessageProcessor; + +#[derive(clap::Subcommand, Debug)] +pub enum TEdgeMappingCli { + /// List pipelines and filters + List { + /// Path to pipeline and filter specs + /// + /// Default to /etc/tedge/gen-mapper + #[clap(long)] + mapping_dir: Option, + + /// List pipelines processing messages published on this topic + /// + /// If none is provided, lists all the pipelines + #[clap(long)] + topic: Option, + }, + + /// Process message samples + Test { + /// Path to pipeline and filter specs + /// + /// Default to /etc/tedge/gen-mapper + #[clap(long)] + mapping_dir: Option, + + /// Path to the javascript filter or TOML pipeline definition + /// + /// If none is provided, applies all the matching pipelines + #[clap(long)] + filter: Option, + + /// Topic of the message sample + /// + /// If none is provided, messages are read from stdout + topic: Option, + + /// Payload of the message sample + /// + /// If none is provided, payloads are read from stdout + payload: Option, + }, +} + +impl BuildCommand for TEdgeMappingCli { + fn build_command(self, config: &TEdgeConfig) -> Result, ConfigError> { + match self { + TEdgeMappingCli::List { mapping_dir, topic } => { + let mapping_dir = mapping_dir.unwrap_or_else(|| Self::default_mapping_dir(config)); + Ok(ListCommand { mapping_dir, topic }.into_boxed()) + } + + TEdgeMappingCli::Test { + mapping_dir, + filter, + topic, + payload, + } => { + let mapping_dir = mapping_dir.unwrap_or_else(|| Self::default_mapping_dir(config)); + let message = match (topic, payload) { + (Some(topic), Some(payload)) => Some(Message { topic, payload }), + (Some(_), None) => Err(anyhow!("Missing sample payload"))?, + (None, Some(_)) => Err(anyhow!("Missing sample topic"))?, + (None, None) => None, + }; + Ok(TestCommand { + mapping_dir, + filter, + message, + } + .into_boxed()) + } + } + } +} + +impl TEdgeMappingCli { + fn default_mapping_dir(config: &TEdgeConfig) -> PathBuf { + config.root_dir().join("gen-mapper").into() + } + + pub async fn load_pipelines(mapping_dir: &PathBuf) -> Result { + MessageProcessor::try_new(mapping_dir) + .await + .with_context(|| { + format!( + "loading pipelines and filters from {}", + mapping_dir.display() + ) + }) + } +} diff --git a/crates/core/tedge/src/cli/mapping/list.rs b/crates/core/tedge/src/cli/mapping/list.rs new file mode 100644 index 00000000000..8c7577350df --- /dev/null +++ b/crates/core/tedge/src/cli/mapping/list.rs @@ -0,0 +1,47 @@ +use crate::cli::mapping::TEdgeMappingCli; +use crate::command::Command; +use crate::log::MaybeFancy; +use anyhow::Error; +use std::path::PathBuf; +use tedge_config::TEdgeConfig; +use tedge_gen_mapper::pipeline::Pipeline; + +pub struct ListCommand { + pub mapping_dir: PathBuf, + pub topic: Option, +} + +#[async_trait::async_trait] +impl Command for ListCommand { + fn description(&self) -> String { + format!( + "list pipelines and filters in {:}", + self.mapping_dir.display() + ) + } + + async fn execute(&self, _config: TEdgeConfig) -> Result<(), MaybeFancy> { + let processor = TEdgeMappingCli::load_pipelines(&self.mapping_dir).await?; + + match &self.topic { + Some(topic) => processor + .pipelines + .iter() + .filter(|(_, pipeline)| pipeline.topics().accept_topic_name(topic)) + .for_each(Self::display), + + None => processor.pipelines.iter().for_each(Self::display), + } + + Ok(()) + } +} + +impl ListCommand { + fn display((pipeline_id, pipeline): (&String, &Pipeline)) { + println!("{pipeline_id}"); + for stage in pipeline.stages.iter() { + println!("\t{}", stage.filter.path.display()); + } + } +} diff --git a/crates/core/tedge/src/cli/mapping/mod.rs b/crates/core/tedge/src/cli/mapping/mod.rs new file mode 100644 index 00000000000..753d9370f59 --- /dev/null +++ b/crates/core/tedge/src/cli/mapping/mod.rs @@ -0,0 +1,5 @@ +mod cli; +mod list; +mod test; + +pub use cli::TEdgeMappingCli; diff --git a/crates/core/tedge/src/cli/mapping/test.rs b/crates/core/tedge/src/cli/mapping/test.rs new file mode 100644 index 00000000000..2737720bc4d --- /dev/null +++ b/crates/core/tedge/src/cli/mapping/test.rs @@ -0,0 +1,61 @@ +use crate::cli::mapping::TEdgeMappingCli; +use crate::command::Command; +use crate::log::MaybeFancy; +use anyhow::Error; +use std::path::PathBuf; +use tedge_config::TEdgeConfig; +use tedge_gen_mapper::pipeline::*; + +pub struct TestCommand { + pub mapping_dir: PathBuf, + pub filter: Option, + pub message: Option, +} + +#[async_trait::async_trait] +impl Command for TestCommand { + fn description(&self) -> String { + format!( + "process message samples using pipelines and filters in {:}", + self.mapping_dir.display() + ) + } + + async fn execute(&self, _config: TEdgeConfig) -> Result<(), MaybeFancy> { + let mut processor = TEdgeMappingCli::load_pipelines(&self.mapping_dir).await?; + if let Some(message) = &self.message { + let timestamp = DateTime::now(); + match &self.filter { + Some(filter) => { + let filter_name = filter.display().to_string(); + print( + processor + .process_with_pipeline(&filter_name, ×tamp, message) + .await, + ) + } + None => processor + .process(×tamp, message) + .await + .into_iter() + .map(|(_, v)| v) + .for_each(print), + } + } + + Ok(()) + } +} + +fn print(messages: Result, FilterError>) { + match messages { + Ok(messages) => { + for message in messages { + println!("[{}] {}", message.topic, message.payload); + } + } + Err(err) => { + eprintln!("Error: {}", err) + } + } +} diff --git a/crates/core/tedge/src/cli/mod.rs b/crates/core/tedge/src/cli/mod.rs index 7d155ccbdd4..4c42a0efa7b 100644 --- a/crates/core/tedge/src/cli/mod.rs +++ b/crates/core/tedge/src/cli/mod.rs @@ -25,6 +25,7 @@ mod disconnect; mod http; mod init; pub mod log; +mod mapping; mod mqtt; mod reconnect; mod refresh_bridges; @@ -138,6 +139,10 @@ pub enum TEdgeOpt { #[clap(subcommand)] Http(http::TEdgeHttpCli), + /// Monitor and test mapping rules + #[clap(subcommand)] + Mapping(mapping::TEdgeMappingCli), + /// Run thin-edge services and plugins Run(ComponentOpt), @@ -208,6 +213,7 @@ impl BuildCommand for TEdgeOpt { TEdgeOpt::Mqtt(opt) => opt.build_command(config), TEdgeOpt::Http(opt) => opt.build_command(config), TEdgeOpt::Reconnect(opt) => opt.build_command(config), + TEdgeOpt::Mapping(opt) => opt.build_command(config), TEdgeOpt::Run(_) => { // This method has to be kept in sync with tedge::redirect_if_multicall() panic!("tedge mapper|agent|write commands are launched as multicall") diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 5e27b2b193d..2f4fc094a4f 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -14,9 +14,9 @@ use tracing::debug; #[derive(Clone)] pub struct JsFilter { - path: PathBuf, - config: JsonValue, - tick_every_seconds: u64, + pub path: PathBuf, + pub config: JsonValue, + pub tick_every_seconds: u64, } #[derive(Clone, Debug, Default)] diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index cad842c304d..84ac172a3f9 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -2,11 +2,11 @@ mod actor; mod config; mod js_filter; mod js_runtime; -mod pipeline; +pub mod pipeline; mod runtime; use crate::actor::GenMapper; -use crate::runtime::MessageProcessor; +pub use crate::runtime::MessageProcessor; use std::convert::Infallible; use std::path::Path; use std::path::PathBuf; diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index e9e993a03f2..a87daef8359 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -27,14 +27,14 @@ pub struct Stage { #[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Eq, PartialEq)] pub struct DateTime { - pub(crate) seconds: u64, - pub(crate) nanoseconds: u32, + pub seconds: u64, + pub nanoseconds: u32, } #[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Eq, PartialEq)] pub struct Message { - pub(crate) topic: String, - pub(crate) payload: String, + pub topic: String, + pub payload: String, } #[derive(thiserror::Error, Debug)] diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 31417716f9d..af0c0d9f053 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -17,8 +17,8 @@ use tracing::error; use tracing::info; pub struct MessageProcessor { - pub(super) config_dir: PathBuf, - pub(super) pipelines: HashMap, + pub config_dir: PathBuf, + pub pipelines: HashMap, pub(super) js_runtime: JsRuntime, } @@ -28,7 +28,7 @@ impl MessageProcessor { let mut js_runtime = JsRuntime::try_new().await?; let mut pipeline_specs = PipelineSpecs::default(); pipeline_specs.load(&mut js_runtime, &config_dir).await; - let pipelines = pipeline_specs.compile(&mut js_runtime, &config_dir); + let pipelines = pipeline_specs.compile(&js_runtime, &config_dir); Ok(MessageProcessor { config_dir, @@ -52,21 +52,32 @@ impl MessageProcessor { ) -> Vec<(String, Result, FilterError>)> { let mut out_messages = vec![]; for (pipeline_id, pipeline) in self.pipelines.iter_mut() { - let pipeline_output = pipeline - .process(&self.js_runtime, ×tamp, &message) - .await; + let pipeline_output = pipeline.process(&self.js_runtime, timestamp, message).await; out_messages.push((pipeline_id.clone(), pipeline_output)); } out_messages } + pub async fn process_with_pipeline( + &mut self, + pipeline_id: &String, + timestamp: &DateTime, + message: &Message, + ) -> Result, FilterError> { + let pipeline = self + .pipelines + .get_mut(pipeline_id) + .ok_or_else(|| anyhow::anyhow!("No such pipeline: {pipeline_id}"))?; + pipeline.process(&self.js_runtime, timestamp, message).await + } + pub async fn tick( &mut self, timestamp: &DateTime, ) -> Vec<(String, Result, FilterError>)> { let mut out_messages = vec![]; for (pipeline_id, pipeline) in self.pipelines.iter_mut() { - let pipeline_output = pipeline.tick(&self.js_runtime, ×tamp).await; + let pipeline_output = pipeline.tick(&self.js_runtime, timestamp).await; out_messages.push((pipeline_id.clone(), pipeline_output)); } out_messages @@ -184,11 +195,7 @@ impl PipelineSpecs { Ok(()) } - fn compile( - mut self, - js_runtime: &JsRuntime, - config_dir: &PathBuf, - ) -> HashMap { + fn compile(mut self, js_runtime: &JsRuntime, config_dir: &Path) -> HashMap { let mut pipelines = HashMap::new(); for (name, (source, specs)) in self.pipeline_specs.drain() { match specs.compile(js_runtime, config_dir, source) { From a104b2bef9930a3b58589b4c92d9172ba0ac2599 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Thu, 26 Jun 2025 14:57:36 +0200 Subject: [PATCH 28/50] Simplify gen-mapper subscription updates Signed-off-by: Didier Wenzek --- .../extensions/tedge_gen_mapper/src/actor.rs | 16 +++++----- crates/extensions/tedge_gen_mapper/src/lib.rs | 7 ++--- .../tedge_gen_mapper/src/runtime.rs | 12 ++++---- crates/extensions/tedge_mqtt_ext/src/trie.rs | 29 +++++++++++-------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index 3497e3f6c9e..e6b4eea94a8 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -5,8 +5,6 @@ use crate::InputMessage; use crate::OutputMessage; use async_trait::async_trait; use camino::Utf8PathBuf; -use std::sync::Arc; -use std::sync::Mutex; use tedge_actors::Actor; use tedge_actors::MessageReceiver; use tedge_actors::RuntimeError; @@ -22,7 +20,7 @@ use tracing::error; pub struct GenMapper { pub(super) messages: SimpleMessageBox, - pub(super) subscriptions: Arc>, + pub(super) subscriptions: TopicFilter, pub(super) processor: MessageProcessor, } @@ -73,18 +71,18 @@ impl Actor for GenMapper { impl GenMapper { async fn send_updated_subscriptions(&mut self) -> Result<(), RuntimeError> { - let topics = self.update_subscriptions(); - let diff = SubscriptionDiff::new(&topics, &TopicFilter::empty()); + let diff = self.update_subscriptions(); self.messages .send(OutputMessage::SubscriptionDiff(diff)) .await?; Ok(()) } - fn update_subscriptions(&self) -> TopicFilter { - let mut topics = self.subscriptions.lock().unwrap(); - topics.add_all(self.processor.subscriptions()); - topics.clone() + fn update_subscriptions(&mut self) -> SubscriptionDiff { + let new_subscriptions = self.processor.subscriptions(); + let diff = SubscriptionDiff::new(&new_subscriptions, &self.subscriptions); + self.subscriptions = new_subscriptions; + diff } async fn filter(&mut self, message: Message) -> Result<(), RuntimeError> { diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index 84ac172a3f9..f2e22e8a283 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -10,8 +10,6 @@ pub use crate::runtime::MessageProcessor; use std::convert::Infallible; use std::path::Path; use std::path::PathBuf; -use std::sync::Arc; -use std::sync::Mutex; use tedge_actors::fan_in_message_type; use tedge_actors::Builder; use tedge_actors::DynSender; @@ -34,7 +32,6 @@ fan_in_message_type!(OutputMessage[MqttMessage, SubscriptionDiff]: Clone, Debug, pub struct GenMapperBuilder { message_box: SimpleMessageBoxBuilder, - subscriptions: Arc>, processor: MessageProcessor, } @@ -43,7 +40,6 @@ impl GenMapperBuilder { let processor = MessageProcessor::try_new(config_dir).await?; Ok(GenMapperBuilder { message_box: SimpleMessageBoxBuilder::new("GenMapper", 16), - subscriptions: Arc::new(Mutex::new(TopicFilter::empty())), processor, }) } @@ -93,9 +89,10 @@ impl Builder for GenMapperBuilder { } fn build(self) -> GenMapper { + let subscriptions = self.topics().clone(); GenMapper { messages: self.message_box.build(), - subscriptions: self.subscriptions, + subscriptions, processor: self.processor, } } diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index af0c0d9f053..63688c7c923 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -93,10 +93,10 @@ impl MessageProcessor { if stage.filter.path() == path { match self.js_runtime.load_file(&path).await { Ok(()) => { - info!("Reloaded filter {path}"); + info!(target: "gen-mapper", "Reloaded filter {path}"); } Err(e) => { - error!("Failed to reload filter {path}: {e}"); + error!(target: "gen-mapper", "Failed to reload filter {path}: {e}"); return; } } @@ -109,23 +109,23 @@ impl MessageProcessor { for pipeline in self.pipelines.values_mut() { if pipeline.source == path { let Ok(source) = tokio::fs::read_to_string(&path).await else { - error!("Failed to read updated filter {path}"); + error!(target: "gen-mapper", "Failed to read updated pipeline {path}"); break; }; let config: PipelineConfig = match toml::from_str(&source) { Ok(config) => config, Err(e) => { - error!("Failed to parse toml for updated filter {path}: {e}"); + error!(target: "gen-mapper", "Failed to parse toml for updated pipeline {path}: {e}"); break; } }; match config.compile(&self.js_runtime, &self.config_dir, path.clone()) { Ok(p) => { *pipeline = p; - info!("Reloaded pipeline {path}"); + info!(target: "gen-mapper", "Reloaded pipeline {path}"); } Err(e) => { - error!("Failed to load updated pipeline {path}: {e}") + error!(target: "gen-mapper", "Failed to load updated pipeline {path}: {e}") } }; } diff --git a/crates/extensions/tedge_mqtt_ext/src/trie.rs b/crates/extensions/tedge_mqtt_ext/src/trie.rs index b961ad35814..f51c83ae95b 100644 --- a/crates/extensions/tedge_mqtt_ext/src/trie.rs +++ b/crates/extensions/tedge_mqtt_ext/src/trie.rs @@ -135,16 +135,7 @@ impl AddAssign for SubscriptionDiff { fn add_assign(&mut self, rhs: Self) { self.subscribe.extend(rhs.subscribe); self.unsubscribe.extend(rhs.unsubscribe); - - let overlap = self - .subscribe - .intersection(&self.unsubscribe) - .cloned() - .collect::>(); - for topic in overlap { - self.subscribe.remove(&topic); - self.unsubscribe.remove(&topic); - } + self.simplify() } } @@ -160,10 +151,12 @@ impl SubscriptionDiff { subscribe: &mqtt_channel::TopicFilter, unsubscribe: &mqtt_channel::TopicFilter, ) -> Self { - Self { + let mut diff = Self { subscribe: subscribe.patterns().iter().cloned().collect(), unsubscribe: unsubscribe.patterns().iter().cloned().collect(), - } + }; + diff.simplify(); + diff } fn with_topic_prefix(self, prefix: &str) -> Self { @@ -180,6 +173,18 @@ impl SubscriptionDiff { .collect(), } } + + fn simplify(&mut self) { + let overlap = self + .subscribe + .intersection(&self.unsubscribe) + .cloned() + .collect::>(); + for topic in overlap { + self.subscribe.remove(&topic); + self.unsubscribe.remove(&topic); + } + } } #[derive(PartialEq)] From 1cf271c2749b95b371f41008a252e2388c39d9b0 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Thu, 26 Jun 2025 16:20:10 +0200 Subject: [PATCH 29/50] Gen mapper dynamically load/reload/remove pipelines and filters Still to be improved: some event are processed twice - `mv pipeline.toml /tmp` leads to update then remove - `rm pipeline.toml` correctly triggers only remove - `cp /tmp/pipeline.toml .` triggers creation and update Signed-off-by: Didier Wenzek --- .../extensions/tedge_gen_mapper/src/actor.rs | 25 ++++- .../tedge_gen_mapper/src/runtime.rs | 94 ++++++++++++++----- 2 files changed, 93 insertions(+), 26 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index e6b4eea94a8..4e1daa381f7 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -57,10 +57,29 @@ impl Actor for GenMapper { self.send_updated_subscriptions().await?; } }, - Some(InputMessage::FsWatchEvent(e)) => { - tracing::warn!("TODO do something with {e:?}") + Some(InputMessage::FsWatchEvent(FsWatchEvent::FileCreated(path))) => { + let Ok(path) = Utf8PathBuf::try_from(path) else { + continue; + }; + if matches!(path.extension(), Some("js" | "ts")) { + self.processor.add_filter(path).await; + } else if path.extension() == Some("toml") { + self.processor.add_pipeline(path).await; + self.send_updated_subscriptions().await?; + } + }, + Some(InputMessage::FsWatchEvent(FsWatchEvent::FileDeleted(path))) => { + let Ok(path) = Utf8PathBuf::try_from(path) else { + continue; + }; + if matches!(path.extension(), Some("js" | "ts")) { + self.processor.remove_filter(path).await; + } else if path.extension() == Some("toml") { + self.processor.remove_pipeline(path).await; + self.send_updated_subscriptions().await?; + } }, - None => break, + _ => break, } } } diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 63688c7c923..0e81b2a485e 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -15,6 +15,7 @@ use tokio::fs::read_dir; use tokio::fs::read_to_string; use tracing::error; use tracing::info; +use tracing::warn; pub struct MessageProcessor { pub config_dir: PathBuf, @@ -87,6 +88,17 @@ impl MessageProcessor { self.js_runtime.dump_memory_stats().await; } + pub async fn add_filter(&mut self, path: Utf8PathBuf) { + match self.js_runtime.load_file(&path).await { + Ok(()) => { + info!(target: "gen-mapper", "Loaded filter {path}"); + } + Err(e) => { + error!(target: "gen-mapper", "Failed to load filter {path}: {e}"); + } + } + } + pub async fn reload_filter(&mut self, path: Utf8PathBuf) { for pipeline in self.pipelines.values_mut() { for stage in &mut pipeline.stages { @@ -105,32 +117,68 @@ impl MessageProcessor { } } - pub async fn reload_pipeline(&mut self, path: Utf8PathBuf) { - for pipeline in self.pipelines.values_mut() { - if pipeline.source == path { - let Ok(source) = tokio::fs::read_to_string(&path).await else { - error!(target: "gen-mapper", "Failed to read updated pipeline {path}"); - break; - }; - let config: PipelineConfig = match toml::from_str(&source) { - Ok(config) => config, - Err(e) => { - error!(target: "gen-mapper", "Failed to parse toml for updated pipeline {path}: {e}"); - break; - } - }; - match config.compile(&self.js_runtime, &self.config_dir, path.clone()) { - Ok(p) => { - *pipeline = p; - info!(target: "gen-mapper", "Reloaded pipeline {path}"); - } - Err(e) => { - error!(target: "gen-mapper", "Failed to load updated pipeline {path}: {e}") - } - }; + pub async fn remove_filter(&mut self, path: Utf8PathBuf) { + for (pipeline_id, pipeline) in self.pipelines.iter() { + for stage in pipeline.stages.iter() { + if stage.filter.path() == path { + warn!(target: "gen-mapper", "Removing a filter used by {pipeline_id}: {path}"); + return; + } } } } + + pub async fn load_pipeline(&mut self, pipeline_id: String, path: Utf8PathBuf) -> bool { + let Ok(source) = tokio::fs::read_to_string(&path).await else { + self.remove_pipeline(path).await; + return false; + }; + let config: PipelineConfig = match toml::from_str(&source) { + Ok(config) => config, + Err(e) => { + error!(target: "gen-mapper", "Failed to parse toml for pipeline {path}: {e}"); + return false; + } + }; + match config.compile(&self.js_runtime, &self.config_dir, path.clone()) { + Ok(pipeline) => { + self.pipelines.insert(pipeline_id, pipeline); + true + } + Err(e) => { + error!(target: "gen-mapper", "Failed to compile pipeline {path}: {e}"); + false + } + } + } + + pub async fn add_pipeline(&mut self, path: Utf8PathBuf) { + let pipeline_id = path.file_name().unwrap(); + if !self.pipelines.contains_key(pipeline_id) + && self + .load_pipeline(pipeline_id.to_string(), path.clone()) + .await + { + info!(target: "gen-mapper", "Loaded new pipeline {path}"); + } + } + + pub async fn reload_pipeline(&mut self, path: Utf8PathBuf) { + let pipeline_id = path.file_name().unwrap(); + if self.pipelines.contains_key(pipeline_id) + && self + .load_pipeline(pipeline_id.to_string(), path.clone()) + .await + { + info!(target: "gen-mapper", "Reloaded updated pipeline {path}"); + } + } + + pub async fn remove_pipeline(&mut self, path: Utf8PathBuf) { + let pipeline_id = path.file_name().unwrap(); + self.pipelines.remove(pipeline_id); + info!(target: "gen-mapper", "Removed deleted pipeline {path}"); + } } #[derive(Default)] From 9024cef9dfa4e037b683f1311b28a78eaa986bcf Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Thu, 26 Jun 2025 19:01:04 +0200 Subject: [PATCH 30/50] Index pipelines using full paths not just filenames Signed-off-by: Didier Wenzek --- .../tedge_gen_mapper/src/runtime.rs | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 0e81b2a485e..3abfab4f1a8 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -24,6 +24,10 @@ pub struct MessageProcessor { } impl MessageProcessor { + pub fn pipeline_id(path: impl AsRef) -> String { + format!("{}", path.as_ref().display()) + } + pub async fn try_new(config_dir: impl AsRef) -> Result { let config_dir = config_dir.as_ref().to_owned(); let mut js_runtime = JsRuntime::try_new().await?; @@ -153,30 +157,26 @@ impl MessageProcessor { } pub async fn add_pipeline(&mut self, path: Utf8PathBuf) { - let pipeline_id = path.file_name().unwrap(); - if !self.pipelines.contains_key(pipeline_id) - && self - .load_pipeline(pipeline_id.to_string(), path.clone()) - .await + let pipeline_id = Self::pipeline_id(&path); + if !self.pipelines.contains_key(&pipeline_id) + && self.load_pipeline(pipeline_id, path.clone()).await { info!(target: "gen-mapper", "Loaded new pipeline {path}"); } } pub async fn reload_pipeline(&mut self, path: Utf8PathBuf) { - let pipeline_id = path.file_name().unwrap(); - if self.pipelines.contains_key(pipeline_id) - && self - .load_pipeline(pipeline_id.to_string(), path.clone()) - .await + let pipeline_id = Self::pipeline_id(&path); + if self.pipelines.contains_key(&pipeline_id) + && self.load_pipeline(pipeline_id, path.clone()).await { info!(target: "gen-mapper", "Reloaded updated pipeline {path}"); } } pub async fn remove_pipeline(&mut self, path: Utf8PathBuf) { - let pipeline_id = path.file_name().unwrap(); - self.pipelines.remove(pipeline_id); + let pipeline_id = Self::pipeline_id(&path); + self.pipelines.remove(&pipeline_id); info!(target: "gen-mapper", "Removed deleted pipeline {path}"); } } @@ -224,12 +224,12 @@ impl PipelineSpecs { } async fn load_pipeline(&mut self, file: impl AsRef) -> Result<(), LoadError> { - if let Some(name) = file.as_ref().file_name() { - let specs = read_to_string(file.as_ref()).await?; - let pipeline: PipelineConfig = toml::from_str(&specs)?; - self.pipeline_specs - .insert(name.to_string(), (file.as_ref().to_owned(), pipeline)); - } + let path = file.as_ref(); + let pipeline_id = MessageProcessor::pipeline_id(path); + let specs = read_to_string(file.as_ref()).await?; + let pipeline: PipelineConfig = toml::from_str(&specs)?; + self.pipeline_specs + .insert(pipeline_id, (path.to_owned(), pipeline)); Ok(()) } From 4117d4a3414b95bdf066081f8bb9bf4bee0465b2 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 27 Jun 2025 09:16:36 +0200 Subject: [PATCH 31/50] tedge mapping test consumes messages from stdin Signed-off-by: Didier Wenzek --- crates/core/tedge/src/cli/mapping/cli.rs | 5 +- crates/core/tedge/src/cli/mapping/test.rs | 88 ++++++++++++++++++----- 2 files changed, 75 insertions(+), 18 deletions(-) diff --git a/crates/core/tedge/src/cli/mapping/cli.rs b/crates/core/tedge/src/cli/mapping/cli.rs index 729bf0ededb..8ff07b4c135 100644 --- a/crates/core/tedge/src/cli/mapping/cli.rs +++ b/crates/core/tedge/src/cli/mapping/cli.rs @@ -44,12 +44,13 @@ pub enum TEdgeMappingCli { /// Topic of the message sample /// - /// If none is provided, messages are read from stdout + /// If none is provided, messages are read from stdin expecting a line per message: + /// [topic] payload topic: Option, /// Payload of the message sample /// - /// If none is provided, payloads are read from stdout + /// If none is provided, payloads are read from stdin payload: Option, }, } diff --git a/crates/core/tedge/src/cli/mapping/test.rs b/crates/core/tedge/src/cli/mapping/test.rs index 2737720bc4d..d597567c0bc 100644 --- a/crates/core/tedge/src/cli/mapping/test.rs +++ b/crates/core/tedge/src/cli/mapping/test.rs @@ -5,6 +5,10 @@ use anyhow::Error; use std::path::PathBuf; use tedge_config::TEdgeConfig; use tedge_gen_mapper::pipeline::*; +use tedge_gen_mapper::MessageProcessor; +use tokio::io::AsyncBufReadExt; +use tokio::io::BufReader; +use tokio::io::Stdin; pub struct TestCommand { pub mapping_dir: PathBuf, @@ -25,28 +29,44 @@ impl Command for TestCommand { let mut processor = TEdgeMappingCli::load_pipelines(&self.mapping_dir).await?; if let Some(message) = &self.message { let timestamp = DateTime::now(); - match &self.filter { - Some(filter) => { - let filter_name = filter.display().to_string(); - print( - processor - .process_with_pipeline(&filter_name, ×tamp, message) - .await, - ) - } - None => processor - .process(×tamp, message) - .await - .into_iter() - .map(|(_, v)| v) - .for_each(print), + self.process(&mut processor, message, ×tamp).await; + } else { + let mut stdin = BufReader::new(tokio::io::stdin()); + while let Some(message) = next_message(&mut stdin).await { + let timestamp = DateTime::now(); + self.process(&mut processor, &message, ×tamp).await; } } - Ok(()) } } +impl TestCommand { + async fn process( + &self, + processor: &mut MessageProcessor, + message: &Message, + timestamp: &DateTime, + ) { + match &self.filter { + Some(filter) => { + let filter_name = filter.display().to_string(); + print( + processor + .process_with_pipeline(&filter_name, timestamp, message) + .await, + ) + } + None => processor + .process(timestamp, message) + .await + .into_iter() + .map(|(_, v)| v) + .for_each(print), + } + } +} + fn print(messages: Result, FilterError>) { match messages { Ok(messages) => { @@ -59,3 +79,39 @@ fn print(messages: Result, FilterError>) { } } } + +fn parse(line: String) -> Result, Error> { + let line = line.trim(); + if line.is_empty() { + return Ok(None); + } + if !line.starts_with("[") { + return Err(anyhow::anyhow!("Missing opening bracket: {}", line)); + } + let Some(closing_bracket) = line.find(']') else { + return Err(anyhow::anyhow!("Missing closing bracket: {}", line)); + }; + + let topic = line[1..closing_bracket].to_string(); + let payload = line[closing_bracket + 1..].to_string(); + + Ok(Some(Message { topic, payload })) +} + +async fn next_message(input: &mut BufReader) -> Option { + let mut line = String::new(); + match input.read_line(&mut line).await { + Ok(0) => None, + Ok(_) => match parse(line) { + Ok(message) => message, + Err(err) => { + eprintln!("Fail to parse input message {}", err); + None + } + }, + Err(err) => { + eprintln!("Fail to read input stream {}", err); + None + } + } +} From 0fffac6d9d6b5f1b8e0f230efe5578befbdb91e9 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 27 Jun 2025 13:42:06 +0200 Subject: [PATCH 32/50] Test and fix te_2_c8y.js Signed-off-by: Didier Wenzek --- crates/core/tedge/Cargo.toml | 2 +- crates/extensions/tedge_gen_mapper/Cargo.toml | 5 ++- .../tedge_gen_mapper/pipelines/te_to_c8y.js | 26 +++++++++----- .../pipelines/measurements.toml | 1 - .../tedge_gen_mapper/pipelines/te_to_c8y.js | 26 +++++++++----- .../tedge_gen_mapper/tedge_gen_mapper.robot | 36 +++++++++++++++---- 6 files changed, 69 insertions(+), 27 deletions(-) diff --git a/crates/core/tedge/Cargo.toml b/crates/core/tedge/Cargo.toml index 83315dc2197..cf18af21e6f 100644 --- a/crates/core/tedge/Cargo.toml +++ b/crates/core/tedge/Cargo.toml @@ -49,12 +49,12 @@ strum_macros = { workspace = true } tar = { workspace = true } tedge-agent = { workspace = true } tedge-apt-plugin = { workspace = true } -tedge_gen_mapper = { workspace = true } tedge-mapper = { workspace = true, default-features = false } tedge-watchdog = { workspace = true } tedge-write = { workspace = true } tedge_api = { workspace = true } tedge_config = { workspace = true } +tedge_gen_mapper = { workspace = true } tedge_utils = { workspace = true } thiserror = { workspace = true } time = { workspace = true } diff --git a/crates/extensions/tedge_gen_mapper/Cargo.toml b/crates/extensions/tedge_gen_mapper/Cargo.toml index 91d491c5b61..d77da72de61 100644 --- a/crates/extensions/tedge_gen_mapper/Cargo.toml +++ b/crates/extensions/tedge_gen_mapper/Cargo.toml @@ -12,7 +12,10 @@ repository.workspace = true anyhow = { workspace = true } async-trait = { workspace = true } camino = { workspace = true, features = ["serde1"] } -rquickjs = { workspace = true, default-features = false, features = ["futures", "parallel"] } +rquickjs = { workspace = true, default-features = false, features = [ + "futures", + "parallel", +] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } tedge_actors = { workspace = true } diff --git a/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js b/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js index ae2d8f157c1..d054c53742e 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js @@ -44,7 +44,7 @@ /// ``` export function process(t, message, config) { let topic_parts = message.topic.split( '/') - let type = topic_parts[6] + let type = topic_parts[6] || "ThinEdgeMeasurement" let payload = JSON.parse(message.payload) let c8y_msg = { @@ -56,7 +56,11 @@ export function process(t, message, config) { for (let [k, v] of Object.entries(payload)) { let k_meta = (meta || {})[k] || {} if (k === "time") { - let fragment = { time: v } + let t = v + if (typeof(v) === "number") { + t = (new Date(v * 1000)).toISOString() + } + let fragment = { time: t } Object.assign(c8y_msg, fragment) } else if (typeof(v) === "number") { @@ -65,15 +69,19 @@ export function process(t, message, config) { } let fragment = { [k]: { [k]: v } } Object.assign(c8y_msg, fragment) - } else for (let [sub_k, sub_v] of Object.entries(v)) { - let sub_k_meta = k_meta[sub_k] - if (typeof(sub_v) === "number") { - if (sub_k_meta) { - sub_v = { value: sub_v, ...sub_k_meta } + } else { + let fragment = {} + for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_k_meta = k_meta[sub_k] + if (typeof(sub_v) === "number") { + if (sub_k_meta) { + sub_v = { value: sub_v, ...sub_k_meta } + } + let sub_fragment = { [sub_k]: sub_v } + Object.assign(fragment, sub_fragment) } - let fragment = { [k]: { [sub_k]: sub_v } } - Object.assign(c8y_msg, fragment) } + Object.assign(c8y_msg, { [k]: fragment}) } } diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.toml b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.toml index a777177165b..3262580a673 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.toml +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.toml @@ -3,5 +3,4 @@ input_topics = ["te/+/+/+/+/m/+"] stages = [ { filter = "add_timestamp.js" }, { filter = "te_to_c8y.js", meta_topics = ["te/+/+/+/+/m/+/meta"] }, - { filter = "set_topic.js", config = { topic = "gen-mapper/c8y" } } ] diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js index ae2d8f157c1..d054c53742e 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js @@ -44,7 +44,7 @@ /// ``` export function process(t, message, config) { let topic_parts = message.topic.split( '/') - let type = topic_parts[6] + let type = topic_parts[6] || "ThinEdgeMeasurement" let payload = JSON.parse(message.payload) let c8y_msg = { @@ -56,7 +56,11 @@ export function process(t, message, config) { for (let [k, v] of Object.entries(payload)) { let k_meta = (meta || {})[k] || {} if (k === "time") { - let fragment = { time: v } + let t = v + if (typeof(v) === "number") { + t = (new Date(v * 1000)).toISOString() + } + let fragment = { time: t } Object.assign(c8y_msg, fragment) } else if (typeof(v) === "number") { @@ -65,15 +69,19 @@ export function process(t, message, config) { } let fragment = { [k]: { [k]: v } } Object.assign(c8y_msg, fragment) - } else for (let [sub_k, sub_v] of Object.entries(v)) { - let sub_k_meta = k_meta[sub_k] - if (typeof(sub_v) === "number") { - if (sub_k_meta) { - sub_v = { value: sub_v, ...sub_k_meta } + } else { + let fragment = {} + for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_k_meta = k_meta[sub_k] + if (typeof(sub_v) === "number") { + if (sub_k_meta) { + sub_v = { value: sub_v, ...sub_k_meta } + } + let sub_fragment = { [sub_k]: sub_v } + Object.assign(fragment, sub_fragment) } - let fragment = { [k]: { [sub_k]: sub_v } } - Object.assign(c8y_msg, fragment) } + Object.assign(c8y_msg, { [k]: fragment}) } } diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot index 67ed106835e..eaea57904eb 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot +++ b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot @@ -8,20 +8,44 @@ Test Tags theme:tedge_mapper *** Test Cases *** Add missing timestamps - Execute Command tedge mqtt pub te/device/main// '{}' - ${transformed_msg} Should Have MQTT Messages gen-mapper/c8y + ${transformed_msg} Execute Command tedge mapping test te/device/main///m/ '{}' Should Contain ${transformed_msg} item=time +Convert timestamps to ISO + ${transformed_msg} Execute Command tedge mapping test te/device/main///m/ '{"time": 1751023862.000}' + Should Contain ${transformed_msg} item="time":"2025-06-27T11:31:02.000Z" + +Extract measurement type from topic + ${transformed_msg} Execute Command + ... tedge mapping test te/device/main///m/environment '{"temperature": 258}' + Should Contain + ... ${transformed_msg} + ... item="type":"environment" + +Use default measurement type + ${transformed_msg} Execute Command + ... tedge mapping test te/device/main///m/ '{"temperature": 258}' + Should Contain + ... ${transformed_msg} + ... item="type":"ThinEdgeMeasurement" + +Translate complex tedge json to c8y json + ${transformed_msg} Execute Command + ... tedge mapping test te/device/main///m/environment '{"time":"2025-06-27T08:11:05.301804125Z", "temperature": 258, "location": {"latitude": 32.54, "longitude": -117.67, "altitude": 98.6 }, "pressure": 98}' + ... strip=True + Should Be Equal + ... ${transformed_msg} + ... [c8y/measurement/measurements/create] {"type":"environment","time":"2025-06-27T08:11:05.301804125Z","temperature":{"temperature":258},"location":{"latitude":32.54,"longitude":-117.67,"altitude":98.6},"pressure":{"pressure":98}} + + *** Keywords *** Custom Setup - ${DEVICE_SN}= Setup + ${DEVICE_SN}= Setup skip_bootstrap=${True} + Execute Command ./bootstrap.sh --no-bootstrap --no-connect Set Suite Variable $DEVICE_SN Copy Configuration Files - Start Generic Mapper Copy Configuration Files Execute Command mkdir /etc/tedge/gen-mapper/ ThinEdgeIO.Transfer To Device ${CURDIR}/pipelines/* /etc/tedge/gen-mapper/ -Start Generic Mapper - Execute Command nohup tedge run tedge-mapper gen & From f5c21fd38ab8255aaecbcdd7984a59988a5b7d6d Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 27 Jun 2025 16:05:17 +0200 Subject: [PATCH 33/50] Test dynamic update of filter config Signed-off-by: Didier Wenzek --- crates/core/tedge/src/cli/mapping/test.rs | 31 +++++++++++++------ .../pipelines/measurements.samples | 24 ++++++++++++++ .../tedge_gen_mapper/tedge_gen_mapper.robot | 11 +++++++ 3 files changed, 56 insertions(+), 10 deletions(-) create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.samples diff --git a/crates/core/tedge/src/cli/mapping/test.rs b/crates/core/tedge/src/cli/mapping/test.rs index d597567c0bc..bbf4d9992cc 100644 --- a/crates/core/tedge/src/cli/mapping/test.rs +++ b/crates/core/tedge/src/cli/mapping/test.rs @@ -98,19 +98,30 @@ fn parse(line: String) -> Result, Error> { Ok(Some(Message { topic, payload })) } -async fn next_message(input: &mut BufReader) -> Option { - let mut line = String::new(); - match input.read_line(&mut line).await { - Ok(0) => None, - Ok(_) => match parse(line) { - Ok(message) => message, +async fn next_line(input: &mut BufReader) -> Option { + loop { + let mut line = String::new(); + match input.read_line(&mut line).await { + Ok(0) => return None, + Ok(_) => { + let line = line.trim(); + if !line.is_empty() { + return Some(line.to_string()); + } + } Err(err) => { - eprintln!("Fail to parse input message {}", err); - None + eprintln!("Fail to read input stream {}", err); + return None } - }, + } + } +} +async fn next_message(input: &mut BufReader) -> Option { + let line = next_line(input).await?; + match parse(line) { + Ok(message) => message, Err(err) => { - eprintln!("Fail to read input stream {}", err); + eprintln!("Fail to parse input message {}", err); None } } diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.samples b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.samples new file mode 100644 index 00000000000..2528b355503 --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.samples @@ -0,0 +1,24 @@ +# The default is to have no units +INPUT: [te/device/main///m/] {"temperature": 25, "time":"2025-06-27T13:33:53.493Z"} +OUTPUT: [c8y/measurement/measurements/create] {"type":"ThinEdgeMeasurement","temperature":{"temperature":25},"time":"2025-06-27T13:33:53.493Z"} + +# Units can be set using the meta topic +INPUT: [te/device/main///m//meta] {"temperature": { "unit": "°C" }} + +# All the subsequent messages use then the configured units +INPUT: [te/device/main///m/] {"temperature": 25, "time":"2025-06-27T12:40:24.122Z"} +OUTPUT: [c8y/measurement/measurements/create] {"type":"ThinEdgeMeasurement","temperature":{"temperature":{"value":25,"unit":"°C"}},"time":"2025-06-27T12:40:24.122Z"} + +# Units can be dynamically updated +INPUT: [te/device/main///m//meta] {"temperature": { "unit": "°F" }} +INPUT: [te/device/main///m/] {"temperature": 77, "time":"2025-06-27T12:40:24.122Z"} +OUTPUT: [c8y/measurement/measurements/create] {"type":"ThinEdgeMeasurement","temperature":{"temperature":{"value":77,"unit":"°F"}},"time":"2025-06-27T12:40:24.122Z"} + +# Units can be set to across measurement types +INPUT: [te/device/main///m/environment/meta] { "temperature": { "unit": "°C" }, "location": { "altitude": { "unit": "m" } }, "pressure": { "unit": "pascal" } } +INPUT: [te/device/main///m/environment] {"time":"2025-06-27T08:11:05.301804125Z", "temperature": 25, "location": {"latitude": 32.54, "longitude": -117.67, "altitude": 98.6 }, "pressure": 98} +OUTPUT: [c8y/measurement/measurements/create] {"type":"environment","time":"2025-06-27T08:11:05.301804125Z","temperature":{"temperature":{"value":25,"unit":"°C"}},"location":{"latitude":32.54,"longitude":-117.67,"altitude":{"value":98.6,"unit":"m"}},"pressure":{"pressure":{"value":98,"unit":"pascal"}}} + +# For the default type °F are still used, while changed to °C for environment measurements +INPUT: [te/device/main///m/] {"temperature": 77, "time":"2025-06-27T12:40:24.122Z"} +OUTPUT: [c8y/measurement/measurements/create] {"type":"ThinEdgeMeasurement","temperature":{"temperature":{"value":77,"unit":"°F"}},"time":"2025-06-27T12:40:24.122Z"} \ No newline at end of file diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot index eaea57904eb..cc0060dccca 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot +++ b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot @@ -37,6 +37,17 @@ Translate complex tedge json to c8y json ... ${transformed_msg} ... [c8y/measurement/measurements/create] {"type":"environment","time":"2025-06-27T08:11:05.301804125Z","temperature":{"temperature":258},"location":{"latitude":32.54,"longitude":-117.67,"altitude":98.6},"pressure":{"pressure":98}} +Units are configured using topic metadata + ${transformed_msg} Execute Command + ... cat /etc/tedge/gen-mapper/measurements.samples | awk '{ print $2 }' FS\='INPUT:' | tedge mapping test + ... strip=True + ${expected_msg} Execute Command + ... cat /etc/tedge/gen-mapper/measurements.samples | awk '{ if ($2) print $2 }' FS\='OUTPUT: ' + ... strip=True + Should Be Equal + ... ${transformed_msg} + ... ${expected_msg} + *** Keywords *** Custom Setup From a75e4470a12861974e550780e86dbdb5274bf4af Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 27 Jun 2025 16:27:03 +0200 Subject: [PATCH 34/50] Update system tests Setup Signed-off-by: Didier Wenzek --- .../tests/tedge_gen_mapper/tedge_gen_mapper.robot | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot index cc0060dccca..0b877cb9682 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot +++ b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot @@ -51,8 +51,7 @@ Units are configured using topic metadata *** Keywords *** Custom Setup - ${DEVICE_SN}= Setup skip_bootstrap=${True} - Execute Command ./bootstrap.sh --no-bootstrap --no-connect + ${DEVICE_SN}= Setup connect=${False} Set Suite Variable $DEVICE_SN Copy Configuration Files From 3f3160ed3f275c6ef6f9c91f5116ba238abad056 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Mon, 30 Jun 2025 10:54:04 +0200 Subject: [PATCH 35/50] Support for testing the tick function of a filter $ tedge mapping test --final-tick Signed-off-by: Didier Wenzek --- crates/core/tedge/src/cli/mapping/cli.rs | 6 + crates/core/tedge/src/cli/mapping/test.rs | 22 +++- .../tedge_gen_mapper/pipelines/average.js | 28 +++-- .../tedge_gen_mapper/src/runtime.rs | 12 ++ .../tedge_gen_mapper/pipelines/average.js | 103 ++++++++++++++++++ .../pipelines/average.samples | 16 +++ .../tedge_gen_mapper/pipelines/average.toml | 5 + .../tedge_gen_mapper/tedge_gen_mapper.robot | 11 ++ 8 files changed, 190 insertions(+), 13 deletions(-) create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.js create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.samples create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.toml diff --git a/crates/core/tedge/src/cli/mapping/cli.rs b/crates/core/tedge/src/cli/mapping/cli.rs index 8ff07b4c135..4adaff621d4 100644 --- a/crates/core/tedge/src/cli/mapping/cli.rs +++ b/crates/core/tedge/src/cli/mapping/cli.rs @@ -42,6 +42,10 @@ pub enum TEdgeMappingCli { #[clap(long)] filter: Option, + /// Send a tick after all the message samples + #[clap(long = "final-tick")] + final_tick: bool, + /// Topic of the message sample /// /// If none is provided, messages are read from stdin expecting a line per message: @@ -66,6 +70,7 @@ impl BuildCommand for TEdgeMappingCli { TEdgeMappingCli::Test { mapping_dir, filter, + final_tick, topic, payload, } => { @@ -80,6 +85,7 @@ impl BuildCommand for TEdgeMappingCli { mapping_dir, filter, message, + final_tick, } .into_boxed()) } diff --git a/crates/core/tedge/src/cli/mapping/test.rs b/crates/core/tedge/src/cli/mapping/test.rs index bbf4d9992cc..79c208b0aab 100644 --- a/crates/core/tedge/src/cli/mapping/test.rs +++ b/crates/core/tedge/src/cli/mapping/test.rs @@ -14,6 +14,7 @@ pub struct TestCommand { pub mapping_dir: PathBuf, pub filter: Option, pub message: Option, + pub final_tick: bool, } #[async_trait::async_trait] @@ -37,6 +38,10 @@ impl Command for TestCommand { self.process(&mut processor, &message, ×tamp).await; } } + if self.final_tick { + let timestamp = DateTime::now(); + self.tick(&mut processor, ×tamp).await; + } Ok(()) } } @@ -65,6 +70,21 @@ impl TestCommand { .for_each(print), } } + + async fn tick(&self, processor: &mut MessageProcessor, timestamp: &DateTime) { + match &self.filter { + Some(filter) => { + let filter_name = filter.display().to_string(); + print(processor.tick_with_pipeline(&filter_name, timestamp).await) + } + None => processor + .tick(timestamp) + .await + .into_iter() + .map(|(_, v)| v) + .for_each(print), + } + } } fn print(messages: Result, FilterError>) { @@ -111,7 +131,7 @@ async fn next_line(input: &mut BufReader) -> Option { } Err(err) => { eprintln!("Fail to read input stream {}", err); - return None + return None; } } } diff --git a/crates/extensions/tedge_gen_mapper/pipelines/average.js b/crates/extensions/tedge_gen_mapper/pipelines/average.js index 1a3f1c7e8dd..128f8774194 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/average.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/average.js @@ -11,7 +11,6 @@ export function process (timestamp, message) { let topic = message.topic let payload = JSON.parse(message.payload) let agg_payload = State.agg_for_topic[topic] - if (agg_payload) { for (let [k, v] of Object.entries(payload)) { let agg = agg_payload[k] @@ -22,7 +21,7 @@ export function process (timestamp, message) { } } else if (typeof (v) === "number") { if (!agg) { - let fragment = {k: {sum: v, count: 1}} + let fragment = {[k]: {sum: v, count: 1}} Object.assign(agg_payload, fragment) } else { agg.sum += v @@ -30,16 +29,17 @@ export function process (timestamp, message) { } } else { if (!agg) { + let fragment = {} for (let [sub_k, sub_v] of Object.entries(v)) { - let fragment = { [k]: { [sub_k]: { sum: sub_v, count: 1 } } } - Object.assign(agg_payload, fragment) + let sub_fragment = { [sub_k]: { sum: sub_v, count: 1 } } + Object.assign(fragment, sub_fragment) } + Object.assign(agg_payload, { [k]: fragment }) } else { for (let [sub_k, sub_v] of Object.entries(v)) { - let sub_agg = agg_payload[sub_k] + let sub_agg = agg[sub_k] if (!sub_agg) { - let fragment = {k: { [sub_k]: { sum: sub_v, count: 1 } } } - Object.assign(agg_payload, fragment) + agg[sub_k] = { sum: sub_v, count: 1 } } else { sub_agg.sum += sub_v sub_agg.count += 1 @@ -56,11 +56,15 @@ export function process (timestamp, message) { Object.assign(agg_payload, fragment) } else if (typeof(v) === "number") { - let fragment = { k: { sum: v, count: 1 } } + let fragment = { [k]: { sum: v, count: 1 } } Object.assign(agg_payload, fragment) - } else for (let [sub_k, sub_v] of Object.entries(v)) { - let fragment = { [k]: { [sub_k]: { sum: sub_v, count: 1 } } } - Object.assign(agg_payload, fragment) + } else { + let fragment = {} + for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_fragment = { [sub_k]: { sum: sub_v, count: 1 } } + Object.assign(fragment, sub_fragment) + } + Object.assign(agg_payload, { [k]: fragment }) } } State.agg_for_topic[topic] = agg_payload @@ -80,7 +84,7 @@ export function tick() { Object.assign(payload, fragment) } else if (v.sum && v.count) { - let fragment = { k: v.sum / v.count } + let fragment = { [k]: v.sum / v.count } Object.assign(payload, fragment) } else for (let [sub_k, sub_v] of Object.entries(v)) { let fragment = { [k]: { [sub_k]: sub_v.sum / sub_v.count } } diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 3abfab4f1a8..830ec3bc5c7 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -88,6 +88,18 @@ impl MessageProcessor { out_messages } + pub async fn tick_with_pipeline( + &mut self, + pipeline_id: &String, + timestamp: &DateTime, + ) -> Result, FilterError> { + let pipeline = self + .pipelines + .get_mut(pipeline_id) + .ok_or_else(|| anyhow::anyhow!("No such pipeline: {pipeline_id}"))?; + pipeline.tick(&self.js_runtime, timestamp).await + } + pub async fn dump_memory_stats(&self) { self.js_runtime.dump_memory_stats().await; } diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.js b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.js new file mode 100644 index 00000000000..128f8774194 --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.js @@ -0,0 +1,103 @@ +// Compute the average value of a series of measurements received during a time windows +// - Take care of the topic: messages received over different topics are not mixed +// - Ignore messages which are not formated as thin-edge JSON +// - Ignore values which are not numbers +// - Use the first timestamp as the timestamp for the aggregate +class State { + static agg_for_topic = {} +} + +export function process (timestamp, message) { + let topic = message.topic + let payload = JSON.parse(message.payload) + let agg_payload = State.agg_for_topic[topic] + if (agg_payload) { + for (let [k, v] of Object.entries(payload)) { + let agg = agg_payload[k] + if (k === "time") { + if (!agg) { + let fragment = {time: v} + Object.assign(agg_payload, fragment) + } + } else if (typeof (v) === "number") { + if (!agg) { + let fragment = {[k]: {sum: v, count: 1}} + Object.assign(agg_payload, fragment) + } else { + agg.sum += v + agg.count += 1 + } + } else { + if (!agg) { + let fragment = {} + for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_fragment = { [sub_k]: { sum: sub_v, count: 1 } } + Object.assign(fragment, sub_fragment) + } + Object.assign(agg_payload, { [k]: fragment }) + } else { + for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_agg = agg[sub_k] + if (!sub_agg) { + agg[sub_k] = { sum: sub_v, count: 1 } + } else { + sub_agg.sum += sub_v + sub_agg.count += 1 + } + } + } + } + } + } else { + let agg_payload = {} + for (let [k, v] of Object.entries(payload)) { + if (k === "time") { + let fragment = { time: v } + Object.assign(agg_payload, fragment) + } + else if (typeof(v) === "number") { + let fragment = { [k]: { sum: v, count: 1 } } + Object.assign(agg_payload, fragment) + } else { + let fragment = {} + for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_fragment = { [sub_k]: { sum: sub_v, count: 1 } } + Object.assign(fragment, sub_fragment) + } + Object.assign(agg_payload, { [k]: fragment }) + } + } + State.agg_for_topic[topic] = agg_payload + } + + return [] +} + +export function tick() { + let messages = [] + + for (let [topic, agg] of Object.entries(State.agg_for_topic)) { + let payload = {} + for (let [k, v] of Object.entries(agg)) { + if (k === "time") { + let fragment = { time: v } + Object.assign(payload, fragment) + } + else if (v.sum && v.count) { + let fragment = { [k]: v.sum / v.count } + Object.assign(payload, fragment) + } else for (let [sub_k, sub_v] of Object.entries(v)) { + let fragment = { [k]: { [sub_k]: sub_v.sum / sub_v.count } } + Object.assign(payload, fragment) + } + } + + messages.push ({ + topic: topic, + payload: JSON.stringify(payload) + }) + } + + State.agg_for_topic = {} + return messages +} \ No newline at end of file diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.samples b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.samples new file mode 100644 index 00000000000..cee47197527 --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.samples @@ -0,0 +1,16 @@ +INPUT: [test/average] {"temperature": 20} +INPUT: [test/average] {"humidity": 50} +INPUT: [test/average] {"temperature": 25, "location": {"altitude": 100.0 }} +INPUT: [test/average] {"temperature": 30} +INPUT: [test/average] {"temperature": 25} +INPUT: [test/average] {"humidity": 70, "location": {"altitude": 120.0 }} +INPUT: [test/average] {"temperature": 20} +INPUT: [test/average/another_topic] {"temperature": 2} +INPUT: [test/average/another_topic] {"temperature": 3} +INPUT: [test/average/another_topic] {"temperature": 5} +INPUT: [test/average/another_topic] {"temperature": 6} +INPUT: [test/average/another_topic] {"temperature": 4} + + +OUTPUT: [test/average] {"temperature":24,"humidity":60,"location":{"altitude":110}} +OUTPUT: [test/average/another_topic] {"temperature":4} \ No newline at end of file diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.toml b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.toml new file mode 100644 index 00000000000..52f2bdf7c4a --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.toml @@ -0,0 +1,5 @@ +input_topics = ["test/average/#"] + +stages = [ + { filter = "average.js", tick_every_seconds = 1 }, +] \ No newline at end of file diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot index 0b877cb9682..0092d7d400a 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot +++ b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot @@ -48,6 +48,17 @@ Units are configured using topic metadata ... ${transformed_msg} ... ${expected_msg} +Computing average over a time window + ${transformed_msg} Execute Command + ... cat /etc/tedge/gen-mapper/average.samples | awk '{ print $2 }' FS\='INPUT:' | tedge mapping test --final-tick + ... strip=True + ${expected_msg} Execute Command + ... cat /etc/tedge/gen-mapper/average.samples | awk '{ if ($2) print $2 }' FS\='OUTPUT: ' + ... strip=True + Should Be Equal + ... ${transformed_msg} + ... ${expected_msg} + *** Keywords *** Custom Setup From 214ccc385f77fafd496e6f89d7e19368c2a9fe7b Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Mon, 30 Jun 2025 18:30:59 +0200 Subject: [PATCH 36/50] Add support for console.log() Signed-off-by: Didier Wenzek --- Cargo.lock | 49 +++++++++++++++++ crates/extensions/tedge_gen_mapper/Cargo.toml | 1 + .../tedge_gen_mapper/pipelines/average.js | 11 +++- .../tedge_gen_mapper/src/js_filter.rs | 11 ++++ .../tedge_gen_mapper/src/js_runtime.rs | 55 ++++++++++++++++++- .../tedge_gen_mapper/pipelines/average.js | 11 +++- .../pipelines/average.samples | 4 +- 7 files changed, 133 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 50a061f5293..9d988c45b71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1178,6 +1178,15 @@ version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2459fc9262a1aa204eb4b5764ad4f189caec88aea9634389c0a25f8be7f6265e" +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.10.0" @@ -3345,6 +3354,15 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "proc-macro-crate" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.95" @@ -3706,6 +3724,12 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "reqwest" version = "0.12.15" @@ -3806,6 +3830,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c5227859c4dfc83f428e58f9569bf439e628c8d139020e7faff437e6f5abaa0" dependencies = [ "rquickjs-core", + "rquickjs-macro", ] [[package]] @@ -3815,9 +3840,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e82e0ca83028ad5b533b53b96c395bbaab905a5774de4aaf1004eeacafa3d85d" dependencies = [ "async-lock", + "relative-path", "rquickjs-sys", ] +[[package]] +name = "rquickjs-macro" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4d2eccd988a924a470a76fbd81a191b22d1f5f4f4619cf5662a8c1ab4ca1db7" +dependencies = [ + "convert_case", + "fnv", + "ident_case", + "indexmap 2.9.0", + "proc-macro-crate", + "proc-macro2", + "quote", + "rquickjs-core", + "syn 2.0.101", +] + [[package]] name = "rquickjs-sys" version = "0.9.0" @@ -5723,6 +5766,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + [[package]] name = "unicode-width" version = "0.1.14" diff --git a/crates/extensions/tedge_gen_mapper/Cargo.toml b/crates/extensions/tedge_gen_mapper/Cargo.toml index d77da72de61..9b5a4b47a23 100644 --- a/crates/extensions/tedge_gen_mapper/Cargo.toml +++ b/crates/extensions/tedge_gen_mapper/Cargo.toml @@ -14,6 +14,7 @@ async-trait = { workspace = true } camino = { workspace = true, features = ["serde1"] } rquickjs = { workspace = true, default-features = false, features = [ "futures", + "macro", "parallel", ] } serde = { workspace = true, features = ["derive"] } diff --git a/crates/extensions/tedge_gen_mapper/pipelines/average.js b/crates/extensions/tedge_gen_mapper/pipelines/average.js index 128f8774194..c9fcd1f4cf8 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/average.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/average.js @@ -70,6 +70,7 @@ export function process (timestamp, message) { State.agg_for_topic[topic] = agg_payload } + console.log("average.state", State.agg_for_topic) return [] } @@ -86,9 +87,13 @@ export function tick() { else if (v.sum && v.count) { let fragment = { [k]: v.sum / v.count } Object.assign(payload, fragment) - } else for (let [sub_k, sub_v] of Object.entries(v)) { - let fragment = { [k]: { [sub_k]: sub_v.sum / sub_v.count } } - Object.assign(payload, fragment) + } else { + let fragment = {} + for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_fragment = { [sub_k]: sub_v.sum / sub_v.count } + Object.assign(fragment, sub_fragment) + } + Object.assign(payload, { [k]: fragment }) } } diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 2f4fc094a4f..316ad827133 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -228,6 +228,12 @@ impl<'a, 'js> IntoJs<'js> for JsonValueRef<'a> { impl<'js> FromJs<'js> for JsonValue { fn from_js(_ctx: &Ctx<'js>, value: Value<'js>) -> rquickjs::Result { + JsonValue::from_js_value(value) + } +} + +impl JsonValue { + fn from_js_value(value: Value<'_>) -> rquickjs::Result { if let Some(b) = value.as_bool() { return Ok(JsonValue(serde_json::Value::Bool(b))); } @@ -260,6 +266,11 @@ impl<'js> FromJs<'js> for JsonValue { Ok(JsonValue(serde_json::Value::Null)) } + + pub(crate) fn display(value: Value<'_>) -> String { + let json = JsonValue::from_js_value(value).unwrap_or_default(); + serde_json::to_string_pretty(&json.0).unwrap() + } } #[cfg(test)] diff --git a/crates/extensions/tedge_gen_mapper/src/js_runtime.rs b/crates/extensions/tedge_gen_mapper/src/js_runtime.rs index 892b9f3793d..b88118b0758 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_runtime.rs @@ -112,6 +112,7 @@ impl JsWorker { async fn run(mut self) { rquickjs::async_with!(self.context => |ctx| { + console::init(&ctx); let mut modules = JsModules::new(); while let Some(request) = self.requests.recv().await { match request { @@ -177,7 +178,6 @@ impl<'js> JsModules<'js> { })?; let f = rquickjs::Function::from_value(f)?; - debug!(target: "MAPPING", "execute({module_name}.{function})"); let r = match &args[..] { [] => f.call(()), [v0] => f.call((v0,)), @@ -202,3 +202,56 @@ impl<'js> JsModules<'js> { }) } } + +mod console { + use crate::js_filter::JsonValue; + use rquickjs::class::Trace; + use rquickjs::function::Rest; + use rquickjs::Ctx; + use rquickjs::JsLifetime; + use rquickjs::Result; + use rquickjs::Value; + use std::fmt::Write; + + #[derive(Clone, Trace, JsLifetime)] + #[rquickjs::class(frozen)] + struct Console {} + + pub fn init(ctx: &Ctx<'_>) { + let console = Console {}; + let _ = ctx.globals().set("console", console); + } + + impl Console { + fn print(&self, _level: tracing::Level, values: Rest>) -> Result<()> { + let mut message = String::new(); + for (i, value) in values.0.into_iter().enumerate() { + if i > 0 { + let _ = write!(&mut message, ", "); + } + let _ = write!(&mut message, "{}", JsonValue::display(value)); + } + eprintln!("JavaScript.Console: {message}"); + Ok(()) + } + } + + #[rquickjs::methods] + impl Console { + fn debug(&self, values: Rest>) -> Result<()> { + self.print(tracing::Level::DEBUG, values) + } + + fn log(&self, values: Rest>) -> Result<()> { + self.print(tracing::Level::INFO, values) + } + + fn warn(&self, values: Rest>) -> Result<()> { + self.print(tracing::Level::WARN, values) + } + + fn error(&self, values: Rest>) -> Result<()> { + self.print(tracing::Level::ERROR, values) + } + } +} diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.js b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.js index 128f8774194..c9fcd1f4cf8 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.js +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.js @@ -70,6 +70,7 @@ export function process (timestamp, message) { State.agg_for_topic[topic] = agg_payload } + console.log("average.state", State.agg_for_topic) return [] } @@ -86,9 +87,13 @@ export function tick() { else if (v.sum && v.count) { let fragment = { [k]: v.sum / v.count } Object.assign(payload, fragment) - } else for (let [sub_k, sub_v] of Object.entries(v)) { - let fragment = { [k]: { [sub_k]: sub_v.sum / sub_v.count } } - Object.assign(payload, fragment) + } else { + let fragment = {} + for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_fragment = { [sub_k]: sub_v.sum / sub_v.count } + Object.assign(fragment, sub_fragment) + } + Object.assign(payload, { [k]: fragment }) } } diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.samples b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.samples index cee47197527..64f7fbf336e 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.samples +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.samples @@ -1,7 +1,7 @@ INPUT: [test/average] {"temperature": 20} INPUT: [test/average] {"humidity": 50} INPUT: [test/average] {"temperature": 25, "location": {"altitude": 100.0 }} -INPUT: [test/average] {"temperature": 30} +INPUT: [test/average] {"temperature": 30, "location": {"latitude": 45.0 }} INPUT: [test/average] {"temperature": 25} INPUT: [test/average] {"humidity": 70, "location": {"altitude": 120.0 }} INPUT: [test/average] {"temperature": 20} @@ -12,5 +12,5 @@ INPUT: [test/average/another_topic] {"temperature": 6} INPUT: [test/average/another_topic] {"temperature": 4} -OUTPUT: [test/average] {"temperature":24,"humidity":60,"location":{"altitude":110}} +OUTPUT: [test/average] {"temperature":24,"humidity":60,"location":{"altitude":110,"latitude":45}} OUTPUT: [test/average/another_topic] {"temperature":4} \ No newline at end of file From d798cb814a776c66f8d796053b5ae5894d7737d0 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 1 Jul 2025 10:33:20 +0200 Subject: [PATCH 37/50] Tedge mapping test a single single filter Signed-off-by: Didier Wenzek --- crates/core/tedge/src/cli/mapping/cli.rs | 15 ++ crates/core/tedge/src/cli/mapping/test.rs | 45 +++--- .../extensions/tedge_gen_mapper/src/config.rs | 15 ++ .../tedge_gen_mapper/src/runtime.rs | 129 ++++++++++++++---- .../tedge_gen_mapper/pipelines/average.toml | 5 - .../tedge_gen_mapper/tedge_gen_mapper.robot | 2 +- 6 files changed, 148 insertions(+), 63 deletions(-) delete mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.toml diff --git a/crates/core/tedge/src/cli/mapping/cli.rs b/crates/core/tedge/src/cli/mapping/cli.rs index 4adaff621d4..04345a1c978 100644 --- a/crates/core/tedge/src/cli/mapping/cli.rs +++ b/crates/core/tedge/src/cli/mapping/cli.rs @@ -108,4 +108,19 @@ impl TEdgeMappingCli { ) }) } + + pub async fn load_filter( + mapping_dir: &PathBuf, + path: &PathBuf, + ) -> Result { + if let Some("toml") = path.extension().and_then(|s| s.to_str()) { + MessageProcessor::try_new_single_pipeline(mapping_dir, path) + .await + .with_context(|| format!("loading pipeline {pipeline}", pipeline = path.display())) + } else { + MessageProcessor::try_new_single_filter(mapping_dir, path) + .await + .with_context(|| format!("loading filter {filter}", filter = path.display())) + } + } } diff --git a/crates/core/tedge/src/cli/mapping/test.rs b/crates/core/tedge/src/cli/mapping/test.rs index 79c208b0aab..0b6009faef6 100644 --- a/crates/core/tedge/src/cli/mapping/test.rs +++ b/crates/core/tedge/src/cli/mapping/test.rs @@ -27,7 +27,10 @@ impl Command for TestCommand { } async fn execute(&self, _config: TEdgeConfig) -> Result<(), MaybeFancy> { - let mut processor = TEdgeMappingCli::load_pipelines(&self.mapping_dir).await?; + let mut processor = match &self.filter { + None => TEdgeMappingCli::load_pipelines(&self.mapping_dir).await?, + Some(filter) => TEdgeMappingCli::load_filter(&self.mapping_dir, filter).await?, + }; if let Some(message) = &self.message { let timestamp = DateTime::now(); self.process(&mut processor, message, ×tamp).await; @@ -53,37 +56,21 @@ impl TestCommand { message: &Message, timestamp: &DateTime, ) { - match &self.filter { - Some(filter) => { - let filter_name = filter.display().to_string(); - print( - processor - .process_with_pipeline(&filter_name, timestamp, message) - .await, - ) - } - None => processor - .process(timestamp, message) - .await - .into_iter() - .map(|(_, v)| v) - .for_each(print), - } + processor + .process(timestamp, message) + .await + .into_iter() + .map(|(_, v)| v) + .for_each(print) } async fn tick(&self, processor: &mut MessageProcessor, timestamp: &DateTime) { - match &self.filter { - Some(filter) => { - let filter_name = filter.display().to_string(); - print(processor.tick_with_pipeline(&filter_name, timestamp).await) - } - None => processor - .tick(timestamp) - .await - .into_iter() - .map(|(_, v)| v) - .for_each(print), - } + processor + .tick(timestamp) + .await + .into_iter() + .map(|(_, v)| v) + .for_each(print) } } diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index dd389bd2bae..c2822adef79 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -45,6 +45,20 @@ pub enum ConfigError { } impl PipelineConfig { + pub fn from_filter(filter: Utf8PathBuf) -> Self { + let input_topic = "#".to_string(); + let stage = StageConfig { + filter: FilterSpec::JavaScript(filter), + config: None, + tick_every_seconds: 1, + meta_topics: vec![], + }; + Self { + input_topics: vec![input_topic], + stages: vec![stage], + } + } + pub fn compile( self, js_runtime: &JsRuntime, @@ -69,6 +83,7 @@ impl StageConfig { pub fn compile(self, _js_runtime: &JsRuntime, config_dir: &Path) -> Result { let path = match self.filter { FilterSpec::JavaScript(path) if path.is_absolute() => path.into(), + FilterSpec::JavaScript(path) if path.starts_with(config_dir) => path.into(), FilterSpec::JavaScript(path) => config_dir.join(path), }; let filter = JsFilter::new(path) diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 830ec3bc5c7..f2b2fcbe4c2 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -42,6 +42,43 @@ impl MessageProcessor { }) } + pub async fn try_new_single_pipeline( + config_dir: impl AsRef, + pipeline: impl AsRef, + ) -> Result { + let config_dir = config_dir.as_ref().to_owned(); + let pipeline = pipeline.as_ref().to_owned(); + let mut js_runtime = JsRuntime::try_new().await?; + let mut pipeline_specs = PipelineSpecs::default(); + pipeline_specs + .load_single_pipeline(&mut js_runtime, &config_dir, &pipeline) + .await; + let pipelines = pipeline_specs.compile(&js_runtime, &config_dir); + Ok(MessageProcessor { + config_dir, + pipelines, + js_runtime, + }) + } + + pub async fn try_new_single_filter( + config_dir: impl AsRef, + filter: impl AsRef, + ) -> Result { + let config_dir = config_dir.as_ref().to_owned(); + let mut js_runtime = JsRuntime::try_new().await?; + let mut pipeline_specs = PipelineSpecs::default(); + pipeline_specs + .load_single_filter(&mut js_runtime, &filter) + .await; + let pipelines = pipeline_specs.compile(&js_runtime, &config_dir); + Ok(MessageProcessor { + config_dir, + pipelines, + js_runtime, + }) + } + pub fn subscriptions(&self) -> TopicFilter { let mut topics = TopicFilter::empty(); for pipeline in self.pipelines.values() { @@ -63,19 +100,6 @@ impl MessageProcessor { out_messages } - pub async fn process_with_pipeline( - &mut self, - pipeline_id: &String, - timestamp: &DateTime, - message: &Message, - ) -> Result, FilterError> { - let pipeline = self - .pipelines - .get_mut(pipeline_id) - .ok_or_else(|| anyhow::anyhow!("No such pipeline: {pipeline_id}"))?; - pipeline.process(&self.js_runtime, timestamp, message).await - } - pub async fn tick( &mut self, timestamp: &DateTime, @@ -88,18 +112,6 @@ impl MessageProcessor { out_messages } - pub async fn tick_with_pipeline( - &mut self, - pipeline_id: &String, - timestamp: &DateTime, - ) -> Result, FilterError> { - let pipeline = self - .pipelines - .get_mut(pipeline_id) - .ok_or_else(|| anyhow::anyhow!("No such pipeline: {pipeline_id}"))?; - pipeline.tick(&self.js_runtime, timestamp).await - } - pub async fn dump_memory_stats(&self) { self.js_runtime.dump_memory_stats().await; } @@ -235,10 +247,71 @@ impl PipelineSpecs { } } + pub async fn load_single_pipeline( + &mut self, + js_runtime: &mut JsRuntime, + config_dir: &PathBuf, + pipeline: &Path, + ) { + let Some(path) = Utf8Path::from_path(pipeline).map(|p| p.to_path_buf()) else { + error!(target: "MAPPING", "Skipping non UTF8 path: {}", pipeline.display()); + return; + }; + if let Err(err) = self.load_pipeline(&path).await { + error!(target: "MAPPING", "Failed to load pipeline {path}: {err}"); + return; + } + + let Ok(mut entries) = read_dir(config_dir).await.map_err(|err| + error!(target: "MAPPING", "Failed to read filters from {}: {err}", config_dir.display()) + ) else { + return; + }; + + while let Ok(Some(entry)) = entries.next_entry().await { + let Some(path) = Utf8Path::from_path(&entry.path()).map(|p| p.to_path_buf()) else { + error!(target: "MAPPING", "Skipping non UTF8 path: {}", entry.path().display()); + continue; + }; + if let Ok(file_type) = entry.file_type().await { + if file_type.is_file() { + match path.extension() { + Some("js") | Some("ts") => { + info!(target: "MAPPING", "Loading filter: {path}"); + if let Err(err) = self.load_filter(js_runtime, path).await { + error!(target: "MAPPING", "Failed to load filter: {err}"); + } + } + _ => {} + } + } + } + } + } + + pub async fn load_single_filter( + &mut self, + js_runtime: &mut JsRuntime, + filter: impl AsRef, + ) { + let filter = filter.as_ref(); + let Some(path) = Utf8Path::from_path(filter).map(|p| p.to_path_buf()) else { + error!(target: "MAPPING", "Skipping non UTF8 path: {}", filter.display()); + return; + }; + if let Err(err) = js_runtime.load_file(&path).await { + error!(target: "MAPPING", "Failed to load filter {path}: {err}"); + } + let pipeline_id = MessageProcessor::pipeline_id(&path); + let pipeline = PipelineConfig::from_filter(path.to_owned()); + self.pipeline_specs + .insert(pipeline_id, (path.to_owned(), pipeline)); + } + async fn load_pipeline(&mut self, file: impl AsRef) -> Result<(), LoadError> { let path = file.as_ref(); let pipeline_id = MessageProcessor::pipeline_id(path); - let specs = read_to_string(file.as_ref()).await?; + let specs = read_to_string(path).await?; let pipeline: PipelineConfig = toml::from_str(&specs)?; self.pipeline_specs .insert(pipeline_id, (path.to_owned(), pipeline)); @@ -249,9 +322,9 @@ impl PipelineSpecs { async fn load_filter( &mut self, js_runtime: &mut JsRuntime, - file: impl AsRef, + file: impl AsRef, ) -> Result<(), LoadError> { - js_runtime.load_file(file.as_ref()).await?; + js_runtime.load_file(file).await?; Ok(()) } diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.toml b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.toml deleted file mode 100644 index 52f2bdf7c4a..00000000000 --- a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.toml +++ /dev/null @@ -1,5 +0,0 @@ -input_topics = ["test/average/#"] - -stages = [ - { filter = "average.js", tick_every_seconds = 1 }, -] \ No newline at end of file diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot index 0092d7d400a..fdd9518bfdd 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot +++ b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot @@ -50,7 +50,7 @@ Units are configured using topic metadata Computing average over a time window ${transformed_msg} Execute Command - ... cat /etc/tedge/gen-mapper/average.samples | awk '{ print $2 }' FS\='INPUT:' | tedge mapping test --final-tick + ... cat /etc/tedge/gen-mapper/average.samples | awk '{ print $2 }' FS\='INPUT:' | tedge mapping test --final-tick --filter /etc/tedge/gen-mapper/average.js ... strip=True ${expected_msg} Execute Command ... cat /etc/tedge/gen-mapper/average.samples | awk '{ if ($2) print $2 }' FS\='OUTPUT: ' From 094119f261d95cd80fbc760db6245b89fd1b664b Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 1 Jul 2025 20:06:50 +0200 Subject: [PATCH 38/50] Fix JS engine with one module per filter instance Now, each instance of a script is given its own static state Signed-off-by: Didier Wenzek --- .../extensions/tedge_gen_mapper/src/actor.rs | 4 +- .../extensions/tedge_gen_mapper/src/config.rs | 33 +++-- .../tedge_gen_mapper/src/js_filter.rs | 25 ++-- .../tedge_gen_mapper/src/js_runtime.rs | 16 +-- .../tedge_gen_mapper/src/runtime.rs | 117 ++++-------------- .../pipelines/count-events.toml | 5 + .../pipelines/count-measurements.toml | 5 + .../pipelines/count-messages.js | 22 ++++ .../pipelines/count-messages.samples | 19 +++ .../tedge_gen_mapper/tedge_gen_mapper.robot | 11 ++ 10 files changed, 135 insertions(+), 122 deletions(-) create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-events.toml create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-measurements.toml create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.js create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.samples diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index 4e1daa381f7..2c2e12a1d7f 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -61,9 +61,7 @@ impl Actor for GenMapper { let Ok(path) = Utf8PathBuf::try_from(path) else { continue; }; - if matches!(path.extension(), Some("js" | "ts")) { - self.processor.add_filter(path).await; - } else if path.extension() == Some("toml") { + if matches!(path.extension(), Some("toml")) { self.processor.add_pipeline(path).await; self.send_updated_subscriptions().await?; } diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index c2822adef79..f3e3c80ab3c 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -3,9 +3,11 @@ use crate::js_runtime::JsRuntime; use crate::pipeline::Pipeline; use crate::pipeline::Stage; use crate::LoadError; +use camino::Utf8Path; use camino::Utf8PathBuf; use serde::Deserialize; use serde_json::Value; +use std::fmt::Debug; use std::path::Path; use tedge_mqtt_ext::TopicFilter; @@ -59,20 +61,24 @@ impl PipelineConfig { } } - pub fn compile( + pub async fn compile( self, - js_runtime: &JsRuntime, + js_runtime: &mut JsRuntime, config_dir: &Path, source: Utf8PathBuf, ) -> Result { - let input = topic_filters(&self.input_topics)?; - let stages = self - .stages - .into_iter() - .map(|stage| stage.compile(js_runtime, config_dir)) - .collect::, _>>()?; + let input_topics = topic_filters(&self.input_topics)?; + let mut stages = vec![]; + for (i, stage) in self.stages.into_iter().enumerate() { + let stage = stage.compile(config_dir, i, &source).await?; + let filter = &stage.filter; + js_runtime + .load_file(filter.module_name(), filter.path()) + .await?; + stages.push(stage); + } Ok(Pipeline { - input_topics: input, + input_topics, stages, source, }) @@ -80,13 +86,18 @@ impl PipelineConfig { } impl StageConfig { - pub fn compile(self, _js_runtime: &JsRuntime, config_dir: &Path) -> Result { + pub async fn compile( + self, + config_dir: &Path, + index: usize, + pipeline: &Utf8Path, + ) -> Result { let path = match self.filter { FilterSpec::JavaScript(path) if path.is_absolute() => path.into(), FilterSpec::JavaScript(path) if path.starts_with(config_dir) => path.into(), FilterSpec::JavaScript(path) => config_dir.join(path), }; - let filter = JsFilter::new(path) + let filter = JsFilter::new(pipeline.to_owned().into(), index, path) .with_config(self.config) .with_tick_every_seconds(self.tick_every_seconds); let config_topics = topic_filters(&self.meta_topics)?; diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 316ad827133..9c1c006b756 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -14,6 +14,7 @@ use tracing::debug; #[derive(Clone)] pub struct JsFilter { + pub module_name: String, pub path: PathBuf, pub config: JsonValue, pub tick_every_seconds: u64, @@ -23,8 +24,10 @@ pub struct JsFilter { pub struct JsonValue(serde_json::Value); impl JsFilter { - pub fn new(path: PathBuf) -> Self { + pub fn new(pipeline: PathBuf, index: usize, path: PathBuf) -> Self { + let module_name = format!("{}|{}|{}", pipeline.display(), index, path.display()); JsFilter { + module_name, path, config: JsonValue::default(), tick_every_seconds: 0, @@ -32,7 +35,7 @@ impl JsFilter { } pub fn module_name(&self) -> String { - self.path.display().to_string() + self.module_name.to_owned() } pub fn with_config(self, config: Option) -> Self { @@ -77,7 +80,7 @@ impl JsFilter { message.clone().into(), self.config.clone(), ]; - js.call_function(&self.path, "process", input) + js.call_function(&self.module_name(), "process", input) .await .map_err(pipeline::error_from_js)? .try_into() @@ -98,7 +101,7 @@ impl JsFilter { debug!(target: "MAPPING", "{}: update_config({message:?})", self.module_name()); let input = vec![message.clone().into(), self.config.clone()]; let config = js - .call_function(&self.path, "update_config", input) + .call_function(&self.module_name(), "update_config", input) .await .map_err(pipeline::error_from_js)?; self.config = config; @@ -122,7 +125,7 @@ impl JsFilter { } debug!(target: "MAPPING", "{}: tick({timestamp:?})", self.module_name()); let input = vec![timestamp.clone().into(), self.config.clone()]; - js.call_function(&self.path, "tick", input) + js.call_function(&self.module_name(), "tick", input) .await .map_err(pipeline::error_from_js)? .try_into() @@ -281,8 +284,8 @@ mod tests { async fn identity_filter() { let script = "export function process(t,msg) { return [msg]; };"; let mut runtime = JsRuntime::try_new().await.unwrap(); - runtime.load_js("id.js", script).await.unwrap(); - let filter = JsFilter::new("id.js".into()); + let filter = JsFilter::new("id.toml".into(), 1, "id.js".into()); + runtime.load_js(filter.module_name(), script).await.unwrap(); let input = Message::new("te/main/device///m/", "hello world"); let output = input.clone(); @@ -299,8 +302,8 @@ mod tests { async fn error_filter() { let script = r#"export function process(t,msg) { throw new Error("Cannot process that message"); };"#; let mut runtime = JsRuntime::try_new().await.unwrap(); - runtime.load_js("err.js", script).await.unwrap(); - let filter = JsFilter::new("err.js".into()); + let filter = JsFilter::new("err.toml".into(), 1, "err.js".into()); + runtime.load_js(filter.module_name(), script).await.unwrap(); let input = Message::new("te/main/device///m/", "hello world"); let error = filter @@ -335,8 +338,8 @@ export function process (timestamp, message, config) { } "#; let mut runtime = JsRuntime::try_new().await.unwrap(); - runtime.load_js("collectd.js", script).await.unwrap(); - let filter = JsFilter::new("collectd.js".into()); + let filter = JsFilter::new("collectd.toml".into(), 1, "collectd.js".into()); + runtime.load_js(filter.module_name(), script).await.unwrap(); let input = Message::new( "collectd/h/memory/percent-used", diff --git a/crates/extensions/tedge_gen_mapper/src/js_runtime.rs b/crates/extensions/tedge_gen_mapper/src/js_runtime.rs index b88118b0758..bd58bdb6a80 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_runtime.rs @@ -23,20 +23,22 @@ impl JsRuntime { Ok(JsRuntime { runtime, worker }) } - pub async fn load_file(&mut self, path: impl AsRef) -> Result<(), LoadError> { + pub async fn load_file( + &mut self, + module_name: String, + path: impl AsRef, + ) -> Result<(), LoadError> { let path = path.as_ref(); let source = tokio::fs::read_to_string(path).await?; - self.load_js(path, source).await + self.load_js(module_name, source).await } pub async fn load_js( &mut self, - path: impl AsRef, + name: String, source: impl Into>, ) -> Result<(), LoadError> { let (sender, receiver) = oneshot::channel(); - let path = path.as_ref().to_path_buf(); - let name = path.display().to_string(); let source = source.into(); self.worker .send(JsRequest::LoadModule { @@ -51,14 +53,14 @@ impl JsRuntime { pub async fn call_function( &self, - module: &Path, + module: &str, function: &str, args: Vec, ) -> Result { let (sender, receiver) = oneshot::channel(); self.worker .send(JsRequest::CallFunction { - module: module.display().to_string(), + module: module.to_string(), function: function.to_string(), args, sender, diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index f2b2fcbe4c2..8991f199b63 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -32,8 +32,8 @@ impl MessageProcessor { let config_dir = config_dir.as_ref().to_owned(); let mut js_runtime = JsRuntime::try_new().await?; let mut pipeline_specs = PipelineSpecs::default(); - pipeline_specs.load(&mut js_runtime, &config_dir).await; - let pipelines = pipeline_specs.compile(&js_runtime, &config_dir); + pipeline_specs.load(&config_dir).await; + let pipelines = pipeline_specs.compile(&mut js_runtime, &config_dir).await; Ok(MessageProcessor { config_dir, @@ -50,10 +50,8 @@ impl MessageProcessor { let pipeline = pipeline.as_ref().to_owned(); let mut js_runtime = JsRuntime::try_new().await?; let mut pipeline_specs = PipelineSpecs::default(); - pipeline_specs - .load_single_pipeline(&mut js_runtime, &config_dir, &pipeline) - .await; - let pipelines = pipeline_specs.compile(&js_runtime, &config_dir); + pipeline_specs.load_single_pipeline(&pipeline).await; + let pipelines = pipeline_specs.compile(&mut js_runtime, &config_dir).await; Ok(MessageProcessor { config_dir, pipelines, @@ -68,10 +66,8 @@ impl MessageProcessor { let config_dir = config_dir.as_ref().to_owned(); let mut js_runtime = JsRuntime::try_new().await?; let mut pipeline_specs = PipelineSpecs::default(); - pipeline_specs - .load_single_filter(&mut js_runtime, &filter) - .await; - let pipelines = pipeline_specs.compile(&js_runtime, &config_dir); + pipeline_specs.load_single_filter(&filter).await; + let pipelines = pipeline_specs.compile(&mut js_runtime, &config_dir).await; Ok(MessageProcessor { config_dir, pipelines, @@ -116,22 +112,15 @@ impl MessageProcessor { self.js_runtime.dump_memory_stats().await; } - pub async fn add_filter(&mut self, path: Utf8PathBuf) { - match self.js_runtime.load_file(&path).await { - Ok(()) => { - info!(target: "gen-mapper", "Loaded filter {path}"); - } - Err(e) => { - error!(target: "gen-mapper", "Failed to load filter {path}: {e}"); - } - } - } - pub async fn reload_filter(&mut self, path: Utf8PathBuf) { for pipeline in self.pipelines.values_mut() { for stage in &mut pipeline.stages { if stage.filter.path() == path { - match self.js_runtime.load_file(&path).await { + match self + .js_runtime + .load_file(stage.filter.module_name(), &path) + .await + { Ok(()) => { info!(target: "gen-mapper", "Reloaded filter {path}"); } @@ -168,7 +157,10 @@ impl MessageProcessor { return false; } }; - match config.compile(&self.js_runtime, &self.config_dir, path.clone()) { + match config + .compile(&mut self.js_runtime, &self.config_dir, path.clone()) + .await + { Ok(pipeline) => { self.pipelines.insert(pipeline_id, pipeline); true @@ -211,7 +203,7 @@ struct PipelineSpecs { } impl PipelineSpecs { - pub async fn load(&mut self, js_runtime: &mut JsRuntime, config_dir: &PathBuf) { + pub async fn load(&mut self, config_dir: &PathBuf) { let Ok(mut entries) = read_dir(config_dir).await.map_err(|err| error!(target: "MAPPING", "Failed to read filters from {}: {err}", config_dir.display()) ) else { @@ -225,21 +217,10 @@ impl PipelineSpecs { }; if let Ok(file_type) = entry.file_type().await { if file_type.is_file() { - match path.extension() { - Some("toml") => { - info!(target: "MAPPING", "Loading pipeline: {path}"); - if let Err(err) = self.load_pipeline(path).await { - error!(target: "MAPPING", "Failed to load pipeline: {err}"); - } - } - Some("js") | Some("ts") => { - info!(target: "MAPPING", "Loading filter: {path}"); - if let Err(err) = self.load_filter(js_runtime, path).await { - error!(target: "MAPPING", "Failed to load filter: {err}"); - } - } - _ => { - info!(target: "MAPPING", "Skipping file which type is unknown: {path}"); + if let Some("toml") = path.extension() { + info!(target: "MAPPING", "Loading pipeline: {path}"); + if let Err(err) = self.load_pipeline(path).await { + error!(target: "MAPPING", "Failed to load pipeline: {err}"); } } } @@ -247,61 +228,22 @@ impl PipelineSpecs { } } - pub async fn load_single_pipeline( - &mut self, - js_runtime: &mut JsRuntime, - config_dir: &PathBuf, - pipeline: &Path, - ) { + pub async fn load_single_pipeline(&mut self, pipeline: &Path) { let Some(path) = Utf8Path::from_path(pipeline).map(|p| p.to_path_buf()) else { error!(target: "MAPPING", "Skipping non UTF8 path: {}", pipeline.display()); return; }; if let Err(err) = self.load_pipeline(&path).await { error!(target: "MAPPING", "Failed to load pipeline {path}: {err}"); - return; - } - - let Ok(mut entries) = read_dir(config_dir).await.map_err(|err| - error!(target: "MAPPING", "Failed to read filters from {}: {err}", config_dir.display()) - ) else { - return; - }; - - while let Ok(Some(entry)) = entries.next_entry().await { - let Some(path) = Utf8Path::from_path(&entry.path()).map(|p| p.to_path_buf()) else { - error!(target: "MAPPING", "Skipping non UTF8 path: {}", entry.path().display()); - continue; - }; - if let Ok(file_type) = entry.file_type().await { - if file_type.is_file() { - match path.extension() { - Some("js") | Some("ts") => { - info!(target: "MAPPING", "Loading filter: {path}"); - if let Err(err) = self.load_filter(js_runtime, path).await { - error!(target: "MAPPING", "Failed to load filter: {err}"); - } - } - _ => {} - } - } - } } } - pub async fn load_single_filter( - &mut self, - js_runtime: &mut JsRuntime, - filter: impl AsRef, - ) { + pub async fn load_single_filter(&mut self, filter: impl AsRef) { let filter = filter.as_ref(); let Some(path) = Utf8Path::from_path(filter).map(|p| p.to_path_buf()) else { error!(target: "MAPPING", "Skipping non UTF8 path: {}", filter.display()); return; }; - if let Err(err) = js_runtime.load_file(&path).await { - error!(target: "MAPPING", "Failed to load filter {path}: {err}"); - } let pipeline_id = MessageProcessor::pipeline_id(&path); let pipeline = PipelineConfig::from_filter(path.to_owned()); self.pipeline_specs @@ -319,19 +261,14 @@ impl PipelineSpecs { Ok(()) } - async fn load_filter( - &mut self, + async fn compile( + mut self, js_runtime: &mut JsRuntime, - file: impl AsRef, - ) -> Result<(), LoadError> { - js_runtime.load_file(file).await?; - Ok(()) - } - - fn compile(mut self, js_runtime: &JsRuntime, config_dir: &Path) -> HashMap { + config_dir: &Path, + ) -> HashMap { let mut pipelines = HashMap::new(); for (name, (source, specs)) in self.pipeline_specs.drain() { - match specs.compile(js_runtime, config_dir, source) { + match specs.compile(js_runtime, config_dir, source).await { Ok(pipeline) => { let _ = pipelines.insert(name, pipeline); } diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-events.toml b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-events.toml new file mode 100644 index 00000000000..9dcc8e32f2f --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-events.toml @@ -0,0 +1,5 @@ +input_topics = ["test/+/+/+/+/e/+"] + +stages = [ + { filter = "count-messages.js", config = { topic = "test/count/e" }, tick_every_seconds = 1 }, +] \ No newline at end of file diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-measurements.toml b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-measurements.toml new file mode 100644 index 00000000000..81895dde972 --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-measurements.toml @@ -0,0 +1,5 @@ +input_topics = ["test/+/+/+/+/m/+"] + +stages = [ + { filter = "count-messages.js", config = { topic = "test/count/m" }, tick_every_seconds = 1 }, +] \ No newline at end of file diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.js b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.js new file mode 100644 index 00000000000..5be6c29aa22 --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.js @@ -0,0 +1,22 @@ +class State { + static count_per_topic = {} +} + +export function process (timestamp, message) { + let topic = message.topic + let count = State.count_per_topic[topic] || 0 + State.count_per_topic[topic] = count + 1 + + console.log("current count", State.count_per_topic) + return [] +} + +export function tick(timestamp, config) { + let message = { + topic: config?.topic || "te/error", + payload: JSON.stringify(State.count_per_topic) + } + + State.count_per_topic = {} + return [message] +} \ No newline at end of file diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.samples b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.samples new file mode 100644 index 00000000000..180e028e3ae --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.samples @@ -0,0 +1,19 @@ +INPUT: [test/device/main///m/] "some measurement" +INPUT: [test/device/child1///m/] "some measurement" +INPUT: [test/device/child2///m/] "some measurement" +INPUT: [test/device/main///m/] "some measurement" +INPUT: [test/device/child1///m/] "some measurement" +INPUT: [test/device/child2///m/] "some measurement" +INPUT: [test/device/main///m/] "some measurement" +INPUT: [test/device/child1///m/] "some measurement" +INPUT: [test/device/main///m/] "some measurement" +INPUT: [test/device/main///e/] "some event" +INPUT: [test/device/child2///e/] "some event" +INPUT: [test/device/main///e/] "some event" +INPUT: [test/device/child1///e/] "some event" +INPUT: [test/device/child2///e/] "some event" + +# Since we have two pipelines using the same javascript filter, one expect two output messages +# A first one for all the measurements, another one for all the events +OUTPUT: [test/count/m] {"test/device/main///m/":4,"test/device/child1///m/":3,"test/device/child2///m/":2} +OUTPUT: [test/count/e] {"test/device/main///e/":2,"test/device/child2///e/":2,"test/device/child1///e/":1} diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot index fdd9518bfdd..5ec963cb7ee 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot +++ b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot @@ -59,6 +59,17 @@ Computing average over a time window ... ${transformed_msg} ... ${expected_msg} +Each instance of a script must have its own static state + ${transformed_msg} Execute Command + ... cat /etc/tedge/gen-mapper/count-messages.samples | awk '{ print $2 }' FS\='INPUT:' | tedge mapping test --final-tick | sort + ... strip=True + ${expected_msg} Execute Command + ... cat /etc/tedge/gen-mapper/count-messages.samples | awk '{ if ($2) print $2 }' FS\='OUTPUT: ' | sort + ... strip=True + Should Be Equal + ... ${transformed_msg} + ... ${expected_msg} + *** Keywords *** Custom Setup From cae0eaa4d7c004a38688336df52ec939781c5004 Mon Sep 17 00:00:00 2001 From: reubenmiller Date: Mon, 7 Jul 2025 08:54:46 +0200 Subject: [PATCH 39/50] use config_dir variable to get configuration file location Signed-off-by: reubenmiller --- crates/core/tedge_mapper/src/gen/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/core/tedge_mapper/src/gen/mod.rs b/crates/core/tedge_mapper/src/gen/mod.rs index e89644506de..ee6995e4748 100644 --- a/crates/core/tedge_mapper/src/gen/mod.rs +++ b/crates/core/tedge_mapper/src/gen/mod.rs @@ -11,13 +11,13 @@ impl TEdgeComponent for GenMapper { async fn start( &self, tedge_config: TEdgeConfig, - _config_dir: &tedge_config::Path, + config_dir: &tedge_config::Path, ) -> Result<(), anyhow::Error> { let (mut runtime, mut mqtt_actor) = start_basic_actors("tedge-gen-mapper", &tedge_config).await?; let mut fs_actor = FsWatchActorBuilder::new(); - let mut gen_mapper = GenMapperBuilder::try_new("/etc/tedge/gen-mapper").await?; + let mut gen_mapper = GenMapperBuilder::try_new(config_dir.join("gen-mapper")).await?; gen_mapper.connect(&mut mqtt_actor); gen_mapper.connect_fs(&mut fs_actor); From b184a825a79a483ff25e9f664f2ca65b63eea30e Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Mon, 7 Jul 2025 15:14:33 +0200 Subject: [PATCH 40/50] Gen-mapper pipelines can drain data out MeaDB Signed-off-by: Didier Wenzek --- Cargo.lock | 1 + crates/extensions/tedge_gen_mapper/Cargo.toml | 1 + .../extensions/tedge_gen_mapper/src/actor.rs | 32 ++++++++++- .../extensions/tedge_gen_mapper/src/config.rs | 56 ++++++++++++++++--- .../tedge_gen_mapper/src/pipeline.rs | 37 ++++++++++-- .../tedge_gen_mapper/src/runtime.rs | 22 ++++++++ 6 files changed, 133 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d988c45b71..01c32d331eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4982,6 +4982,7 @@ dependencies = [ "anyhow", "async-trait", "camino", + "humantime", "rquickjs", "serde", "serde_json", diff --git a/crates/extensions/tedge_gen_mapper/Cargo.toml b/crates/extensions/tedge_gen_mapper/Cargo.toml index 9b5a4b47a23..79fbe3b3add 100644 --- a/crates/extensions/tedge_gen_mapper/Cargo.toml +++ b/crates/extensions/tedge_gen_mapper/Cargo.toml @@ -12,6 +12,7 @@ repository.workspace = true anyhow = { workspace = true } async-trait = { workspace = true } camino = { workspace = true, features = ["serde1"] } +humantime = { workspace = true } rquickjs = { workspace = true, default-features = false, features = [ "futures", "macro", diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index 2c2e12a1d7f..6b7ac9c0c5c 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -5,6 +5,7 @@ use crate::InputMessage; use crate::OutputMessage; use async_trait::async_trait; use camino::Utf8PathBuf; +use std::fmt::Debug; use tedge_actors::Actor; use tedge_actors::MessageReceiver; use tedge_actors::RuntimeError; @@ -37,11 +38,14 @@ impl Actor for GenMapper { tokio::select! { _ = interval.tick() => { self.tick().await?; + + let drained_messages = self.drain_db().await?; + self.filter_all(drained_messages).await?; } message = self.messages.recv() => { match message { Some(InputMessage::MqttMessage(message)) => match Message::try_from(message) { - Ok(message) => self.filter(message).await?, + Ok(message) => self.filter(DateTime::now(), message).await?, Err(err) => { error!(target: "gen-mapper", "Cannot process message: {err}"); } @@ -102,8 +106,14 @@ impl GenMapper { diff } - async fn filter(&mut self, message: Message) -> Result<(), RuntimeError> { - let timestamp = DateTime::now(); + async fn filter_all(&mut self, messages: Vec<(DateTime, Message)>) -> Result<(), RuntimeError> { + for (timestamp, message) in messages { + self.filter(timestamp, message).await? + } + Ok(()) + } + + async fn filter(&mut self, timestamp: DateTime, message: Message) -> Result<(), RuntimeError> { for (pipeline_id, pipeline_messages) in self.processor.process(×tamp, &message).await { match pipeline_messages { Ok(messages) => { @@ -158,4 +168,20 @@ impl GenMapper { Ok(()) } + + async fn drain_db(&mut self) -> Result, RuntimeError> { + let timestamp = DateTime::now(); + let mut messages = vec![]; + for (pipeline_id, pipeline_messages) in self.processor.drain_db(×tamp).await { + match pipeline_messages { + Ok(pipeline_messages) => { + messages.extend(pipeline_messages); + } + Err(err) => { + error!(target: "gen-mapper", "{pipeline_id}: {err}"); + } + } + } + Ok(messages) + } } diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index f3e3c80ab3c..e44a7db6508 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -1,6 +1,7 @@ use crate::js_filter::JsFilter; use crate::js_runtime::JsRuntime; use crate::pipeline::Pipeline; +use crate::pipeline::PipelineInput; use crate::pipeline::Stage; use crate::LoadError; use camino::Utf8Path; @@ -9,11 +10,13 @@ use serde::Deserialize; use serde_json::Value; use std::fmt::Debug; use std::path::Path; +use std::time::Duration; use tedge_mqtt_ext::TopicFilter; #[derive(Deserialize)] pub struct PipelineConfig { - input_topics: Vec, + #[serde(flatten)] + input: InputConfig, stages: Vec, } @@ -37,6 +40,19 @@ pub enum FilterSpec { JavaScript(Utf8PathBuf), } +#[derive(Deserialize)] +#[serde(untagged)] +pub enum InputConfig { + MQTT { + input_topics: Vec, + }, + MeaDB { + input_series: String, + input_frequency: Duration, + input_span: Duration, + }, +} + #[derive(thiserror::Error, Debug)] pub enum ConfigError { #[error("Not a valid MQTT topic filter: {0}")] @@ -56,7 +72,9 @@ impl PipelineConfig { meta_topics: vec![], }; Self { - input_topics: vec![input_topic], + input: InputConfig::MQTT { + input_topics: vec![input_topic], + }, stages: vec![stage], } } @@ -67,7 +85,7 @@ impl PipelineConfig { config_dir: &Path, source: Utf8PathBuf, ) -> Result { - let input_topics = topic_filters(&self.input_topics)?; + let input = self.input.try_into()?; let mut stages = vec![]; for (i, stage) in self.stages.into_iter().enumerate() { let stage = stage.compile(config_dir, i, &source).await?; @@ -78,7 +96,7 @@ impl PipelineConfig { stages.push(stage); } Ok(Pipeline { - input_topics, + input, stages, source, }) @@ -100,7 +118,7 @@ impl StageConfig { let filter = JsFilter::new(pipeline.to_owned().into(), index, path) .with_config(self.config) .with_tick_every_seconds(self.tick_every_seconds); - let config_topics = topic_filters(&self.meta_topics)?; + let config_topics = topic_filters(self.meta_topics)?; Ok(Stage { filter, config_topics, @@ -108,12 +126,36 @@ impl StageConfig { } } -fn topic_filters(patterns: &Vec) -> Result { +impl TryFrom for PipelineInput { + type Error = ConfigError; + + fn try_from(input: InputConfig) -> Result { + match input { + InputConfig::MQTT { input_topics } => Ok(PipelineInput::MQTT { + input_topics: topic_filters(input_topics)?, + }), + InputConfig::MeaDB { + input_series, + input_frequency, + input_span, + } => { + let input_frequency = input_frequency.as_secs(); + Ok(PipelineInput::MeaDB { + input_series, + input_frequency, + input_span, + }) + } + } + } +} + +fn topic_filters(patterns: Vec) -> Result { let mut topics = TopicFilter::empty(); for pattern in patterns { topics .add(pattern.as_str()) - .map_err(|_| ConfigError::IncorrectTopicFilter(pattern.clone()))?; + .map_err(|_| ConfigError::IncorrectTopicFilter(pattern))?; } Ok(topics) } diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index a87daef8359..7ff6e04f46a 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -4,6 +4,7 @@ use crate::LoadError; use camino::Utf8PathBuf; use serde_json::json; use serde_json::Value; +use std::time::Duration; use tedge_mqtt_ext::MqttMessage; use tedge_mqtt_ext::TopicFilter; use time::OffsetDateTime; @@ -11,11 +12,12 @@ use time::OffsetDateTime; /// A chain of transformation of MQTT messages pub struct Pipeline { /// The source topics - pub input_topics: TopicFilter, + pub input: PipelineInput, /// Transformation stages to apply in order to the messages pub stages: Vec, + /// Path to pipeline source code pub source: Utf8PathBuf, } @@ -25,6 +27,17 @@ pub struct Stage { pub config_topics: TopicFilter, } +pub enum PipelineInput { + MQTT { + input_topics: TopicFilter, + }, + MeaDB { + input_series: String, + input_frequency: u64, + input_span: Duration, + }, +} + #[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Eq, PartialEq)] pub struct DateTime { pub seconds: u64, @@ -51,11 +64,23 @@ pub enum FilterError { impl Pipeline { pub fn topics(&self) -> TopicFilter { - let mut topics = self.input_topics.clone(); - for stage in self.stages.iter() { - topics.add_all(stage.config_topics.clone()) + match &self.input { + PipelineInput::MQTT { input_topics } => { + let mut topics = input_topics.clone(); + for stage in self.stages.iter() { + topics.add_all(stage.config_topics.clone()) + } + topics + } + PipelineInput::MeaDB { .. } => TopicFilter::empty(), + } + } + + pub fn accept(&self, message_topic: &str) -> bool { + match &self.input { + PipelineInput::MQTT { input_topics } => input_topics.accept_topic_name(message_topic), + PipelineInput::MeaDB { .. } => true, } - topics } pub async fn update_config( @@ -78,7 +103,7 @@ impl Pipeline { message: &Message, ) -> Result, FilterError> { self.update_config(js_runtime, message).await?; - if !self.input_topics.accept_topic_name(&message.topic) { + if !self.accept(&message.topic) { return Ok(vec![]); } diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 8991f199b63..c0e74339a39 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -4,6 +4,7 @@ use crate::pipeline::DateTime; use crate::pipeline::FilterError; use crate::pipeline::Message; use crate::pipeline::Pipeline; +use crate::pipeline::PipelineInput; use crate::LoadError; use camino::Utf8Path; use camino::Utf8PathBuf; @@ -108,6 +109,27 @@ impl MessageProcessor { out_messages } + pub async fn drain_db( + &mut self, + timestamp: &DateTime, + ) -> Vec<(String, Result, FilterError>)> { + let mut out_messages = vec![]; + for (pipeline_id, pipeline) in self.pipelines.iter() { + if let PipelineInput::MeaDB { + input_series, + input_frequency, + input_span, + } = &pipeline.input + { + if timestamp.tick_now(*input_frequency) { + let drained_messages = Ok(vec![]); // db.drain_older(input_series, timestamp, input_span).await; + out_messages.push((pipeline_id.to_owned(), drained_messages)); + } + } + } + out_messages + } + pub async fn dump_memory_stats(&self) { self.js_runtime.dump_memory_stats().await; } From cba0b4b31de365324dd6518786f670f4ebc3d0cd Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Mon, 7 Jul 2025 16:12:43 +0200 Subject: [PATCH 41/50] Gen-mapper pipelines can persist data in MeaDB Signed-off-by: Didier Wenzek --- .../extensions/tedge_gen_mapper/src/actor.rs | 44 ++++++++++------- .../extensions/tedge_gen_mapper/src/config.rs | 49 +++++++++++++++++-- .../tedge_gen_mapper/src/pipeline.rs | 19 +++++++ .../tedge_gen_mapper/src/runtime.rs | 25 +++++++++- 4 files changed, 115 insertions(+), 22 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index 6b7ac9c0c5c..53b72154825 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -1,11 +1,10 @@ -use crate::pipeline::DateTime; +use crate::pipeline::{DateTime, PipelineOutput}; use crate::pipeline::Message; use crate::runtime::MessageProcessor; use crate::InputMessage; use crate::OutputMessage; use async_trait::async_trait; use camino::Utf8PathBuf; -use std::fmt::Debug; use tedge_actors::Actor; use tedge_actors::MessageReceiver; use tedge_actors::RuntimeError; @@ -117,18 +116,7 @@ impl GenMapper { for (pipeline_id, pipeline_messages) in self.processor.process(×tamp, &message).await { match pipeline_messages { Ok(messages) => { - for message in messages { - match MqttMessage::try_from(message) { - Ok(message) => { - self.messages - .send(OutputMessage::MqttMessage(message)) - .await? - } - Err(err) => { - error!(target: "gen-mapper", "{pipeline_id}: cannot send transformed message: {err}") - } - } - } + self.publish_messages(pipeline_id, timestamp.clone(), messages).await?; } Err(err) => { error!(target: "gen-mapper", "{pipeline_id}: {err}"); @@ -147,21 +135,43 @@ impl GenMapper { for (pipeline_id, pipeline_messages) in self.processor.tick(×tamp).await { match pipeline_messages { Ok(messages) => { + self.publish_messages(pipeline_id, timestamp.clone(), messages).await?; + } + Err(err) => { + error!(target: "gen-mapper", "{pipeline_id}: {err}"); + } + } + } + + Ok(()) + } + + async fn publish_messages(&mut self, pipeline_id: String, timestamp: DateTime, messages: Vec) -> Result<(), RuntimeError> { + if let Some(pipeline) = self.processor.pipelines.get(&pipeline_id) { + match &pipeline.output { + PipelineOutput::MQTT { output_topics } => { for message in messages { match MqttMessage::try_from(message) { - Ok(message) => { + Ok(message) if output_topics.accept_topic(&message.topic)=> { self.messages .send(OutputMessage::MqttMessage(message)) .await? } + Ok(message) => { + error!(target: "gen-mapper", "{pipeline_id}: reject out-of-scope message: {}", message.topic) + } Err(err) => { error!(target: "gen-mapper", "{pipeline_id}: cannot send transformed message: {err}") } } } } - Err(err) => { - error!(target: "gen-mapper", "{pipeline_id}: {err}"); + PipelineOutput::MeaDB { output_series } => { + for message in messages { + if let Err(err) = self.processor.database.store(output_series.clone(), timestamp.clone(), message).await { + error!(target: "gen-mapper", "{pipeline_id}: fail to persist message: {err}"); + } + } } } } diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index e44a7db6508..7aea8589cb0 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -2,6 +2,7 @@ use crate::js_filter::JsFilter; use crate::js_runtime::JsRuntime; use crate::pipeline::Pipeline; use crate::pipeline::PipelineInput; +use crate::pipeline::PipelineOutput; use crate::pipeline::Stage; use crate::LoadError; use camino::Utf8Path; @@ -17,7 +18,11 @@ use tedge_mqtt_ext::TopicFilter; pub struct PipelineConfig { #[serde(flatten)] input: InputConfig, + stages: Vec, + + #[serde(flatten)] + output: OutputConfig, } #[derive(Deserialize)] @@ -43,7 +48,7 @@ pub enum FilterSpec { #[derive(Deserialize)] #[serde(untagged)] pub enum InputConfig { - MQTT { + Mqtt { input_topics: Vec, }, MeaDB { @@ -53,6 +58,17 @@ pub enum InputConfig { }, } +#[derive(Deserialize)] +#[serde(untagged)] +pub enum OutputConfig { + Mqtt { + output_topics: Vec, + }, + MeaDB { + output_series: String, + }, +} + #[derive(thiserror::Error, Debug)] pub enum ConfigError { #[error("Not a valid MQTT topic filter: {0}")] @@ -65,6 +81,7 @@ pub enum ConfigError { impl PipelineConfig { pub fn from_filter(filter: Utf8PathBuf) -> Self { let input_topic = "#".to_string(); + let output_topic = "#".to_string(); let stage = StageConfig { filter: FilterSpec::JavaScript(filter), config: None, @@ -72,10 +89,13 @@ impl PipelineConfig { meta_topics: vec![], }; Self { - input: InputConfig::MQTT { + input: InputConfig::Mqtt { input_topics: vec![input_topic], }, stages: vec![stage], + output: OutputConfig::Mqtt { + output_topics: vec![output_topic], + }, } } @@ -86,6 +106,7 @@ impl PipelineConfig { source: Utf8PathBuf, ) -> Result { let input = self.input.try_into()?; + let output = self.output.try_into()?; let mut stages = vec![]; for (i, stage) in self.stages.into_iter().enumerate() { let stage = stage.compile(config_dir, i, &source).await?; @@ -99,6 +120,7 @@ impl PipelineConfig { input, stages, source, + output, }) } } @@ -131,7 +153,7 @@ impl TryFrom for PipelineInput { fn try_from(input: InputConfig) -> Result { match input { - InputConfig::MQTT { input_topics } => Ok(PipelineInput::MQTT { + InputConfig::Mqtt { input_topics } => Ok(PipelineInput::MQTT { input_topics: topic_filters(input_topics)?, }), InputConfig::MeaDB { @@ -159,3 +181,24 @@ fn topic_filters(patterns: Vec) -> Result { } Ok(topics) } + +impl Default for OutputConfig { + fn default() -> Self { + OutputConfig::Mqtt { + output_topics: vec!["#".to_string()], + } + } +} + +impl TryFrom for PipelineOutput { + type Error = ConfigError; + + fn try_from(value: OutputConfig) -> Result { + match value { + OutputConfig::Mqtt { output_topics } => Ok(PipelineOutput::MQTT { + output_topics: topic_filters(output_topics)?, + }), + OutputConfig::MeaDB { output_series } => Ok(PipelineOutput::MeaDB { output_series }), + } + } +} \ No newline at end of file diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index 7ff6e04f46a..bb396ca526f 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -19,6 +19,9 @@ pub struct Pipeline { /// Path to pipeline source code pub source: Utf8PathBuf, + + /// Target of the transformed messages + pub output: PipelineOutput, } /// A message transformation stage @@ -38,6 +41,15 @@ pub enum PipelineInput { }, } +pub enum PipelineOutput { + MQTT { + output_topics: TopicFilter, + }, + MeaDB { + output_series: String, + }, +} + #[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Eq, PartialEq)] pub struct DateTime { pub seconds: u64, @@ -155,6 +167,13 @@ impl DateTime { pub fn json(&self) -> Value { json!({"seconds": self.seconds, "nanoseconds": self.nanoseconds}) } + + pub fn sub(&self, duration: &Duration) -> Self { + DateTime { + seconds: self.seconds - duration.as_secs(), + nanoseconds: self.nanoseconds, + } + } } impl TryFrom for DateTime { diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index c0e74339a39..77c1080d0f5 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -22,6 +22,24 @@ pub struct MessageProcessor { pub config_dir: PathBuf, pub pipelines: HashMap, pub(super) js_runtime: JsRuntime, + pub database: MeaDB, +} + +pub struct MeaDB {} + +#[derive(thiserror::Error, Debug)] +pub enum DatabaseError {} + +impl MeaDB { + pub async fn open(_path: &Path) -> Result { + Ok(MeaDB{}) + } + pub async fn store(&mut self, _series: String, _timestamp: DateTime, _message: Message) -> Result<(), DatabaseError> { + Ok(()) + } + pub async fn drain_older(&mut self, _series: &str, _timestamp: &DateTime) -> Result, DatabaseError> { + Ok(vec![]) + } } impl MessageProcessor { @@ -40,6 +58,7 @@ impl MessageProcessor { config_dir, pipelines, js_runtime, + database: MeaDB{}, }) } @@ -57,6 +76,7 @@ impl MessageProcessor { config_dir, pipelines, js_runtime, + database: MeaDB{}, }) } @@ -73,6 +93,7 @@ impl MessageProcessor { config_dir, pipelines, js_runtime, + database: MeaDB{}, }) } @@ -112,7 +133,7 @@ impl MessageProcessor { pub async fn drain_db( &mut self, timestamp: &DateTime, - ) -> Vec<(String, Result, FilterError>)> { + ) -> Vec<(String, Result, DatabaseError>)> { let mut out_messages = vec![]; for (pipeline_id, pipeline) in self.pipelines.iter() { if let PipelineInput::MeaDB { @@ -122,7 +143,7 @@ impl MessageProcessor { } = &pipeline.input { if timestamp.tick_now(*input_frequency) { - let drained_messages = Ok(vec![]); // db.drain_older(input_series, timestamp, input_span).await; + let drained_messages = self.database.drain_older(input_series, ×tamp.sub(input_span)).await; out_messages.push((pipeline_id.to_owned(), drained_messages)); } } From 663d408d09eed004f9ee82713ac5219868483da8 Mon Sep 17 00:00:00 2001 From: James Rhodes Date: Mon, 7 Jul 2025 17:35:46 +0100 Subject: [PATCH 42/50] add initial mea database implementation to the generic mapper Signed-off-by: James Rhodes --- Cargo.lock | 186 ++++++++++++ Cargo.toml | 1 + crates/extensions/tedge_gen_mapper/Cargo.toml | 1 + .../extensions/tedge_gen_mapper/src/actor.rs | 25 +- .../extensions/tedge_gen_mapper/src/config.rs | 10 +- crates/extensions/tedge_gen_mapper/src/lib.rs | 3 + .../tedge_gen_mapper/src/pipeline.rs | 26 +- .../tedge_gen_mapper/src/runtime.rs | 269 ++++++++++++++++-- 8 files changed, 483 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 01c32d331eb..5ab01d43832 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -783,6 +783,12 @@ dependencies = [ "serde", ] +[[package]] +name = "byteview" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5" + [[package]] name = "c8y-firmware-plugin" version = "1.5.1" @@ -1144,6 +1150,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "compare" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1255,6 +1267,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-skiplist" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -1385,6 +1407,20 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "data-encoding" version = "2.9.0" @@ -1497,6 +1533,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "double-ended-peekable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57" + [[package]] name = "downcast" version = "0.11.0" @@ -1555,6 +1597,18 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "env_logger" version = "0.10.2" @@ -1654,6 +1708,23 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "fjall" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5cb653019268f6dc8de3b254b633a2d233a775054349b804b9cfbf18bbe3426" +dependencies = [ + "byteorder", + "byteview", + "dashmap", + "log", + "lsm-tree", + "path-absolutize", + "std-semaphore", + "tempfile", + "xxhash-rust", +] + [[package]] name = "flate2" version = "1.1.1" @@ -1895,6 +1966,12 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +[[package]] +name = "guardian" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f" + [[package]] name = "h2" version = "0.4.10" @@ -2357,6 +2434,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "interval-heap" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11274e5e8e89b8607cfedc2910b6626e998779b48a019151c7604d0adcb86ac6" +dependencies = [ + "compare", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -2567,6 +2653,36 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lsm-tree" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2bd4cdc451a8dcf11329190afb9b78eb8988bed07a3da29b8d73d2e0c731ff" +dependencies = [ + "byteorder", + "crossbeam-skiplist", + "double-ended-peekable", + "enum_dispatch", + "guardian", + "interval-heap", + "log", + "lz4_flex", + "path-absolutize", + "quick_cache", + "rustc-hash", + "self_cell", + "tempfile", + "value-log", + "varint-rs", + "xxhash-rust", +] + +[[package]] +name = "lz4_flex" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" + [[package]] name = "mach2" version = "0.4.2" @@ -3088,12 +3204,30 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "path-absolutize" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5" +dependencies = [ + "path-dedot", +] + [[package]] name = "path-clean" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef" +[[package]] +name = "path-dedot" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397" +dependencies = [ + "once_cell", +] + [[package]] name = "pathdiff" version = "0.2.3" @@ -3427,6 +3561,16 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick_cache" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b450dad8382b1b95061d5ca1eb792081fb082adf48c678791fe917509596d5f" +dependencies = [ + "equivalent", + "hashbrown 0.15.3", +] + [[package]] name = "quinn" version = "0.11.8" @@ -4209,6 +4353,12 @@ dependencies = [ "libc", ] +[[package]] +name = "self_cell" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749" + [[package]] name = "semver" version = "1.0.26" @@ -4423,6 +4573,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "std-semaphore" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e" + [[package]] name = "strsim" version = "0.10.0" @@ -4982,6 +5138,7 @@ dependencies = [ "anyhow", "async-trait", "camino", + "fjall", "humantime", "rquickjs", "serde", @@ -5868,6 +6025,29 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "value-log" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c" +dependencies = [ + "byteorder", + "byteview", + "interval-heap", + "log", + "path-absolutize", + "rustc-hash", + "tempfile", + "varint-rs", + "xxhash-rust", +] + +[[package]] +name = "varint-rs" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" + [[package]] name = "version_check" version = "0.9.5" @@ -6444,6 +6624,12 @@ dependencies = [ "rustix 1.0.7", ] +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/Cargo.toml b/Cargo.toml index 7c7b41d9704..cd40a854112 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,6 +110,7 @@ env_logger = "0.10" fastrand = "2.0" figment = { version = "0.10" } filetime = "0.2" +fjall = "2.11" flate2 = "1.1.1" freedesktop_entry_parser = "1.3.0" futures = "0.3" diff --git a/crates/extensions/tedge_gen_mapper/Cargo.toml b/crates/extensions/tedge_gen_mapper/Cargo.toml index 79fbe3b3add..16ec1581622 100644 --- a/crates/extensions/tedge_gen_mapper/Cargo.toml +++ b/crates/extensions/tedge_gen_mapper/Cargo.toml @@ -12,6 +12,7 @@ repository.workspace = true anyhow = { workspace = true } async-trait = { workspace = true } camino = { workspace = true, features = ["serde1"] } +fjall = { workspace = true } humantime = { workspace = true } rquickjs = { workspace = true, default-features = false, features = [ "futures", diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index 53b72154825..31d33ff1708 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -1,5 +1,6 @@ -use crate::pipeline::{DateTime, PipelineOutput}; +use crate::pipeline::DateTime; use crate::pipeline::Message; +use crate::pipeline::PipelineOutput; use crate::runtime::MessageProcessor; use crate::InputMessage; use crate::OutputMessage; @@ -116,7 +117,8 @@ impl GenMapper { for (pipeline_id, pipeline_messages) in self.processor.process(×tamp, &message).await { match pipeline_messages { Ok(messages) => { - self.publish_messages(pipeline_id, timestamp.clone(), messages).await?; + self.publish_messages(pipeline_id, timestamp.clone(), messages) + .await?; } Err(err) => { error!(target: "gen-mapper", "{pipeline_id}: {err}"); @@ -135,7 +137,8 @@ impl GenMapper { for (pipeline_id, pipeline_messages) in self.processor.tick(×tamp).await { match pipeline_messages { Ok(messages) => { - self.publish_messages(pipeline_id, timestamp.clone(), messages).await?; + self.publish_messages(pipeline_id, timestamp.clone(), messages) + .await?; } Err(err) => { error!(target: "gen-mapper", "{pipeline_id}: {err}"); @@ -146,13 +149,18 @@ impl GenMapper { Ok(()) } - async fn publish_messages(&mut self, pipeline_id: String, timestamp: DateTime, messages: Vec) -> Result<(), RuntimeError> { + async fn publish_messages( + &mut self, + pipeline_id: String, + timestamp: DateTime, + messages: Vec, + ) -> Result<(), RuntimeError> { if let Some(pipeline) = self.processor.pipelines.get(&pipeline_id) { match &pipeline.output { PipelineOutput::MQTT { output_topics } => { for message in messages { match MqttMessage::try_from(message) { - Ok(message) if output_topics.accept_topic(&message.topic)=> { + Ok(message) if output_topics.accept_topic(&message.topic) => { self.messages .send(OutputMessage::MqttMessage(message)) .await? @@ -168,7 +176,12 @@ impl GenMapper { } PipelineOutput::MeaDB { output_series } => { for message in messages { - if let Err(err) = self.processor.database.store(output_series.clone(), timestamp.clone(), message).await { + if let Err(err) = self + .processor + .database + .store(output_series, timestamp.clone(), message) + .await + { error!(target: "gen-mapper", "{pipeline_id}: fail to persist message: {err}"); } } diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index 7aea8589cb0..6e3f499b86c 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -61,12 +61,8 @@ pub enum InputConfig { #[derive(Deserialize)] #[serde(untagged)] pub enum OutputConfig { - Mqtt { - output_topics: Vec, - }, - MeaDB { - output_series: String, - }, + Mqtt { output_topics: Vec }, + MeaDB { output_series: String }, } #[derive(thiserror::Error, Debug)] @@ -201,4 +197,4 @@ impl TryFrom for PipelineOutput { OutputConfig::MeaDB { output_series } => Ok(PipelineOutput::MeaDB { output_series }), } } -} \ No newline at end of file +} diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index f2e22e8a283..f60deed3c4e 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -120,4 +120,7 @@ pub enum LoadError { #[error(transparent)] Anyhow(#[from] anyhow::Error), + + #[error(transparent)] + Fjall(#[from] fjall::Error), } diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index bb396ca526f..86990473c76 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -42,20 +42,32 @@ pub enum PipelineInput { } pub enum PipelineOutput { - MQTT { - output_topics: TopicFilter, - }, - MeaDB { - output_series: String, - }, + MQTT { output_topics: TopicFilter }, + MeaDB { output_series: String }, } -#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, serde::Deserialize, serde::Serialize, Eq, PartialEq)] pub struct DateTime { pub seconds: u64, pub nanoseconds: u32, } +impl Ord for DateTime { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + use std::cmp::Ordering; + match self.seconds.cmp(&other.seconds) { + Ordering::Equal => self.nanoseconds.cmp(&other.nanoseconds), + ordering => ordering, + } + } +} + +impl PartialOrd for DateTime { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + #[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Eq, PartialEq)] pub struct Message { pub topic: String, diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 77c1080d0f5..173de713ab7 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -8,12 +8,18 @@ use crate::pipeline::PipelineInput; use crate::LoadError; use camino::Utf8Path; use camino::Utf8PathBuf; +use fjall::Keyspace; +use fjall::PartitionCreateOptions; +use fjall::Slice; +use std::collections::BTreeMap; use std::collections::HashMap; +use std::marker::PhantomData; use std::path::Path; use std::path::PathBuf; use tedge_mqtt_ext::TopicFilter; use tokio::fs::read_dir; use tokio::fs::read_to_string; +use tokio::task::spawn_blocking; use tracing::error; use tracing::info; use tracing::warn; @@ -25,23 +31,14 @@ pub struct MessageProcessor { pub database: MeaDB, } -pub struct MeaDB {} - #[derive(thiserror::Error, Debug)] -pub enum DatabaseError {} - -impl MeaDB { - pub async fn open(_path: &Path) -> Result { - Ok(MeaDB{}) - } - pub async fn store(&mut self, _series: String, _timestamp: DateTime, _message: Message) -> Result<(), DatabaseError> { - Ok(()) - } - pub async fn drain_older(&mut self, _series: &str, _timestamp: &DateTime) -> Result, DatabaseError> { - Ok(vec![]) - } +pub enum DatabaseError { + #[error(transparent)] + Fjall(#[from] fjall::Error), } +pub type MeaDB = MeaDb; + impl MessageProcessor { pub fn pipeline_id(path: impl AsRef) -> String { format!("{}", path.as_ref().display()) @@ -58,7 +55,7 @@ impl MessageProcessor { config_dir, pipelines, js_runtime, - database: MeaDB{}, + database: MeaDb::open(todo!()).await?, }) } @@ -76,7 +73,7 @@ impl MessageProcessor { config_dir, pipelines, js_runtime, - database: MeaDB{}, + database: MeaDb::open(todo!()).await?, }) } @@ -93,7 +90,7 @@ impl MessageProcessor { config_dir, pipelines, js_runtime, - database: MeaDB{}, + database: MeaDb::open(todo!()).await?, }) } @@ -143,7 +140,11 @@ impl MessageProcessor { } = &pipeline.input { if timestamp.tick_now(*input_frequency) { - let drained_messages = self.database.drain_older(input_series, ×tamp.sub(input_span)).await; + let drained_messages = self + .database + .drain_older_than(timestamp.sub(input_span), input_series) + .await + .map_err(DatabaseError::from); out_messages.push((pipeline_id.to_owned(), drained_messages)); } } @@ -323,3 +324,235 @@ impl PipelineSpecs { pipelines } } + +pub struct MeaDb { + keyspace: Keyspace, + oldest: BTreeMap, + _payload: PhantomData, +} + +pub trait ToFromSlice { + fn to_slice(&self) -> Slice; + fn from_slice(slice: Slice) -> Self; +} + +impl ToFromSlice for DateTime { + fn to_slice(&self) -> Slice { + let mut arr = [0u8; 12]; + unsafe { + *std::mem::transmute::<_, *mut u64>(arr.as_mut_ptr()) = self.seconds.to_be(); + *std::mem::transmute::<_, *mut u32>(arr.as_mut_ptr().offset(8)) = + self.nanoseconds.to_be(); + } + Slice::new(&arr) + } + + fn from_slice(slice: Slice) -> Self { + let secs_be = &slice[..8]; + let nanos_be = &slice[8..]; + let secs = u64::from_be_bytes(secs_be.try_into().unwrap()); + let nanos = u32::from_be_bytes(nanos_be.try_into().unwrap()); + + Self { + seconds: secs, + nanoseconds: nanos, + } + } +} + +impl ToFromSlice for Message { + fn to_slice(&self) -> Slice { + Slice::new(self.json().to_string().as_bytes()) + } + + fn from_slice(slice: Slice) -> Self { + serde_json::from_slice(&*slice).unwrap() + } +} + +impl MeaDb +where + Payload: ToFromSlice + Send + 'static, + Timestamp: ToFromSlice + Ord + Copy + Send + 'static, +{ + pub async fn drain_older_than( + &mut self, + timestamp: Timestamp, + series: &str, + ) -> Result, fjall::Error> { + let ks = self.keyspace.clone(); + let (messages, new_oldest) = spawn_blocking({ + let series = series.to_owned(); + move || { + let partition = ks.open_partition(&series, PartitionCreateOptions::default())?; + let messages = partition + .range(..=timestamp.to_slice()) + .map(|res| res.map(Self::decode)) + .collect::, _>>()?; + for msg in &messages { + partition.remove(msg.0.to_slice())?; + } + Ok::<_, fjall::Error>((messages, partition.first_key_value()?)) + } + }) + .await + .unwrap()?; + + self.oldest.remove(series); + if let Some((ts, _payload)) = new_oldest { + self.update_oldest(&series, Timestamp::from_slice(ts)); + } + Ok(messages) + } + + fn decode((key, value): (Slice, Slice)) -> (Timestamp, Payload) { + (Timestamp::from_slice(key), Payload::from_slice(value)) + } + + pub async fn open(path: impl AsRef + Send) -> Result { + let path = path.as_ref().to_owned(); + let keyspace = spawn_blocking(move || fjall::Config::new(path).open()) + .await + .unwrap()?; + Ok(Self { + keyspace, + oldest: <_>::default(), + _payload: PhantomData, + }) + } + + pub async fn store( + &mut self, + series: &str, + timestamp: Timestamp, + payload: Payload, + ) -> Result<(), fjall::Error> { + let result = spawn_blocking({ + let ks = self.keyspace.clone(); + let series = series.to_owned(); + move || { + let partition = ks.open_partition(&series, PartitionCreateOptions::default())?; + partition.insert(timestamp.to_slice(), payload.to_slice())?; + Ok(()) + } + }) + .await + .unwrap(); + self.update_oldest(&series, timestamp); + result + } + + fn update_oldest(&mut self, topic: &str, inserted_ts: Timestamp) { + if let Some(value) = self.oldest.get_mut(topic) { + *value = std::cmp::min(*value, inserted_ts) + } else { + self.oldest.insert(topic.to_owned(), inserted_ts); + } + } +} + +#[cfg(test)] +mod tests { + use time::macros::datetime; + + use super::*; + use std::path::PathBuf; + + // Helper function to create a dummy path + fn dummy_path() -> PathBuf { + PathBuf::from("/tmp/test_db") + } + + impl ToFromSlice for String { + fn to_slice(&self) -> Slice { + Slice::new(self.as_bytes()) + } + + fn from_slice(slice: Slice) -> Self { + String::from_utf8(slice.to_vec()).unwrap() + } + } + + #[tokio::test] + async fn test_store_single_message() { + let path = dummy_path(); + let mut db = MeaDb::open(&path).await.unwrap(); + + let series = "sensor_data"; + let timestamp = datetime!(2023-01-01 10:00 UTC).into(); + let message = "temp: 25C".to_string(); + + let result = db.store(series, timestamp, message.clone()).await; + assert!(result.is_ok()); + + // Verify the message was stored + let stored_messages = db.drain_older_than(timestamp, &series).await.unwrap(); + assert_eq!(stored_messages.len(), 1); + assert_eq!(stored_messages[0], (timestamp, message)); + } + + #[tokio::test] + async fn test_store_multiple_messages_same_series() { + let path = dummy_path(); + let mut db = MeaDb::open(&path).await.unwrap(); + + let series = "sensor_data".to_string(); + let ts1 = datetime!(2023-01-01 10:00 UTC).into(); + let msg1 = "temp: 25C".to_string(); + let ts2 = datetime!(2023-01-01 10:05 UTC).into(); + let msg2 = "temp: 26C".to_string(); + let ts3 = datetime!(2023-01-01 09:55 UTC).into(); + let msg3 = "temp: 24C".to_string(); + + db.store(&series, ts1, msg1.clone()).await.unwrap(); + db.store(&series, ts2, msg2.clone()).await.unwrap(); + db.store(&series, ts3, msg3.clone()).await.unwrap(); + + let stored_messages = db.drain_older_than(ts2, &series).await.unwrap(); + + assert_eq!(stored_messages.len(), 3); + // Verify messages are sorted by timestamp + assert_eq!(stored_messages[0], (ts3, msg3)); + assert_eq!(stored_messages[1], (ts1, msg1)); + assert_eq!(stored_messages[2], (ts2, msg2)); + } + + #[tokio::test] + async fn test_store_messages_different_series() { + let path = dummy_path(); + let mut db = MeaDb::open(&path).await.unwrap(); + + let series1 = "sensor_data_a".to_string(); + let ts1 = datetime!(2023-01-01 10:00 UTC).into(); + let msg1 = "data A1".to_string(); + + let series2 = "sensor_data_b".to_string(); + let ts2 = datetime!(2023-01-01 10:01 UTC).into(); + let msg2 = "data B1".to_string(); + + db.store(&series1, ts1, msg1.clone()).await.unwrap(); + db.store(&series2, ts2, msg2.clone()).await.unwrap(); + + let s1_data = db.drain_older_than(ts1, &series1).await.unwrap(); + let s2_data = db.drain_older_than(ts2, &series2).await.unwrap(); + assert_eq!(s1_data.len(), 1); + assert_eq!(s2_data.len(), 1); + } + + #[tokio::test] + async fn test_drain_removes_data() { + let path = dummy_path(); + let mut db = MeaDb::open(&path).await.unwrap(); + + let series = "sensor_data_a".to_string(); + let timestamp = datetime!(2023-01-01 10:00 UTC).into(); + let msg = "data A1".to_string(); + + db.store(&series, timestamp, msg.clone()).await.unwrap(); + + let data = db.drain_older_than(timestamp, &series).await.unwrap(); + assert_eq!(data.len(), 1); + let data_after_drain = db.drain_older_than(timestamp, &series).await.unwrap(); + assert_eq!(data_after_drain.len(), 0); + } +} From 3f434ef38b8eebf3748242f75eeb21b4acb4caa7 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Mon, 7 Jul 2025 18:45:12 +0200 Subject: [PATCH 43/50] fixup! add initial mea database implementation to the generic mapper --- crates/extensions/tedge_gen_mapper/src/runtime.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 173de713ab7..193d6dac755 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -55,10 +55,14 @@ impl MessageProcessor { config_dir, pipelines, js_runtime, - database: MeaDb::open(todo!()).await?, + database: MeaDb::open(Self::db_path()).await?, }) } + fn db_path() -> Utf8PathBuf { + todo!() + } + pub async fn try_new_single_pipeline( config_dir: impl AsRef, pipeline: impl AsRef, @@ -73,7 +77,7 @@ impl MessageProcessor { config_dir, pipelines, js_runtime, - database: MeaDb::open(todo!()).await?, + database: MeaDb::open(Self::db_path()).await?, }) } @@ -90,7 +94,7 @@ impl MessageProcessor { config_dir, pipelines, js_runtime, - database: MeaDb::open(todo!()).await?, + database: MeaDb::open(Self::db_path()).await?, }) } From fdd578debdd330f994b8be0c65057fe2e4da5ff9 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Mon, 7 Jul 2025 18:55:25 +0200 Subject: [PATCH 44/50] Add example: pipeline using mea-db Signed-off-by: Didier Wenzek --- .../tedge_gen_mapper/pipelines/collectd.toml | 6 +- .../pipelines/daily_aggregate.toml | 9 +++ .../pipelines/hourly_aggregate.toml | 9 +++ .../tedge_gen_mapper/pipelines/loop.toml | 2 +- .../pipelines/measurements.toml | 4 +- .../extensions/tedge_gen_mapper/src/actor.rs | 8 ++- .../extensions/tedge_gen_mapper/src/config.rs | 65 +++++++++++-------- .../tedge_gen_mapper/src/pipeline.rs | 20 +++--- .../tedge_gen_mapper/src/runtime.rs | 8 +-- 9 files changed, 86 insertions(+), 45 deletions(-) create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/daily_aggregate.toml create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/hourly_aggregate.toml diff --git a/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml b/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml index 05330b870f8..b742f7246df 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml +++ b/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml @@ -1,6 +1,8 @@ -input_topics = ["collectd/+/+/+"] +input.mqtt.topics = ["collectd/+/+/+"] stages = [ { filter = "collectd-to-te.js" }, - { filter = "average.js", tick_every_seconds = 10 } + { filter = "average.js", tick_every_seconds = 60 } ] + +output.db.series = "latest-data-points" diff --git a/crates/extensions/tedge_gen_mapper/pipelines/daily_aggregate.toml b/crates/extensions/tedge_gen_mapper/pipelines/daily_aggregate.toml new file mode 100644 index 00000000000..6fee9991f26 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/daily_aggregate.toml @@ -0,0 +1,9 @@ +input.db.series = "latest-15min-average" +input.db.frequency = "4h" +input.db.max_age = "1d" + +stages = [ + { filter = "average.js" } +] + +output.db.series = "latest-4hour-average" \ No newline at end of file diff --git a/crates/extensions/tedge_gen_mapper/pipelines/hourly_aggregate.toml b/crates/extensions/tedge_gen_mapper/pipelines/hourly_aggregate.toml new file mode 100644 index 00000000000..fd1fd6e866b --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/hourly_aggregate.toml @@ -0,0 +1,9 @@ +input.db.series = "latest-data-points" +input.db.frequency = "15min" +input.db.max_age = "1h" + +stages = [ + { filter = "average.js" } +] + +output.db.series = "latest-15min-average" \ No newline at end of file diff --git a/crates/extensions/tedge_gen_mapper/pipelines/loop.toml b/crates/extensions/tedge_gen_mapper/pipelines/loop.toml index 64304daf4a3..c871df9b93b 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/loop.toml +++ b/crates/extensions/tedge_gen_mapper/pipelines/loop.toml @@ -1,5 +1,5 @@ # This pipeline is on purpose looping: the messages are published to the same topic -input_topics = ["loopback/#"] +input.mqtt.topics = ["loopback/#"] stages = [ { filter = "add_timestamp.js" }, diff --git a/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml b/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml index 1edde2696e6..d34609d0913 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml +++ b/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml @@ -1,7 +1,9 @@ -input_topics = ["te/+/+/+/+/m/+"] +input.mqtt.topics = ["te/+/+/+/+/m/+"] stages = [ { filter = "add_timestamp.js" }, { filter = "drop_stragglers.js", config = { max_delay = 60 } }, { filter = "te_to_c8y.js", meta_topics = ["te/+/+/+/+/m/+/meta"] } ] + +output.mqtt.topics = ["c8y/#"] diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index 31d33ff1708..6d8903f357f 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -157,7 +157,9 @@ impl GenMapper { ) -> Result<(), RuntimeError> { if let Some(pipeline) = self.processor.pipelines.get(&pipeline_id) { match &pipeline.output { - PipelineOutput::MQTT { output_topics } => { + PipelineOutput::MQTT { + topics: output_topics, + } => { for message in messages { match MqttMessage::try_from(message) { Ok(message) if output_topics.accept_topic(&message.topic) => { @@ -174,7 +176,9 @@ impl GenMapper { } } } - PipelineOutput::MeaDB { output_series } => { + PipelineOutput::MeaDB { + series: output_series, + } => { for message in messages { if let Err(err) = self .processor diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index 6e3f499b86c..fd9e65c7f6f 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -16,12 +16,11 @@ use tedge_mqtt_ext::TopicFilter; #[derive(Deserialize)] pub struct PipelineConfig { - #[serde(flatten)] input: InputConfig, stages: Vec, - #[serde(flatten)] + #[serde(default)] output: OutputConfig, } @@ -46,23 +45,25 @@ pub enum FilterSpec { } #[derive(Deserialize)] -#[serde(untagged)] pub enum InputConfig { - Mqtt { - input_topics: Vec, - }, + #[serde(rename = "mqtt")] + Mqtt { topics: Vec }, + #[serde(rename = "db")] MeaDB { - input_series: String, - input_frequency: Duration, - input_span: Duration, + series: String, + #[serde(deserialize_with = "parse_human_duration")] + frequency: Duration, + #[serde(deserialize_with = "parse_human_duration")] + max_age: Duration, }, } #[derive(Deserialize)] -#[serde(untagged)] pub enum OutputConfig { - Mqtt { output_topics: Vec }, - MeaDB { output_series: String }, + #[serde(rename = "mqtt")] + Mqtt { topics: Vec }, + #[serde(rename = "db")] + MeaDB { series: String }, } #[derive(thiserror::Error, Debug)] @@ -86,11 +87,11 @@ impl PipelineConfig { }; Self { input: InputConfig::Mqtt { - input_topics: vec![input_topic], + topics: vec![input_topic], }, stages: vec![stage], output: OutputConfig::Mqtt { - output_topics: vec![output_topic], + topics: vec![output_topic], }, } } @@ -149,19 +150,19 @@ impl TryFrom for PipelineInput { fn try_from(input: InputConfig) -> Result { match input { - InputConfig::Mqtt { input_topics } => Ok(PipelineInput::MQTT { - input_topics: topic_filters(input_topics)?, + InputConfig::Mqtt { topics } => Ok(PipelineInput::MQTT { + topics: topic_filters(topics)?, }), InputConfig::MeaDB { - input_series, - input_frequency, - input_span, + series, + frequency, + max_age: span, } => { - let input_frequency = input_frequency.as_secs(); + let frequency = frequency.as_secs(); Ok(PipelineInput::MeaDB { - input_series, - input_frequency, - input_span, + series, + frequency, + max_age: span, }) } } @@ -181,7 +182,7 @@ fn topic_filters(patterns: Vec) -> Result { impl Default for OutputConfig { fn default() -> Self { OutputConfig::Mqtt { - output_topics: vec!["#".to_string()], + topics: vec!["#".to_string()], } } } @@ -191,10 +192,20 @@ impl TryFrom for PipelineOutput { fn try_from(value: OutputConfig) -> Result { match value { - OutputConfig::Mqtt { output_topics } => Ok(PipelineOutput::MQTT { - output_topics: topic_filters(output_topics)?, + OutputConfig::Mqtt { + topics: output_topics, + } => Ok(PipelineOutput::MQTT { + topics: topic_filters(output_topics)?, }), - OutputConfig::MeaDB { output_series } => Ok(PipelineOutput::MeaDB { output_series }), + OutputConfig::MeaDB { series } => Ok(PipelineOutput::MeaDB { series }), } } } + +pub fn parse_human_duration<'de, D>(deserializer: D) -> Result +where + D: serde::de::Deserializer<'de>, +{ + let value = String::deserialize(deserializer)?; + humantime::parse_duration(&value).map_err(|_| serde::de::Error::custom("Invalid duration")) +} diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index 86990473c76..0b1561e8ff2 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -32,18 +32,18 @@ pub struct Stage { pub enum PipelineInput { MQTT { - input_topics: TopicFilter, + topics: TopicFilter, }, MeaDB { - input_series: String, - input_frequency: u64, - input_span: Duration, + series: String, + frequency: u64, + max_age: Duration, }, } pub enum PipelineOutput { - MQTT { output_topics: TopicFilter }, - MeaDB { output_series: String }, + MQTT { topics: TopicFilter }, + MeaDB { series: String }, } #[derive(Clone, Copy, Debug, serde::Deserialize, serde::Serialize, Eq, PartialEq)] @@ -89,7 +89,9 @@ pub enum FilterError { impl Pipeline { pub fn topics(&self) -> TopicFilter { match &self.input { - PipelineInput::MQTT { input_topics } => { + PipelineInput::MQTT { + topics: input_topics, + } => { let mut topics = input_topics.clone(); for stage in self.stages.iter() { topics.add_all(stage.config_topics.clone()) @@ -102,7 +104,9 @@ impl Pipeline { pub fn accept(&self, message_topic: &str) -> bool { match &self.input { - PipelineInput::MQTT { input_topics } => input_topics.accept_topic_name(message_topic), + PipelineInput::MQTT { + topics: input_topics, + } => input_topics.accept_topic_name(message_topic), PipelineInput::MeaDB { .. } => true, } } diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 193d6dac755..64eaddf67fc 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -60,7 +60,7 @@ impl MessageProcessor { } fn db_path() -> Utf8PathBuf { - todo!() + "/etc/tedge/tedge-gen.db".into() } pub async fn try_new_single_pipeline( @@ -138,9 +138,9 @@ impl MessageProcessor { let mut out_messages = vec![]; for (pipeline_id, pipeline) in self.pipelines.iter() { if let PipelineInput::MeaDB { - input_series, - input_frequency, - input_span, + series: input_series, + frequency: input_frequency, + max_age: input_span, } = &pipeline.input { if timestamp.tick_now(*input_frequency) { From 3e8bbf46b1a898f41111938c5a3af717de7fa403 Mon Sep 17 00:00:00 2001 From: James Rhodes Date: Tue, 8 Jul 2025 09:17:50 +0100 Subject: [PATCH 45/50] Fix segfault Signed-off-by: James Rhodes --- .../tedge_gen_mapper/src/runtime.rs | 49 ++++++++++++++----- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 64eaddf67fc..685c4a68e91 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -343,11 +343,8 @@ pub trait ToFromSlice { impl ToFromSlice for DateTime { fn to_slice(&self) -> Slice { let mut arr = [0u8; 12]; - unsafe { - *std::mem::transmute::<_, *mut u64>(arr.as_mut_ptr()) = self.seconds.to_be(); - *std::mem::transmute::<_, *mut u32>(arr.as_mut_ptr().offset(8)) = - self.nanoseconds.to_be(); - } + *&mut arr[0..8].copy_from_slice(&self.seconds.to_be_bytes()); + *&mut arr[8..12].copy_from_slice(&self.nanoseconds.to_be_bytes()); Slice::new(&arr) } @@ -483,7 +480,11 @@ mod tests { let mut db = MeaDb::open(&path).await.unwrap(); let series = "sensor_data"; - let timestamp = datetime!(2023-01-01 10:00 UTC).into(); + let seconds = datetime!(2023-01-01 10:00 UTC).unix_timestamp(); + let timestamp = DateTime { + seconds: seconds as u64, + nanoseconds: 0, + }; let message = "temp: 25C".to_string(); let result = db.store(series, timestamp, message.clone()).await; @@ -501,11 +502,23 @@ mod tests { let mut db = MeaDb::open(&path).await.unwrap(); let series = "sensor_data".to_string(); - let ts1 = datetime!(2023-01-01 10:00 UTC).into(); + let ts1 = datetime!(2023-01-01 10:00 UTC).unix_timestamp(); + let ts1 = DateTime { + seconds: ts1 as u64, + nanoseconds: 0, + }; let msg1 = "temp: 25C".to_string(); - let ts2 = datetime!(2023-01-01 10:05 UTC).into(); + let ts2 = datetime!(2023-01-01 10:05 UTC).unix_timestamp(); + let ts2 = DateTime { + seconds: ts2 as u64, + nanoseconds: 0, + }; let msg2 = "temp: 26C".to_string(); - let ts3 = datetime!(2023-01-01 09:55 UTC).into(); + let ts3 = datetime!(2023-01-01 09:55 UTC).unix_timestamp(); + let ts3 = DateTime { + seconds: ts3 as u64, + nanoseconds: 0, + }; let msg3 = "temp: 24C".to_string(); db.store(&series, ts1, msg1.clone()).await.unwrap(); @@ -527,11 +540,19 @@ mod tests { let mut db = MeaDb::open(&path).await.unwrap(); let series1 = "sensor_data_a".to_string(); - let ts1 = datetime!(2023-01-01 10:00 UTC).into(); + let ts1 = datetime!(2023-01-01 10:00 UTC).unix_timestamp(); + let ts1 = DateTime { + seconds: ts1 as u64, + nanoseconds: 0, + }; let msg1 = "data A1".to_string(); let series2 = "sensor_data_b".to_string(); - let ts2 = datetime!(2023-01-01 10:01 UTC).into(); + let ts2 = datetime!(2023-01-01 10:01 UTC).unix_timestamp(); + let ts2 = DateTime { + seconds: ts2 as u64, + nanoseconds: 0, + }; let msg2 = "data B1".to_string(); db.store(&series1, ts1, msg1.clone()).await.unwrap(); @@ -549,7 +570,11 @@ mod tests { let mut db = MeaDb::open(&path).await.unwrap(); let series = "sensor_data_a".to_string(); - let timestamp = datetime!(2023-01-01 10:00 UTC).into(); + let timestamp = datetime!(2023-01-01 10:00 UTC).unix_timestamp(); + let timestamp = DateTime { + seconds: timestamp as u64, + nanoseconds: 0, + }; let msg = "data A1".to_string(); db.store(&series, timestamp, msg.clone()).await.unwrap(); From a6dc1ecdfaee584738aea31473ac0e9055a1e35f Mon Sep 17 00:00:00 2001 From: James Rhodes Date: Tue, 8 Jul 2025 09:45:08 +0100 Subject: [PATCH 46/50] Remove oldest message timestamp cache Signed-off-by: James Rhodes --- .../tedge_gen_mapper/src/runtime.rs | 55 ++++++------------- 1 file changed, 18 insertions(+), 37 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 685c4a68e91..36c49d51865 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -11,7 +11,6 @@ use camino::Utf8PathBuf; use fjall::Keyspace; use fjall::PartitionCreateOptions; use fjall::Slice; -use std::collections::BTreeMap; use std::collections::HashMap; use std::marker::PhantomData; use std::path::Path; @@ -331,8 +330,7 @@ impl PipelineSpecs { pub struct MeaDb { keyspace: Keyspace, - oldest: BTreeMap, - _payload: PhantomData, + _types: PhantomData<(Timestamp, Payload)>, } pub trait ToFromSlice { @@ -376,14 +374,25 @@ where Payload: ToFromSlice + Send + 'static, Timestamp: ToFromSlice + Ord + Copy + Send + 'static, { + pub async fn open(path: impl AsRef + Send) -> Result { + let path = path.as_ref().to_owned(); + let keyspace = spawn_blocking(move || fjall::Config::new(path).open()) + .await + .unwrap()?; + Ok(Self { + keyspace, + _types: PhantomData, + }) + } + pub async fn drain_older_than( &mut self, timestamp: Timestamp, series: &str, ) -> Result, fjall::Error> { let ks = self.keyspace.clone(); - let (messages, new_oldest) = spawn_blocking({ - let series = series.to_owned(); + let series = series.to_owned(); + spawn_blocking({ move || { let partition = ks.open_partition(&series, PartitionCreateOptions::default())?; let messages = partition @@ -393,42 +402,24 @@ where for msg in &messages { partition.remove(msg.0.to_slice())?; } - Ok::<_, fjall::Error>((messages, partition.first_key_value()?)) + Ok(messages) } }) .await - .unwrap()?; - - self.oldest.remove(series); - if let Some((ts, _payload)) = new_oldest { - self.update_oldest(&series, Timestamp::from_slice(ts)); - } - Ok(messages) + .unwrap() } fn decode((key, value): (Slice, Slice)) -> (Timestamp, Payload) { (Timestamp::from_slice(key), Payload::from_slice(value)) } - pub async fn open(path: impl AsRef + Send) -> Result { - let path = path.as_ref().to_owned(); - let keyspace = spawn_blocking(move || fjall::Config::new(path).open()) - .await - .unwrap()?; - Ok(Self { - keyspace, - oldest: <_>::default(), - _payload: PhantomData, - }) - } - pub async fn store( &mut self, series: &str, timestamp: Timestamp, payload: Payload, ) -> Result<(), fjall::Error> { - let result = spawn_blocking({ + spawn_blocking({ let ks = self.keyspace.clone(); let series = series.to_owned(); move || { @@ -438,17 +429,7 @@ where } }) .await - .unwrap(); - self.update_oldest(&series, timestamp); - result - } - - fn update_oldest(&mut self, topic: &str, inserted_ts: Timestamp) { - if let Some(value) = self.oldest.get_mut(topic) { - *value = std::cmp::min(*value, inserted_ts) - } else { - self.oldest.insert(topic.to_owned(), inserted_ts); - } + .unwrap() } } From 8b9ce3211af5447eba07ca4899d493f8d1ca6878 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 8 Jul 2025 11:26:44 +0200 Subject: [PATCH 47/50] Distinguish two message sources: MQTT vs DB Signed-off-by: Didier Wenzek --- crates/core/tedge/src/cli/mapping/test.rs | 2 +- .../tedge_gen_mapper/pipelines/average.js | 3 +- .../pipelines/hourly_aggregate.toml | 4 +-- .../extensions/tedge_gen_mapper/src/actor.rs | 29 +++++++++++++++---- .../tedge_gen_mapper/src/pipeline.rs | 15 +++++++--- .../tedge_gen_mapper/src/runtime.rs | 6 +++- 6 files changed, 44 insertions(+), 15 deletions(-) diff --git a/crates/core/tedge/src/cli/mapping/test.rs b/crates/core/tedge/src/cli/mapping/test.rs index 0b6009faef6..4b98975b84c 100644 --- a/crates/core/tedge/src/cli/mapping/test.rs +++ b/crates/core/tedge/src/cli/mapping/test.rs @@ -57,7 +57,7 @@ impl TestCommand { timestamp: &DateTime, ) { processor - .process(timestamp, message) + .process(MessageSource::MQTT, timestamp, message) .await .into_iter() .map(|(_, v)| v) diff --git a/crates/extensions/tedge_gen_mapper/pipelines/average.js b/crates/extensions/tedge_gen_mapper/pipelines/average.js index c9fcd1f4cf8..6cb7a8741dc 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/average.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/average.js @@ -8,6 +8,7 @@ class State { } export function process (timestamp, message) { + //console.log("average.process", timestamp, message) let topic = message.topic let payload = JSON.parse(message.payload) let agg_payload = State.agg_for_topic[topic] @@ -70,7 +71,7 @@ export function process (timestamp, message) { State.agg_for_topic[topic] = agg_payload } - console.log("average.state", State.agg_for_topic) + //console.log("average.state", State.agg_for_topic) return [] } diff --git a/crates/extensions/tedge_gen_mapper/pipelines/hourly_aggregate.toml b/crates/extensions/tedge_gen_mapper/pipelines/hourly_aggregate.toml index fd1fd6e866b..984e8563309 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/hourly_aggregate.toml +++ b/crates/extensions/tedge_gen_mapper/pipelines/hourly_aggregate.toml @@ -1,6 +1,6 @@ input.db.series = "latest-data-points" -input.db.frequency = "15min" -input.db.max_age = "1h" +input.db.frequency = "3min" # "15min" +input.db.max_age = "15min" # "1h" stages = [ { filter = "average.js" } diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index 6d8903f357f..e32e4455099 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -1,5 +1,6 @@ use crate::pipeline::DateTime; use crate::pipeline::Message; +use crate::pipeline::MessageSource; use crate::pipeline::PipelineOutput; use crate::runtime::MessageProcessor; use crate::InputMessage; @@ -18,6 +19,7 @@ use tedge_mqtt_ext::TopicFilter; use tokio::time::interval; use tokio::time::Duration; use tracing::error; +use tracing::info; pub struct GenMapper { pub(super) messages: SimpleMessageBox, @@ -40,12 +42,12 @@ impl Actor for GenMapper { self.tick().await?; let drained_messages = self.drain_db().await?; - self.filter_all(drained_messages).await?; + self.filter_all(MessageSource::MeaDB, drained_messages).await?; } message = self.messages.recv() => { match message { Some(InputMessage::MqttMessage(message)) => match Message::try_from(message) { - Ok(message) => self.filter(DateTime::now(), message).await?, + Ok(message) => self.filter(MessageSource::MQTT, DateTime::now(), message).await?, Err(err) => { error!(target: "gen-mapper", "Cannot process message: {err}"); } @@ -106,15 +108,26 @@ impl GenMapper { diff } - async fn filter_all(&mut self, messages: Vec<(DateTime, Message)>) -> Result<(), RuntimeError> { + async fn filter_all( + &mut self, + source: MessageSource, + messages: Vec<(DateTime, Message)>, + ) -> Result<(), RuntimeError> { for (timestamp, message) in messages { - self.filter(timestamp, message).await? + self.filter(source, timestamp, message).await? } Ok(()) } - async fn filter(&mut self, timestamp: DateTime, message: Message) -> Result<(), RuntimeError> { - for (pipeline_id, pipeline_messages) in self.processor.process(×tamp, &message).await { + async fn filter( + &mut self, + source: MessageSource, + timestamp: DateTime, + message: Message, + ) -> Result<(), RuntimeError> { + for (pipeline_id, pipeline_messages) in + self.processor.process(source, ×tamp, &message).await + { match pipeline_messages { Ok(messages) => { self.publish_messages(pipeline_id, timestamp.clone(), messages) @@ -180,6 +193,7 @@ impl GenMapper { series: output_series, } => { for message in messages { + info!(target: "gen-mapper", "store {output_series} @{}.{} [{}] {}", timestamp.seconds, timestamp.nanoseconds, message.topic, message.payload); if let Err(err) = self .processor .database @@ -202,6 +216,9 @@ impl GenMapper { for (pipeline_id, pipeline_messages) in self.processor.drain_db(×tamp).await { match pipeline_messages { Ok(pipeline_messages) => { + for (t, m) in pipeline_messages.iter() { + info!(target: "gen-mapper", "drained: @{}.{} [{}] {}", t.seconds, t.nanoseconds, m.topic, m.payload); + } messages.extend(pipeline_messages); } Err(err) => { diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index 0b1561e8ff2..e4ea1c18649 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -46,6 +46,12 @@ pub enum PipelineOutput { MeaDB { series: String }, } +#[derive(Copy, Clone, Eq, PartialEq)] +pub enum MessageSource { + MQTT, + MeaDB, +} + #[derive(Clone, Copy, Debug, serde::Deserialize, serde::Serialize, Eq, PartialEq)] pub struct DateTime { pub seconds: u64, @@ -102,12 +108,12 @@ impl Pipeline { } } - pub fn accept(&self, message_topic: &str) -> bool { + pub fn accept(&self, source: MessageSource, message_topic: &str) -> bool { match &self.input { PipelineInput::MQTT { topics: input_topics, - } => input_topics.accept_topic_name(message_topic), - PipelineInput::MeaDB { .. } => true, + } => source == MessageSource::MeaDB && input_topics.accept_topic_name(message_topic), + PipelineInput::MeaDB { .. } => source == MessageSource::MeaDB, } } @@ -127,11 +133,12 @@ impl Pipeline { pub async fn process( &mut self, js_runtime: &JsRuntime, + source: MessageSource, timestamp: &DateTime, message: &Message, ) -> Result, FilterError> { self.update_config(js_runtime, message).await?; - if !self.accept(&message.topic) { + if !self.accept(source, &message.topic) { return Ok(vec![]); } diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 36c49d51865..bbe38f87479 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -3,6 +3,7 @@ use crate::js_runtime::JsRuntime; use crate::pipeline::DateTime; use crate::pipeline::FilterError; use crate::pipeline::Message; +use crate::pipeline::MessageSource; use crate::pipeline::Pipeline; use crate::pipeline::PipelineInput; use crate::LoadError; @@ -107,12 +108,15 @@ impl MessageProcessor { pub async fn process( &mut self, + source: MessageSource, timestamp: &DateTime, message: &Message, ) -> Vec<(String, Result, FilterError>)> { let mut out_messages = vec![]; for (pipeline_id, pipeline) in self.pipelines.iter_mut() { - let pipeline_output = pipeline.process(&self.js_runtime, timestamp, message).await; + let pipeline_output = pipeline + .process(&self.js_runtime, source, timestamp, message) + .await; out_messages.push((pipeline_id.clone(), pipeline_output)); } out_messages From 3723b111024ad44dcaa147c49da39a4c31e9b92b Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 8 Jul 2025 11:59:59 +0200 Subject: [PATCH 48/50] fixup! Distinguish two message sources: MQTT vs DB --- crates/extensions/tedge_gen_mapper/pipelines/collectd.toml | 3 +-- crates/extensions/tedge_gen_mapper/src/pipeline.rs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml b/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml index b742f7246df..0be2f55c9ba 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml +++ b/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml @@ -1,8 +1,7 @@ input.mqtt.topics = ["collectd/+/+/+"] stages = [ - { filter = "collectd-to-te.js" }, - { filter = "average.js", tick_every_seconds = 60 } + { filter = "collectd-to-te.js" } ] output.db.series = "latest-data-points" diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index e4ea1c18649..66d38bf4203 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -112,7 +112,7 @@ impl Pipeline { match &self.input { PipelineInput::MQTT { topics: input_topics, - } => source == MessageSource::MeaDB && input_topics.accept_topic_name(message_topic), + } => source == MessageSource::MQTT && input_topics.accept_topic_name(message_topic), PipelineInput::MeaDB { .. } => source == MessageSource::MeaDB, } } From 44968f7564c18363dd15a8f3456047ac58e5d6e5 Mon Sep 17 00:00:00 2001 From: Marcel Guzik Date: Tue, 8 Jul 2025 10:26:26 +0000 Subject: [PATCH 49/50] Add db_dump bin to read and display db contents Signed-off-by: Marcel Guzik --- .../tedge_gen_mapper/src/bin/db_dump.rs | 33 +++++++++++++++++++ crates/extensions/tedge_gen_mapper/src/lib.rs | 2 ++ .../tedge_gen_mapper/src/runtime.rs | 20 +++++++++++ 3 files changed, 55 insertions(+) create mode 100644 crates/extensions/tedge_gen_mapper/src/bin/db_dump.rs diff --git a/crates/extensions/tedge_gen_mapper/src/bin/db_dump.rs b/crates/extensions/tedge_gen_mapper/src/bin/db_dump.rs new file mode 100644 index 00000000000..f4531c6b089 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/src/bin/db_dump.rs @@ -0,0 +1,33 @@ +use anyhow::Context; +use tedge_gen_mapper::pipeline::{DateTime, Message}; + +const DB_PATH: &str = "/etc/tedge/tedge-gen.db"; + +type Record = (DateTime, Message); + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let mut args = std::env::args(); + let series_name = args.nth(1).unwrap_or("latest-data-points".to_string()); + println!("Reading series name: {series_name}"); + + let mut db = tedge_gen_mapper::MeaDB::open(DB_PATH) + .await + .with_context(|| format!("Failed to open DB at path={DB_PATH}"))?; + + let items = db + .query_all(&series_name) + .await + .context("Failed to query for all items")?; + + items.iter().for_each(print_record); + + Ok(()) +} + +fn print_record(record: &Record) { + let time = record.0.seconds; + let topic = &record.1.topic; + let payload = &record.1.payload; + println!("[{time}]\t{topic}\t{payload}") +} diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index f60deed3c4e..4935aac55be 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -27,6 +27,8 @@ use tedge_mqtt_ext::SubscriptionDiff; use tedge_mqtt_ext::TopicFilter; use tracing::error; +pub use runtime::MeaDB; + fan_in_message_type!(InputMessage[MqttMessage, FsWatchEvent]: Clone, Debug, Eq, PartialEq); fan_in_message_type!(OutputMessage[MqttMessage, SubscriptionDiff]: Clone, Debug, Eq, PartialEq); diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index bbe38f87479..cb51eb21bd6 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -389,6 +389,26 @@ where }) } + pub async fn query_all( + &mut self, + series: &str, + ) -> Result, fjall::Error> { + let ks = self.keyspace.clone(); + let series = series.to_owned(); + spawn_blocking({ + move || { + let partition = ks.open_partition(&series, PartitionCreateOptions::default())?; + let messages = partition + .iter() + .map(|res| res.map(Self::decode)) + .collect::, _>>()?; + Ok(messages) + } + }) + .await + .unwrap() + } + pub async fn drain_older_than( &mut self, timestamp: Timestamp, From 4ec15fbee1e1379b49b73ea82f561019284c0d58 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 8 Jul 2025 13:47:42 +0200 Subject: [PATCH 50/50] Fix cascading pipelines Signed-off-by: Didier Wenzek --- .../tedge_gen_mapper/pipelines/daily_aggregate.toml | 8 +++----- .../tedge_gen_mapper/pipelines/hourly_aggregate.toml | 2 +- crates/extensions/tedge_gen_mapper/src/actor.rs | 4 ++-- crates/extensions/tedge_gen_mapper/src/bin/db_dump.rs | 3 ++- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/pipelines/daily_aggregate.toml b/crates/extensions/tedge_gen_mapper/pipelines/daily_aggregate.toml index 6fee9991f26..7ce53dc6a35 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/daily_aggregate.toml +++ b/crates/extensions/tedge_gen_mapper/pipelines/daily_aggregate.toml @@ -1,9 +1,7 @@ input.db.series = "latest-15min-average" -input.db.frequency = "4h" -input.db.max_age = "1d" +input.db.frequency = "4h" # "1d" +input.db.max_age = "1h" # "7d" stages = [ - { filter = "average.js" } + { filter = "set_topic.js", config = { topic = "te/discarded" } } ] - -output.db.series = "latest-4hour-average" \ No newline at end of file diff --git a/crates/extensions/tedge_gen_mapper/pipelines/hourly_aggregate.toml b/crates/extensions/tedge_gen_mapper/pipelines/hourly_aggregate.toml index 984e8563309..8b6e396e628 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/hourly_aggregate.toml +++ b/crates/extensions/tedge_gen_mapper/pipelines/hourly_aggregate.toml @@ -3,7 +3,7 @@ input.db.frequency = "3min" # "15min" input.db.max_age = "15min" # "1h" stages = [ - { filter = "average.js" } + { filter = "average.js", tick_every_seconds = 1 } ] output.db.series = "latest-15min-average" \ No newline at end of file diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index e32e4455099..ef6e2f069cf 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -39,10 +39,10 @@ impl Actor for GenMapper { loop { tokio::select! { _ = interval.tick() => { - self.tick().await?; - let drained_messages = self.drain_db().await?; self.filter_all(MessageSource::MeaDB, drained_messages).await?; + + self.tick().await?; } message = self.messages.recv() => { match message { diff --git a/crates/extensions/tedge_gen_mapper/src/bin/db_dump.rs b/crates/extensions/tedge_gen_mapper/src/bin/db_dump.rs index f4531c6b089..a3385b47093 100644 --- a/crates/extensions/tedge_gen_mapper/src/bin/db_dump.rs +++ b/crates/extensions/tedge_gen_mapper/src/bin/db_dump.rs @@ -1,5 +1,6 @@ use anyhow::Context; -use tedge_gen_mapper::pipeline::{DateTime, Message}; +use tedge_gen_mapper::pipeline::DateTime; +use tedge_gen_mapper::pipeline::Message; const DB_PATH: &str = "/etc/tedge/tedge-gen.db";