Skip to content

Commit d28f51a

Browse files
dkhokhlovdkhokhlov
authored
fix: log flood "hit rate limit snoozing" (#550)
* feat: print env config to log at startup (#548) * fix: missing _REGEX suffix in envvar names: LINE_EXCLUSION_REGEX, LINE_INCLUSION_REGEX, REDACT_REGEX, * fix: log flood "hit rate limit, snoozing" Ref: LOG-11879, LOG-18170
1 parent d42c9cc commit d28f51a

File tree

7 files changed

+123
-60
lines changed

7 files changed

+123
-60
lines changed

‎Cargo.lock‎

Lines changed: 12 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎common/config/src/argv.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -171,12 +171,12 @@ pub struct ArgumentOptions {
171171

172172
/// List of regex patterns to exclude log lines.
173173
/// When set, the Agent will NOT send log lines that match any of these patterns.
174-
#[structopt(long, env = env_vars::LINE_EXCLUSION)]
174+
#[structopt(long, env = env_vars::LINE_EXCLUSION_REGEX)]
175175
line_exclusion: Vec<String>,
176176

177177
/// List of regex patterns to include log lines.
178178
/// When set, the Agent will send ONLY log lines that match any of these patterns.
179-
#[structopt(long, env = env_vars::LINE_INCLUSION)]
179+
#[structopt(long, env = env_vars::LINE_INCLUSION_REGEX)]
180180
line_inclusion: Vec<String>,
181181

182182
/// List of Kubernetes pod metadata to include in log lines.
@@ -189,7 +189,7 @@ pub struct ArgumentOptions {
189189

190190
/// List of regex patterns used to mask matching sensitive information (such as PII) before
191191
/// sending it in the log line.
192-
#[structopt(long, env = env_vars::REDACT)]
192+
#[structopt(long, env = env_vars::REDACT_REGEX)]
193193
line_redact: Vec<String>,
194194

195195
/// Show the current agent settings from the configuration sources (default config file

‎common/config/src/env_vars.rs‎

Lines changed: 79 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,68 @@
1-
pub const INGESTION_KEY: &str = "MZ_INGESTION_KEY";
2-
pub const CONFIG_FILE: &str = "MZ_CONFIG_FILE";
3-
pub const LOG_DIRS: &str = "MZ_LOG_DIRS";
4-
pub const TAGS: &str = "MZ_TAGS";
5-
pub const HOST: &str = "MZ_HOST";
6-
pub const ENDPOINT: &str = "MZ_ENDPOINT";
7-
pub const USE_SSL: &str = "MZ_USE_SSL";
8-
pub const USE_COMPRESSION: &str = "MZ_USE_COMPRESSION";
9-
pub const GZIP_LEVEL: &str = "MZ_GZIP_LEVEL";
10-
pub const EXCLUSION_RULES: &str = "MZ_EXCLUSION_RULES";
11-
pub const EXCLUSION_REGEX_RULES: &str = "MZ_EXCLUSION_REGEX_RULES";
12-
pub const INCLUSION_RULES: &str = "MZ_INCLUSION_RULES";
13-
pub const INCLUSION_REGEX_RULES: &str = "MZ_INCLUSION_REGEX_RULES";
14-
pub const K8S_METADATA_LINE_INCLUSION: &str = "MZ_K8S_METADATA_LINE_INCLUSION";
15-
pub const K8S_METADATA_LINE_EXCLUSION: &str = "MZ_K8S_METADATA_LINE_EXCLUSION";
16-
pub const HOSTNAME: &str = "MZ_HOSTNAME";
17-
pub const IP: &str = "MZ_IP";
18-
pub const MAC: &str = "MZ_MAC";
19-
pub const SYSTEMD_JOURNAL_TAILER: &str = "MZ_SYSTEMD_JOURNAL_TAILER";
20-
pub const JOURNALD_PATHS: &str = "MZ_JOURNALD_PATHS";
21-
pub const LOOKBACK: &str = "MZ_LOOKBACK";
22-
pub const DB_PATH: &str = "MZ_DB_PATH";
23-
pub const METRICS_PORT: &str = "MZ_METRICS_PORT";
24-
pub const USE_K8S_LOG_ENRICHMENT: &str = "MZ_USE_K8S_LOG_ENRICHMENT";
25-
pub const LOG_K8S_EVENTS: &str = "MZ_LOG_K8S_EVENTS";
26-
pub const LOG_METRIC_SERVER_STATS: &str = "MZ_LOG_METRIC_SERVER_STATS";
27-
pub const K8S_STARTUP_LEASE: &str = "MZ_K8S_STARTUP_LEASE";
28-
pub const LINE_EXCLUSION: &str = "MZ_LINE_EXCLUSION_REGEX";
29-
pub const LINE_INCLUSION: &str = "MZ_LINE_INCLUSION_REGEX";
30-
pub const REDACT: &str = "MZ_REDACT_REGEX";
31-
pub const INGEST_TIMEOUT: &str = "MZ_INGEST_TIMEOUT";
32-
pub const INGEST_BUFFER_SIZE: &str = "MZ_INGEST_BUFFER_SIZE";
33-
pub const RETRY_DIR: &str = "MZ_RETRY_DIR";
34-
pub const RETRY_DISK_LIMIT: &str = "MZ_RETRY_DISK_LIMIT";
35-
pub const INTERNAL_FS_DELAY: &str = "MZ_INTERNAL_FS_DELAY";
36-
pub const CLEAR_CACHE_INTERVAL: &str = "MZ_CLEAR_CACHE_INTERVAL";
1+
macro_rules! define_env_var {
2+
($name:ident) => {
3+
pub const $name: &str = concat!("MZ_", stringify!($name));
4+
};
5+
}
6+
7+
macro_rules! define_env_vars {
8+
($($name:ident),+ $(,)? ) => {
9+
$(
10+
define_env_var!($name);
11+
)*
12+
pub const ENV_VARS_LIST: &'static [&'static str] = &[$($name),*];
13+
};
14+
}
15+
16+
// env vars names prefixed with "MZ_"
17+
define_env_vars!(
18+
INGESTION_KEY,
19+
CONFIG_FILE,
20+
LOG_DIRS,
21+
TAGS,
22+
HOST,
23+
ENDPOINT,
24+
USE_SSL,
25+
USE_COMPRESSION,
26+
GZIP_LEVEL,
27+
EXCLUSION_RULES,
28+
EXCLUSION_REGEX_RULES,
29+
INCLUSION_RULES,
30+
INCLUSION_REGEX_RULES,
31+
K8S_METADATA_LINE_INCLUSION,
32+
K8S_METADATA_LINE_EXCLUSION,
33+
HOSTNAME,
34+
IP,
35+
MAC,
36+
SYSTEMD_JOURNAL_TAILER,
37+
JOURNALD_PATHS,
38+
LOOKBACK,
39+
DB_PATH,
40+
METRICS_PORT,
41+
USE_K8S_LOG_ENRICHMENT,
42+
LOG_K8S_EVENTS,
43+
LOG_METRIC_SERVER_STATS,
44+
K8S_STARTUP_LEASE,
45+
LINE_EXCLUSION_REGEX,
46+
LINE_INCLUSION_REGEX,
47+
REDACT_REGEX,
48+
INGEST_TIMEOUT,
49+
INGEST_BUFFER_SIZE,
50+
RETRY_DIR,
51+
RETRY_DISK_LIMIT,
52+
INTERNAL_FS_DELAY,
53+
CLEAR_CACHE_INTERVAL,
54+
METADATA_RETRY_DELAY,
55+
META_APP,
56+
META_HOST,
57+
META_ENV,
58+
META_FILE,
59+
META_K8S_FILE,
60+
META_JSON,
61+
META_ANNOTATIONS,
62+
META_LABELS,
63+
NO_CAP,
64+
MOCK_NO_PODS
65+
);
3766

3867
// unused or deprecated
3968
pub const INGESTION_KEY_ALTERNATE: &str = "LOGDNA_AGENT_KEY";
@@ -50,13 +79,17 @@ pub const EXCLUSION_REGEX_RULES_DEPRECATED: &str = "LOGDNA_EXCLUDE_REGEX";
5079
pub const INCLUSION_RULES_DEPRECATED: &str = "LOGDNA_INCLUDE";
5180
pub const INCLUSION_REGEX_RULES_DEPRECATED: &str = "LOGDNA_INCLUDE_REGEX";
5281

53-
pub const META_APP: &str = "MZ_META_APP";
54-
pub const META_HOST: &str = "MZ_META_HOST";
55-
pub const META_ENV: &str = "MZ_META_ENV";
56-
pub const META_FILE: &str = "MZ_META_FILE";
57-
pub const META_K8S_FILE: &str = "MZ_META_K8S_FILE";
58-
pub const META_JSON: &str = "MZ_META_JSON";
59-
pub const META_ANNOTATIONS: &str = "MZ_META_ANNOTATIONS";
60-
pub const META_LABELS: &str = "MZ_META_LABELS";
61-
62-
pub const NO_CAP: &str = "MZ_NO_CAP";
82+
pub(crate) const DEPRECATED_ENV_VARS_LIST: &[&str] = &[
83+
INGESTION_KEY_ALTERNATE,
84+
CONFIG_FILE_DEPRECATED,
85+
HOST_DEPRECATED,
86+
IBM_HOST_DEPRECATED,
87+
ENDPOINT_DEPRECATED,
88+
USE_SSL_DEPRECATED,
89+
GZIP_LEVEL_DEPRECATED,
90+
LOG_DIRS_DEPRECATED,
91+
EXCLUSION_RULES_DEPRECATED,
92+
EXCLUSION_REGEX_RULES_DEPRECATED,
93+
INCLUSION_RULES_DEPRECATED,
94+
INCLUSION_REGEX_RULES_DEPRECATED,
95+
];

‎common/config/src/lib.rs‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -194,8 +194,24 @@ impl Config {
194194
print_settings(&yaml_str, &config_path);
195195
}
196196

197+
let env_config: String = env_vars::ENV_VARS_LIST
198+
.iter()
199+
.chain(env_vars::DEPRECATED_ENV_VARS_LIST.iter())
200+
.filter_map(|&key| {
201+
std::env::var(key).ok().map(|value| {
202+
if key.contains("KEY") || key.contains("PIN") {
203+
format!("{}: REDACTED", key)
204+
} else {
205+
format!("{}: {}", key, value)
206+
}
207+
})
208+
})
209+
.collect::<Vec<String>>()
210+
.join("\n");
211+
info!("env config: \n{}", env_config);
212+
197213
info!(
198-
"read the following options from cli, env and config: \n{}",
214+
"effective configuration collected from cli, env and config:\n{}",
199215
yaml_str
200216
);
201217

@@ -670,7 +686,7 @@ mod tests {
670686
.open(&path)
671687
.unwrap();
672688

673-
guard(file, |mut file| {
689+
let _ = guard(file, |mut file| {
674690
let args = vec![OsString::new()];
675691
serde_yaml::to_writer(&mut file, &RawConfig::default()).unwrap();
676692
file.flush().unwrap();

‎common/config/src/properties.rs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,9 @@ from_env_name!(K8S_METADATA_LINE_INCLUSION);
4949
from_env_name!(K8S_METADATA_LINE_EXCLUSION);
5050
from_env_name!(LOG_METRIC_SERVER_STATS);
5151
from_env_name!(K8S_STARTUP_LEASE);
52-
from_env_name!(LINE_EXCLUSION);
53-
from_env_name!(LINE_INCLUSION);
54-
from_env_name!(REDACT);
52+
from_env_name!(LINE_EXCLUSION_REGEX);
53+
from_env_name!(LINE_INCLUSION_REGEX);
54+
from_env_name!(REDACT_REGEX);
5555
from_env_name!(INGEST_TIMEOUT);
5656
from_env_name!(INGEST_BUFFER_SIZE);
5757
from_env_name!(RETRY_DIR);
@@ -270,14 +270,14 @@ fn from_property_map(map: HashMap<String, String>) -> Result<Config, ConfigError
270270
result.log.db_path = map.get(&DB_PATH).map(PathBuf::from);
271271
result.startup.option = map.get_string(&K8S_STARTUP_LEASE);
272272

273-
if let Some(value) = map.get(&LINE_EXCLUSION) {
273+
if let Some(value) = map.get(&LINE_EXCLUSION_REGEX) {
274274
let regex_rules = result.log.line_exclusion_regex.get_or_insert(Vec::new());
275275
argv::split_by_comma(value)
276276
.iter()
277277
.for_each(|v| regex_rules.push(v.to_string()));
278278
}
279279

280-
if let Some(value) = map.get(&LINE_INCLUSION) {
280+
if let Some(value) = map.get(&LINE_INCLUSION_REGEX) {
281281
let regex_rules = result.log.line_inclusion_regex.get_or_insert(Vec::new());
282282
argv::split_by_comma(value)
283283
.iter()
@@ -298,7 +298,7 @@ fn from_property_map(map: HashMap<String, String>) -> Result<Config, ConfigError
298298
.for_each(|v| k8s_rules.push(v.to_string()));
299299
}
300300

301-
if let Some(value) = map.get(&REDACT) {
301+
if let Some(value) = map.get(&REDACT_REGEX) {
302302
let regex_rules = result.log.line_redact_regex.get_or_insert(Vec::new());
303303
argv::split_by_comma(value)
304304
.iter()

‎common/http/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ futures = "0.3"
3030
futures-timer = "3"
3131
prometheus = { version = "0.13", features = ["process"] }
3232
async-compression = { version = "0.3.8", features = ["tokio"] }
33+
rate-limit-macro = { version = "1" }
3334

3435
[dev-dependencies]
3536
tempfile = "3"

‎common/http/src/limit.rs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use std::sync::Arc;
66
use metrics::Metrics;
77
use serde::{Serialize, Serializer};
88
use std::time::Duration;
9-
use tracing::info;
9+
use tracing::debug;
1010

1111
pub struct RateLimiter {
1212
pub slots: Arc<AtomicUsize>,
@@ -128,8 +128,9 @@ impl Backoff {
128128

129129
pub async fn snooze(&self) {
130130
let step = self.step.load(Ordering::SeqCst);
131-
// TODO make debug
132-
info!("hit rate limit, snoozing");
131+
rate_limit_macro::rate_limit!(rate = 1, interval = 5, {
132+
debug!("hit rate limit, snoozing");
133+
});
133134
tokio::time::sleep(Duration::from_millis(self.base.pow(step) * self.multipler)).await;
134135
self.step.fetch_add(1, Ordering::SeqCst);
135136
}

0 commit comments

Comments
 (0)