Skip to content

Commit 9cf404c

Browse files
elena-kolevskaelena-kolevska
committed
wip
Signed-off-by: Elena Kolevska <elena@kolevska.com>
1 parent c17657c commit 9cf404c

File tree

12 files changed

+1061
-13
lines changed

12 files changed

+1061
-13
lines changed

‎example/otel_metrics.go‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ package main
55
import (
66
"context"
77
"log"
8+
"math/rand"
89
"strconv"
10+
"sync"
911
"time"
1012

1113
redisotel "github.com/redis/go-redis/extra/redisotel-native/v9"
@@ -73,16 +75,47 @@ func main() {
7375
// STEP_START redis_operations
7476
// Execute Redis operations - metrics are automatically collected
7577
log.Println("Executing Redis operations...")
76-
for i := range 100 {
77-
if err := rdb.Set(ctx, "key"+strconv.Itoa(i), "value", 0).Err(); err != nil {
78+
var wg sync.WaitGroup
79+
wg.Add(50)
80+
for i := range 50 {
81+
go func(i int) {
82+
defer wg.Done()
83+
84+
for j := range 10 {
85+
if err := rdb.Set(ctx, "key"+strconv.Itoa(i*10+j), "value", 0).Err(); err != nil {
86+
log.Printf("Error setting key: %v", err)
87+
}
88+
time.Sleep(time.Millisecond * time.Duration(rand.Intn(400)))
89+
}
90+
}(i)
91+
}
92+
wg.Wait()
93+
94+
wg.Add(10)
95+
for i := range 10 {
96+
go func(i int) {
97+
defer wg.Done()
98+
99+
for j := range 10 {
100+
if err := rdb.Set(ctx, "key"+strconv.Itoa(i*10+j), "value", 0).Err(); err != nil {
101+
log.Printf("Error setting key: %v", err)
102+
}
103+
time.Sleep(time.Millisecond * time.Duration(rand.Intn(400)))
104+
}
105+
}(i)
106+
}
107+
wg.Wait()
108+
109+
for j := range 10 {
110+
if err := rdb.Set(ctx, "key"+strconv.Itoa(j), "value", 0).Err(); err != nil {
78111
log.Printf("Error setting key: %v", err)
79112
}
80-
time.Sleep(time.Millisecond * 100)
113+
time.Sleep(time.Millisecond * time.Duration(rand.Intn(400)))
81114
}
115+
82116
log.Println("Operations complete. Waiting for metrics to be exported...")
83117

84118
// Wait for metrics to be exported
85119
time.Sleep(15 * time.Second)
86120
// STEP_END
87121
}
88-

‎extra/redisotel-native/GRAFANA_QUERIES.md‎

Lines changed: 296 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,296 @@
1+
# Grafana Queries for go-redis OTel Metrics
2+
3+
This document provides ready-to-use Grafana queries for visualizing go-redis OpenTelemetry metrics.
4+
5+
## Connection Creation Time Metrics
6+
7+
### Metric Name
8+
`db.client.connection.create_time`
9+
10+
### Available Attributes
11+
- `db.system` - Always "redis"
12+
- `server.address` - Redis server address
13+
- `server.port` - Redis server port (if not default 6379)
14+
- `redis.client.library` - Client library name and version (e.g., "go-redis:9.16.0-beta.1")
15+
- `db.client.connection.pool.name` - Unique pool identifier
16+
17+
---
18+
19+
## Query Examples
20+
21+
### 1. Average Connection Creation Time
22+
23+
**Use Case**: Monitor average time to create connections over time
24+
25+
**PromQL**:
26+
```promql
27+
rate(db_client_connection_create_time_sum{db_system="redis"}[5m])
28+
/
29+
rate(db_client_connection_create_time_count{db_system="redis"}[5m])
30+
```
31+
32+
**Panel Settings**:
33+
- Type: Time series
34+
- Unit: seconds (s)
35+
- Legend: `{{server_address}} - {{db_client_connection_pool_name}}`
36+
37+
---
38+
39+
### 2. Connection Creation Time Percentiles
40+
41+
**Use Case**: Understand latency distribution (P50, P95, P99)
42+
43+
**PromQL (P50)**:
44+
```promql
45+
histogram_quantile(0.50,
46+
rate(db_client_connection_create_time_bucket{db_system="redis"}[5m])
47+
)
48+
```
49+
50+
**PromQL (P95)**:
51+
```promql
52+
histogram_quantile(0.95,
53+
rate(db_client_connection_create_time_bucket{db_system="redis"}[5m])
54+
)
55+
```
56+
57+
**PromQL (P99)**:
58+
```promql
59+
histogram_quantile(0.99,
60+
rate(db_client_connection_create_time_bucket{db_system="redis"}[5m])
61+
)
62+
```
63+
64+
**Panel Settings**:
65+
- Type: Time series
66+
- Unit: seconds (s)
67+
- Multiple queries in one panel
68+
- Color coding: P50 (green), P95 (yellow), P99 (red)
69+
70+
---
71+
72+
### 3. Connection Creation Rate
73+
74+
**Use Case**: Monitor how frequently new connections are being created
75+
76+
**PromQL**:
77+
```promql
78+
rate(db_client_connection_create_time_count{db_system="redis"}[5m])
79+
```
80+
81+
**Panel Settings**:
82+
- Type: Time series
83+
- Unit: connections per second (cps)
84+
- Legend: `{{server_address}}`
85+
86+
---
87+
88+
### 4. Total Connections Created
89+
90+
**Use Case**: See total number of connections created since start
91+
92+
**PromQL**:
93+
```promql
94+
sum(db_client_connection_create_time_count{db_system="redis"})
95+
```
96+
97+
**Panel Settings**:
98+
- Type: Stat
99+
- Unit: short
100+
- Graph mode: Area
101+
- Thresholds: 0 (green), 1000 (yellow), 10000 (red)
102+
103+
---
104+
105+
### 5. Connection Creation Time Heatmap
106+
107+
**Use Case**: Visualize distribution of connection creation times
108+
109+
**PromQL**:
110+
```promql
111+
sum(rate(db_client_connection_create_time_bucket{db_system="redis"}[5m])) by (le)
112+
```
113+
114+
**Panel Settings**:
115+
- Type: Heatmap
116+
- Format: Heatmap
117+
- Color scheme: Spectral
118+
119+
---
120+
121+
### 6. Slow Connection Creations Alert
122+
123+
**Use Case**: Identify pools with slow connection creation (P99 > 100ms)
124+
125+
**PromQL**:
126+
```promql
127+
histogram_quantile(0.99,
128+
rate(db_client_connection_create_time_bucket{db_system="redis"}[5m])
129+
) > 0.1
130+
```
131+
132+
**Panel Settings**:
133+
- Type: Table
134+
- Format: Table
135+
- Instant query: true
136+
- Columns: server_address, db_client_connection_pool_name, redis_client_library, Value
137+
138+
---
139+
140+
### 7. Connection Creation Time by Pool
141+
142+
**Use Case**: Compare connection creation time across different pools
143+
144+
**PromQL**:
145+
```promql
146+
avg by (db_client_connection_pool_name) (
147+
rate(db_client_connection_create_time_sum{db_system="redis"}[5m])
148+
/
149+
rate(db_client_connection_create_time_count{db_system="redis"}[5m])
150+
)
151+
```
152+
153+
**Panel Settings**:
154+
- Type: Bar chart or Time series
155+
- Unit: seconds (s)
156+
- Legend: `{{db_client_connection_pool_name}}`
157+
158+
---
159+
160+
## Filtering Examples
161+
162+
### Filter by Server Address
163+
```promql
164+
rate(db_client_connection_create_time_sum{
165+
db_system="redis",
166+
server_address="localhost"
167+
}[5m])
168+
```
169+
170+
### Filter by Pool Name
171+
```promql
172+
rate(db_client_connection_create_time_sum{
173+
db_system="redis",
174+
db_client_connection_pool_name="localhost:6379"
175+
}[5m])
176+
```
177+
178+
### Filter by Client Library Version
179+
```promql
180+
rate(db_client_connection_create_time_sum{
181+
db_system="redis",
182+
redis_client_library=~"go-redis:9.*"
183+
}[5m])
184+
```
185+
186+
---
187+
188+
## Alert Rules
189+
190+
### Alert: High Connection Creation Latency
191+
192+
**Condition**: P99 connection creation time > 500ms for 5 minutes
193+
194+
**PromQL**:
195+
```promql
196+
histogram_quantile(0.99,
197+
rate(db_client_connection_create_time_bucket{db_system="redis"}[5m])
198+
) > 0.5
199+
```
200+
201+
**Alert Rule**:
202+
```yaml
203+
- alert: HighRedisConnectionCreationLatency
204+
expr: |
205+
histogram_quantile(0.99,
206+
rate(db_client_connection_create_time_bucket{db_system="redis"}[5m])
207+
) > 0.5
208+
for: 5m
209+
labels:
210+
severity: warning
211+
annotations:
212+
summary: "High Redis connection creation latency"
213+
description: "P99 connection creation time is {{ $value }}s for pool {{ $labels.db_client_connection_pool_name }}"
214+
```
215+
216+
---
217+
218+
### Alert: High Connection Creation Rate
219+
220+
**Condition**: Creating more than 10 connections/second for 5 minutes
221+
222+
**PromQL**:
223+
```promql
224+
rate(db_client_connection_create_time_count{db_system="redis"}[5m]) > 10
225+
```
226+
227+
**Alert Rule**:
228+
```yaml
229+
- alert: HighRedisConnectionCreationRate
230+
expr: |
231+
rate(db_client_connection_create_time_count{db_system="redis"}[5m]) > 10
232+
for: 5m
233+
labels:
234+
severity: warning
235+
annotations:
236+
summary: "High Redis connection creation rate"
237+
description: "Creating {{ $value }} connections/sec for pool {{ $labels.db_client_connection_pool_name }}"
238+
```
239+
240+
---
241+
242+
## Dashboard Import
243+
244+
To import the complete dashboard:
245+
246+
1. Copy the contents of `grafana-dashboard-connection-create-time.json`
247+
2. In Grafana, go to **Dashboards** → **Import**
248+
3. Paste the JSON
249+
4. Select your Prometheus data source
250+
5. Click **Import**
251+
252+
---
253+
254+
## Troubleshooting
255+
256+
### No data showing up?
257+
258+
1. **Check if metrics are enabled**:
259+
```go
260+
redisotel.Init(rdb)
261+
```
262+
263+
2. **Verify Prometheus is scraping**:
264+
```promql
265+
db_client_connection_create_time_count
266+
```
267+
268+
3. **Check time range**: Metrics only appear after connections are created
269+
270+
4. **Verify labels**: Use Prometheus query browser to see available labels:
271+
```promql
272+
db_client_connection_create_time_count{db_system="redis"}
273+
```
274+
275+
### Histogram buckets not showing?
276+
277+
Make sure your Prometheus is configured to scrape histogram buckets:
278+
```yaml
279+
# prometheus.yml
280+
scrape_configs:
281+
- job_name: 'my-app'
282+
static_configs:
283+
- targets: ['localhost:8080']
284+
```
285+
286+
---
287+
288+
## Next Steps
289+
290+
- See `connection_create_time_test.go` for testing examples
291+
- Check `README.md` for full instrumentation guide
292+
- Explore other connection metrics (coming soon):
293+
- `db.client.connection.count`
294+
- `db.client.connection.wait_time`
295+
- `db.client.connection.use_time`
296+

0 commit comments

Comments
 (0)