Skip to content

Commit 918f87a

Browse files
rchicolirchicoli
authored
Merge pull request #75 from rchicoli/development
implement elasticsearch index name regexp
2 parents 0c10d62 + 6feaea4 commit 918f87a

File tree

5 files changed

+149
-13
lines changed

5 files changed

+149
-13
lines changed

‎README.md‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,19 @@ Before creating a docker container, a healthy instance of Elasticsearch service
8585

8686
###### elasticsearch-index ######
8787
- *index* to write log messages to
88-
- *examples*: docker, logging
88+
- *examples*: docker, logging-%F, docker-%Y.%m.%d
89+
90+
```bash
91+
# FORMAT controls the output. Interpreted sequences are:
92+
%b locale's abbreviated month name (Jan)
93+
%B locale's full month name (January)
94+
%d day of month (01)
95+
%F full date; same as %Y.%m.%d
96+
%j day of year (001..366)
97+
%m month (01..12)
98+
%y last two digits of year (00..99)
99+
%Y year (2018)
100+
```
89101

90102
###### elasticsearch-username ######
91103
- *username* to authenticate to a secure Elasticsearch cluster

‎pkg/docker/config.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,8 @@ type Grok struct {
4949

5050
func defaultLogOpt() LogOpt {
5151
return LogOpt{
52-
// TODO: update index name to docker-YYYY.MM.dd
53-
index: "docker",
52+
index: "docker",
53+
// index: "docker-%F",
5454
tzpe: "log",
5555
timeout: 1,
5656
fields: "containerID,containerName,containerImageName,containerCreated",

‎pkg/docker/driver.go‎

Lines changed: 24 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ import (
1212
"syscall"
1313
"time"
1414

15+
"github.com/robfig/cron"
16+
1517
"golang.org/x/sync/errgroup"
1618

1719
"github.com/Sirupsen/logrus"
@@ -44,10 +46,12 @@ type pipeline struct {
4446
}
4547

4648
type container struct {
47-
stream io.ReadCloser
48-
info logger.Info
49-
esClient elasticsearch.Client
50-
pipeline pipeline
49+
stream io.ReadCloser
50+
info logger.Info
51+
esClient elasticsearch.Client
52+
pipeline pipeline
53+
cron *cron.Cron
54+
indexName string
5155
}
5256

5357
// LogMessage ...
@@ -166,7 +170,15 @@ func (d *Driver) StartLogging(file string, info logger.Info) error {
166170
c.pipeline.group, c.pipeline.ctx = errgroup.WithContext(ctx)
167171
c.pipeline.inputCh = make(chan logdriver.LogEntry)
168172
c.pipeline.outputCh = make(chan LogMessage)
169-
// c.pipeline.stopCh = make(chan struct{})
173+
174+
c.cron = cron.New()
175+
c.indexName = indexRegex(time.Now(), config.index)
176+
c.cron.AddFunc("@daily", func() {
177+
d.mu.Lock()
178+
c.indexName = indexRegex(time.Now(), config.index)
179+
d.mu.Unlock()
180+
})
181+
c.cron.Start()
170182

171183
c.pipeline.group.Go(func() error {
172184

@@ -200,9 +212,7 @@ func (d *Driver) StartLogging(file string, info logger.Info) error {
200212

201213
// logrus.WithField("containerID", c.info.ContainerID).WithField("line", string(buf.Line)).Debugf("pipe1")
202214

203-
// I guess this problem has been fixed with the break function above
204-
// test it again
205-
// BUG: (17.09.0~ce-0~debian) docker run command throws lots empty line messages
215+
// in case docker run command throws lots empty line messages
206216
if len(bytes.TrimSpace(buf.Line)) == 0 {
207217
logrus.WithField("containerID", c.info.ContainerID).WithField("line", string(buf.Line)).Debugf("trim")
208218
continue
@@ -289,7 +299,8 @@ func (d *Driver) StartLogging(file string, info logger.Info) error {
289299
for doc := range c.pipeline.outputCh {
290300
// logrus.WithField("containerID", c.info.ContainerID).WithField("line", string(doc.Line)).WithField("grok", doc.GrokLine).Debugf("pipe3")
291301

292-
c.esClient.Add(config.index, config.tzpe, doc)
302+
c.esClient.Add(c.indexName, config.tzpe, doc)
303+
293304
select {
294305
case <-c.pipeline.ctx.Done():
295306
logrus.WithField("containerID", c.info.ContainerID).WithError(c.pipeline.ctx.Err()).Error("context closing pipe 3")
@@ -363,6 +374,10 @@ func (d *Driver) StopLogging(file string) error {
363374
}
364375
}
365376

377+
if c.cron != nil {
378+
c.cron.Stop()
379+
}
380+
366381
// if c.esClient != nil {
367382
// close client connection on last pipeline
368383
// }

‎pkg/docker/index.go‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package docker
2+
3+
import (
4+
"fmt"
5+
"regexp"
6+
"strings"
7+
"time"
8+
9+
"github.com/Sirupsen/logrus"
10+
)
11+
12+
// %. matches each strftime format sequence and ReplaceAllStringFunc
13+
// looks up each format sequence in the conversion table strftimeToRegex
14+
// to replace with a defined regular expression
15+
var percent = regexp.MustCompile("%.")
16+
17+
func indexRegex(now time.Time, indexRegex string) string {
18+
19+
// %b locale's abbreviated month name (Jan)
20+
// %B locale's full month name (January)
21+
// %d day of month (01)
22+
// %F full date; same as %Y.%m.%d
23+
// %j day of year (001..366)
24+
// %m month (01..12)
25+
// %y last two digits of year (00..99)
26+
// %Y year (2018)
27+
var strftimeToRegex = map[string]string{
28+
/*dayZeroPadded */ `%d`: fmt.Sprintf("%02d", now.Day()),
29+
/*monthShort */ `%b`: now.Month().String()[:3],
30+
/*monthFull */ `%B`: now.Month().String(),
31+
/*monthFull */ `%F`: fmt.Sprintf("%d.%02d.%02d", now.Year(), int(now.Month()), now.Day()),
32+
/*monthZeroPadded */ `%m`: fmt.Sprintf("%02d", int(now.Month())),
33+
/*yearCentury */ `%Y`: fmt.Sprintf("%d", now.Year()),
34+
/*yearZeroPadded */ `%y`: fmt.Sprintf("%d", now.Year())[2:],
35+
/*dayOfYearZeroPadded */ `%j`: fmt.Sprintf("%d", now.YearDay()),
36+
// /*testSecond */ `%z`: fmt.Sprintf("%02d", now.Second()),
37+
}
38+
39+
var indexName = percent.ReplaceAllStringFunc(indexRegex, func(s string) string {
40+
// org.elasticsearch.indices.InvalidIndexNameException: ... must be lowercase
41+
return strings.ToLower(strftimeToRegex[s])
42+
})
43+
logrus.WithField("indexname", indexName).Debug("index name generated from regex")
44+
45+
return indexName
46+
47+
}

‎tests/acceptance-tests/03-index-and-type.sh‎

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,68 @@ function test_different_index_and_type(){
4444
basht_assert "echo '${output}' | jq -r '.hits.hits[0]._source.message'" equals "$message"
4545
basht_assert "echo '${output}' | jq -r '.hits.hits[0]._source.partial'" equals "true"
4646

47-
47+
}
48+
49+
function test_index_regex_year_month_day(){
50+
51+
description="[${BASHT_TEST_FILENAME##*/}] acceptance-tests (v${CLIENT_VERSION}): $BASHT_TEST_NUMBER - index regex %F are converted to the current date"
52+
echo "$description"
53+
54+
basht_run make -f "$MAKEFILE" deploy_elasticsearch
55+
56+
export ELASTICSEARCH_INDEX="docker-%F"
57+
58+
name="${BASHT_TEST_FILENAME##*/}.${BASHT_TEST_NUMBER}"
59+
message="$((RANDOM)) $description"
60+
61+
basht_run docker run --rm -ti \
62+
--log-driver rchicoli/docker-log-elasticsearch:development \
63+
--log-opt elasticsearch-url="${ELASTICSEARCH_URL}" \
64+
--log-opt elasticsearch-version="${CLIENT_VERSION}" \
65+
--name "$name" \
66+
--log-opt elasticsearch-index="${ELASTICSEARCH_INDEX}" \
67+
alpine echo -n "$message"
68+
69+
sleep "${SLEEP_TIME}"
70+
71+
index_name="$(echo "docker-$(date +%Y.%m.%d)")"
72+
73+
basht_run curl -s -G --connect-timeout 5 \
74+
"${ELASTICSEARCH_URL}/${index_name}/_search?pretty=true&size=1" \
75+
--data-urlencode "q=message:\"$message\""
76+
77+
basht_assert "echo '${output}' | jq -r '.hits.hits[0]._source.message'" equals "$message"
78+
79+
}
80+
81+
function test_index_regex_year_month_day(){
82+
83+
description="[${BASHT_TEST_FILENAME##*/}] acceptance-tests (v${CLIENT_VERSION}): $BASHT_TEST_NUMBER - all index regex together are converted to the current date"
84+
echo "$description"
85+
86+
basht_run make -f "$MAKEFILE" deploy_elasticsearch
87+
88+
export ELASTICSEARCH_INDEX="test-%b-%B-%d-%j-%m-%y-%y"
89+
90+
name="${BASHT_TEST_FILENAME##*/}.${BASHT_TEST_NUMBER}"
91+
message="$((RANDOM)) $description"
92+
93+
basht_run docker run --rm -ti \
94+
--log-driver rchicoli/docker-log-elasticsearch:development \
95+
--log-opt elasticsearch-url="${ELASTICSEARCH_URL}" \
96+
--log-opt elasticsearch-version="${CLIENT_VERSION}" \
97+
--name "$name" \
98+
--log-opt elasticsearch-index="${ELASTICSEARCH_INDEX}" \
99+
alpine echo -n "$message"
100+
101+
sleep "${SLEEP_TIME}"
102+
103+
index_name="$(echo "test-$(date +%b-%B-%d-%j-%m-%y-%y)")"
104+
105+
basht_run curl -s -G --connect-timeout 5 \
106+
"${ELASTICSEARCH_URL}/${index_name,,}/_search?pretty=true&size=1" \
107+
--data-urlencode "q=message:\"$message\""
108+
109+
basht_assert "echo '${output}' | jq -r '.hits.hits[0]._source.message'" equals "$message"
48110

49111
}

0 commit comments

Comments
 (0)