Cache statistics and provide estimation methods

modules/metrics/collector.go

@@ -227,7 +227,7 @@ func (c Collector) Describe(ch chan<- *prometheus.Desc) {
 
 // Collect returns the metrics with values
 func (c Collector) Collect(ch chan<- prometheus.Metric) {
-	stats := GetStatistic(setting.Metrics.EstimateCounts, setting.Metrics.StatisticTTL)
+	stats := <-GetStatistic(setting.Metrics.EstimateCounts, setting.Metrics.StatisticTTL, true)
 
 	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Accesses,

modules/metrics/statistics.go

@@ -10,29 +10,72 @@ import (
 	"time"
 
 	"code.gitea.io/gitea/models"
-	"code.gitea.io/gitea/modules/cache"
-	"code.gitea.io/gitea/modules/setting"
 )
 
-var statisticsLock sync.Mutex
+var (
+	statisticsLock        sync.Mutex
+	statisticsMap         = map[string]*models.Statistic{}
+	statisticsWorkingChan = map[string]chan struct{}{}
+)
+
+func GetStatistic(estimate bool, statisticsTTL time.Duration, metrics bool) <-chan *models.Statistic {
+	cacheKey := "models/statistic.Statistic." + strconv.FormatBool(estimate) + strconv.FormatBool(metrics)
 
-func GetStatistic(estimate bool, statisticsTTL time.Duration, metrics bool) models.Statistic {
+	statisticsLock.Lock() // CAREFUL: no defer!
+	ourChan := make(chan *models.Statistic, 1)
+
+	// Check for a cached statistic
 	if statisticsTTL > 0 {
-		c := cache.GetCache()
-		if c != nil {
-			statisticsLock.Lock()
-			defer statisticsLock.Unlock()
-			cacheKey := "models/statistic.Statistic." + strconv.FormatBool(estimate) + strconv.FormatBool(metrics)
+		if stats, ok := statisticsMap[cacheKey]; ok && stats.Time.Add(statisticsTTL).After(time.Now()) {
+			// Found a valid cached statistic for these params, so unlock and send this down the channel
+			statisticsLock.Unlock() // Unlock from line 24
+
+			ourChan <- stats
+			close(ourChan)
+			return ourChan
+		}
+	}
+
+	// We need to calculate a statistic - however, we should only do this one at a time (NOTE: we are still within the lock)
+	//
+	// So check if we have a worker already and get a marker channel
+	workingChan, ok := statisticsWorkingChan[cacheKey]
 
-			if stats, ok := c.Get(cacheKey).(*models.Statistic); ok {
-				return *stats
-			}
+	if !ok {
+		// we need to make our own worker... (NOTE: we are still within the lock)
 
+		// create a marker channel which will be closed when our worker is finished
+		// and assign it to the working map.
+		workingChan = make(chan struct{})
+		statisticsWorkingChan[cacheKey] = workingChan
+
+		// Create the working go-routine
+		go func() {
 			stats := models.GetStatistic(estimate, metrics)
-			c.Put(cacheKey, &stats, setting.DurationToCacheTTL(statisticsTTL))
-			return stats
-		}
+
+			// cache the result, remove this worker and inform anyone waiting we are done
+			statisticsLock.Lock() // Lock within goroutine
+			statisticsMap[cacheKey] = &stats
+			delete(statisticsWorkingChan, cacheKey)
+			close(workingChan)
+			statisticsLock.Unlock() // Unlock within goroutine
+		}()
 	}
 
-	return models.GetStatistic(estimate, metrics)
+	statisticsLock.Unlock() // Unlock from line 24
+
+	// Create our goroutine for the channel waiting for the statistics to be generated
+	go func() {
+		<-workingChan // Wait for the worker to finish
+
+		// Now lock and get the last stats completed
+		statisticsLock.Lock()
+		stats := statisticsMap[cacheKey]
+		statisticsLock.Unlock()
+
+		ourChan <- stats
+		close(ourChan)
+	}()
+
+	return ourChan
 }

options/locale/locale_en-US.ini

@@ -2397,6 +2397,8 @@ dashboard.statistic = Summary
 dashboard.operations = Maintenance Operations
 dashboard.system_status = System Status
 dashboard.statistic_info = The Gitea database holds <b>%d</b> users, <b>%d</b> organizations, <b>%d</b> public keys, <b>%d</b> repositories, <b>%d</b> watches, <b>%d</b> stars, <b>%d</b> actions, <b>%d</b> accesses, <b>%d</b> issues, <b>%d</b> comments, <b>%d</b> social accounts, <b>%d</b> follows, <b>%d</b> mirrors, <b>%d</b> releases, <b>%d</b> authentication sources, <b>%d</b> webhooks, <b>%d</b> milestones, <b>%d</b> labels, <b>%d</b> hook tasks, <b>%d</b> teams, <b>%d</b> update tasks, <b>%d</b> attachments.
+dashboard.statistic_info_last_updated = Last updated %s
+dashboard.statistic_info_in_progress = Statistics are being calculated
 dashboard.operation_name = Operation Name
 dashboard.operation_switch = Switch
 dashboard.operation_run = Run
@@ -3086,7 +3088,7 @@ settings.link = Link this package to a repository
 settings.link.description = If you link a package with a repository, the package is listed in the repository's package list.
 settings.link.select = Select Repository
 settings.link.button = Update Repository Link
-settings.link.success = Repository link was successfully updated. 
+settings.link.success = Repository link was successfully updated.
 settings.link.error = Failed to update repository link.
 settings.delete = Delete package
 settings.delete.description = Deleting a package is permanent and cannot be undone.

routers/web/admin/admin.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 	"time"
 
+	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/git"
@@ -121,12 +122,25 @@ func updateSystemStatus() {
 	sysStatus.NumGC = m.NumGC
 }
 
+func getStatistics() *models.Statistic {
+	if setting.UI.Admin.StatisticTTL > 0 {
+		select {
+		case stats := <-metrics.GetStatistic(setting.UI.Admin.EstimateCounts, setting.UI.Admin.StatisticTTL, false):
+			return stats
+		case <-time.After(1 * time.Second):
+			return nil
+		}
+	}
+
+	return <-metrics.GetStatistic(setting.UI.Admin.EstimateCounts, setting.UI.Admin.StatisticTTL, false)
+}
+
 // Dashboard show admin panel dashboard
 func Dashboard(ctx *context.Context) {
 	ctx.Data["Title"] = ctx.Tr("admin.dashboard")
 	ctx.Data["PageIsAdmin"] = true
 	ctx.Data["PageIsAdminDashboard"] = true
-	ctx.Data["Stats"] = metrics.GetStatistic(setting.UI.Admin.EstimateCounts, setting.UI.Admin.StatisticTTL)
+	ctx.Data["Stats"] = getStatistics()
 	ctx.Data["NeedUpdate"] = updatechecker.GetNeedUpdate()
 	ctx.Data["RemoteVersion"] = updatechecker.GetRemoteVersion()
 	// FIXME: update periodically
@@ -142,7 +156,7 @@ func DashboardPost(ctx *context.Context) {
 	ctx.Data["Title"] = ctx.Tr("admin.dashboard")
 	ctx.Data["PageIsAdmin"] = true
 	ctx.Data["PageIsAdminDashboard"] = true
-	ctx.Data["Stats"] = metrics.GetStatistic(setting.UI.Admin.EstimateCounts, setting.UI.Admin.StatisticTTL)
+	ctx.Data["Stats"] = getStatistics()
 	updateSystemStatus()
 	ctx.Data["SysStatus"] = sysStatus
 

templates/admin/dashboard.tmpl

@@ -12,10 +12,14 @@
 			{{.i18n.Tr "admin.dashboard.statistic"}}
 		</h4>
 		<div class="ui attached segment">
-			<p>
-				{{.i18n.Tr "admin.dashboard.statistic_info" .Stats.Counter.User .Stats.Counter.Org .Stats.Counter.PublicKey .Stats.Counter.Repo .Stats.Counter.Watch .Stats.Counter.Star .Stats.Counter.Action .Stats.Counter.Access .Stats.Counter.Issue .Stats.Counter.Comment .Stats.Counter.Oauth .Stats.Counter.Follow .Stats.Counter.Mirror .Stats.Counter.Release .Stats.Counter.AuthSource .Stats.Counter.Webhook .Stats.Counter.Milestone .Stats.Counter.Label .Stats.Counter.HookTask .Stats.Counter.Team .Stats.Counter.UpdateTask .Stats.Counter.Attachment | Str2html}}
-				<span class="text italic light grey">{{TimeSince .Stats.Time $.i18n.Lang}}</span>
-			</p>
+			{{if .Stats}}
+				<p>
+					{{.i18n.Tr "admin.dashboard.statistic_info" .Stats.Counter.User .Stats.Counter.Org .Stats.Counter.PublicKey .Stats.Counter.Repo .Stats.Counter.Watch .Stats.Counter.Star .Stats.Counter.Action .Stats.Counter.Access .Stats.Counter.Issue .Stats.Counter.Comment .Stats.Counter.Oauth .Stats.Counter.Follow .Stats.Counter.Mirror .Stats.Counter.Release .Stats.Counter.AuthSource .Stats.Counter.Webhook .Stats.Counter.Milestone .Stats.Counter.Label .Stats.Counter.HookTask .Stats.Counter.Team .Stats.Counter.UpdateTask .Stats.Counter.Attachment | Str2html}}
+					<span class="text italic light grey">{{.i18n.Tr "admin.dashboard.statistic_info_last_updated" (TimeSince .Stats.Time $.i18n.Lang) | Str2html}}</span>
+				</p>
+			{{else}}
+				<p class="text italic light grey">{{.i18n.Tr "admin.dashboard.statistic_info_in_progress"}}</p>
+			{{end}}
 		</div>
 		<h4 class="ui top attached header">
 			{{.i18n.Tr "admin.dashboard.operations"}}