Use a separate admin page to show global stats, remove actions stat (#25062)

Before, Gitea shows the database table stats on the `admin dashboard`
page.

It has some problems:

* `count(*)` is quite heavy. If tables have many records, this blocks
loading the admin page blocks for a long time
* Some users had even reported issues that they can't visit their admin
page because this page causes blocking or `50x error (reverse proxy
timeout)`
* The `actions` stat is not useful. The table is simply too large. Does
it really matter if it contains 1,000,000 rows or 9,999,999 rows?
* The translation `admin.dashboard.statistic_info` is difficult to
maintain.

So, this PR uses a separate page to show the stats and removes the
`actions` stat.


![image](https://github.com/go-gitea/gitea/assets/2114189/babf7c61-b93b-4a62-bfaa-22983636427e)

## ⚠️ BREAKING

The `actions` Prometheus metrics collector has been removed for the
reasons mentioned beforehand.
Please do not rely on its output anymore.

models/activities/statistic.go

@@ -21,7 +21,7 @@ import (
 type Statistic struct {
 	Counter struct {
 		User, Org, PublicKey,
-		Repo, Watch, Star, Action, Access,
+		Repo, Watch, Star, Access,
 		Issue, IssueClosed, IssueOpen,
 		Comment, Oauth, Follow,
 		Mirror, Release, AuthSource, Webhook,
@@ -55,7 +55,6 @@ func GetStatistic() (stats Statistic) {
 	stats.Counter.Repo, _ = repo_model.CountRepositories(db.DefaultContext, repo_model.CountRepositoryOptions{})
 	stats.Counter.Watch, _ = e.Count(new(repo_model.Watch))
 	stats.Counter.Star, _ = e.Count(new(repo_model.Star))
-	stats.Counter.Action, _ = db.EstimateCount(db.DefaultContext, new(Action))
 	stats.Counter.Access, _ = e.Count(new(access_model.Access))
 
 	type IssueCount struct {
@@ -83,7 +82,7 @@ func GetStatistic() (stats Statistic) {
 			Find(&stats.Counter.IssueByRepository)
 	}
 
-	issueCounts := []IssueCount{}
+	var issueCounts []IssueCount
 
 	_ = e.Select("COUNT(*) AS count, is_closed").Table("issue").GroupBy("is_closed").Find(&issueCounts)
 	for _, c := range issueCounts {

models/db/context.go

@@ -9,7 +9,6 @@ import (
 
 	"xorm.io/builder"
 	"xorm.io/xorm"
-	"xorm.io/xorm/schemas"
 )
 
 // DefaultContext is the default context to run xorm queries in
@@ -241,30 +240,6 @@ func TableName(bean interface{}) string {
 	return x.TableName(bean)
 }
 
-// EstimateCount returns an estimate of total number of rows in table
-func EstimateCount(ctx context.Context, bean interface{}) (int64, error) {
-	e := GetEngine(ctx)
-	e.Context(ctx)
-
-	var rows int64
-	var err error
-	tablename := TableName(bean)
-	switch x.Dialect().URI().DBType {
-	case schemas.MYSQL:
-		_, err = e.Context(ctx).SQL("SELECT table_rows FROM information_schema.tables WHERE tables.table_name = ? AND tables.table_schema = ?;", tablename, x.Dialect().URI().DBName).Get(&rows)
-	case schemas.POSTGRES:
-		// the table can live in multiple schemas of a postgres database
-		// See https://wiki.postgresql.org/wiki/Count_estimate
-		tablename = x.TableName(bean, true)
-		_, err = e.Context(ctx).SQL("SELECT reltuples::bigint AS estimate FROM pg_class WHERE oid = ?::regclass;", tablename).Get(&rows)
-	case schemas.MSSQL:
-		_, err = e.Context(ctx).SQL("sp_spaceused ?;", tablename).Get(&rows)
-	default:
-		return e.Context(ctx).Count(tablename)
-	}
-	return rows, err
-}
-
 // InTransaction returns true if the engine is in a transaction otherwise return false
 func InTransaction(ctx context.Context) bool {
 	_, ok := inTransaction(ctx)

modules/metrics/collector.go

@@ -18,7 +18,6 @@ const namespace = "gitea_"
 // exposes gitea metrics for prometheus
 type Collector struct {
 	Accesses           *prometheus.Desc
-	Actions            *prometheus.Desc
 	Attachments        *prometheus.Desc
 	BuildInfo          *prometheus.Desc
 	Comments           *prometheus.Desc
@@ -56,11 +55,6 @@ func NewCollector() Collector {
 			"Number of Accesses",
 			nil, nil,
 		),
-		Actions: prometheus.NewDesc(
-			namespace+"actions",
-			"Number of Actions",
-			nil, nil,
-		),
 		Attachments: prometheus.NewDesc(
 			namespace+"attachments",
 			"Number of Attachments",
@@ -207,7 +201,6 @@ func NewCollector() Collector {
 // Describe returns all possible prometheus.Desc
 func (c Collector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- c.Accesses
-	ch <- c.Actions
 	ch <- c.Attachments
 	ch <- c.BuildInfo
 	ch <- c.Comments
@@ -246,11 +239,6 @@ func (c Collector) Collect(ch chan<- prometheus.Metric) {
 		prometheus.GaugeValue,
 		float64(stats.Counter.Access),
 	)
-	ch <- prometheus.MustNewConstMetric(
-		c.Actions,
-		prometheus.GaugeValue,
-		float64(stats.Counter.Action),
-	)
 	ch <- prometheus.MustNewConstMetric(
 		c.Attachments,
 		prometheus.GaugeValue,

options/locale/locale_en-US.ini

@@ -2619,7 +2619,6 @@ dashboard.new_version_hint = Gitea %s is now available, you are running %s. Chec
 dashboard.statistic = Summary
 dashboard.operations = Maintenance Operations
 dashboard.system_status = System Status
-dashboard.statistic_info = The Gitea database holds <b>%d</b> users, <b>%d</b> organizations, <b>%d</b> public keys, <b>%d</b> repositories, <b>%d</b> watches, <b>%d</b> stars, ~<b>%d</b> actions, <b>%d</b> accesses, <b>%d</b> issues, <b>%d</b> comments, <b>%d</b> social accounts, <b>%d</b> follows, <b>%d</b> mirrors, <b>%d</b> releases, <b>%d</b> authentication sources, <b>%d</b> webhooks, <b>%d</b> milestones, <b>%d</b> labels, <b>%d</b> hook tasks, <b>%d</b> teams, <b>%d</b> update tasks, <b>%d</b> attachments.
 dashboard.operation_name = Operation Name
 dashboard.operation_switch = Switch
 dashboard.operation_run = Run
@@ -3060,6 +3059,8 @@ config.xorm_log_sql = Log SQL
 config.get_setting_failed = Get setting %s failed
 config.set_setting_failed = Set setting %s failed
 
+monitor.stats = Stats
+
 monitor.cron = Cron Tasks
 monitor.name = Name
 monitor.schedule = Schedule

routers/web/admin/admin.go

@@ -8,11 +8,13 @@ import (
 	"fmt"
 	"net/http"
 	"runtime"
+	"sort"
 	"time"
 
 	activities_model "code.gitea.io/gitea/models/activities"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/updatechecker"
 	"code.gitea.io/gitea/modules/web"
@@ -26,6 +28,7 @@ const (
 	tplQueue       base.TplName = "admin/queue"
 	tplStacktrace  base.TplName = "admin/stacktrace"
 	tplQueueManage base.TplName = "admin/queue_manage"
+	tplStats       base.TplName = "admin/stats"
 )
 
 var sysStatus struct {
@@ -111,7 +114,6 @@ func updateSystemStatus() {
 func Dashboard(ctx *context.Context) {
 	ctx.Data["Title"] = ctx.Tr("admin.dashboard")
 	ctx.Data["PageIsAdminDashboard"] = true
-	ctx.Data["Stats"] = activities_model.GetStatistic()
 	ctx.Data["NeedUpdate"] = updatechecker.GetNeedUpdate()
 	ctx.Data["RemoteVersion"] = updatechecker.GetRemoteVersion()
 	// FIXME: update periodically
@@ -126,7 +128,6 @@ func DashboardPost(ctx *context.Context) {
 	form := web.GetForm(ctx).(*forms.AdminDashboardForm)
 	ctx.Data["Title"] = ctx.Tr("admin.dashboard")
 	ctx.Data["PageIsAdminDashboard"] = true
-	ctx.Data["Stats"] = activities_model.GetStatistic()
 	updateSystemStatus()
 	ctx.Data["SysStatus"] = sysStatus
 
@@ -153,3 +154,30 @@ func CronTasks(ctx *context.Context) {
 	ctx.Data["Entries"] = cron.ListTasks()
 	ctx.HTML(http.StatusOK, tplCron)
 }
+
+func MonitorStats(ctx *context.Context) {
+	ctx.Data["Title"] = ctx.Tr("admin.monitor.stats")
+	ctx.Data["PageIsAdminMonitorStats"] = true
+	bs, err := json.Marshal(activities_model.GetStatistic().Counter)
+	if err != nil {
+		ctx.ServerError("MonitorStats", err)
+		return
+	}
+	statsCounter := map[string]any{}
+	err = json.Unmarshal(bs, &statsCounter)
+	if err != nil {
+		ctx.ServerError("MonitorStats", err)
+		return
+	}
+	statsKeys := make([]string, 0, len(statsCounter))
+	for k := range statsCounter {
+		if statsCounter[k] == nil {
+			continue
+		}
+		statsKeys = append(statsKeys, k)
+	}
+	sort.Strings(statsKeys)
+	ctx.Data["StatsKeys"] = statsKeys
+	ctx.Data["StatsCounter"] = statsCounter
+	ctx.HTML(http.StatusOK, tplStats)
+}

routers/web/web.go

@@ -538,8 +538,8 @@ func registerRoutes(m *web.Route) {
 
 	// ***** START: Admin *****
 	m.Group("/admin", func() {
-		m.Get("", adminReq, admin.Dashboard)
-		m.Post("", adminReq, web.Bind(forms.AdminDashboardForm{}), admin.DashboardPost)
+		m.Get("", admin.Dashboard)
+		m.Post("", web.Bind(forms.AdminDashboardForm{}), admin.DashboardPost)
 
 		m.Group("/config", func() {
 			m.Get("", admin.Config)
@@ -548,6 +548,7 @@ func registerRoutes(m *web.Route) {
 		})
 
 		m.Group("/monitor", func() {
+			m.Get("/stats", admin.MonitorStats)
 			m.Get("/cron", admin.CronTasks)
 			m.Get("/stacktrace", admin.Stacktrace)
 			m.Post("/stacktrace/cancel/{pid}", admin.StacktraceCancel)

templates/admin/dashboard.tmpl

@@ -5,14 +5,6 @@
 				<p>{{(.locale.Tr "admin.dashboard.new_version_hint" .RemoteVersion AppVer) | Str2html}}</p>
 			</div>
 		{{end}}
-		<h4 class="ui top attached header">
-			{{.locale.Tr "admin.dashboard.statistic"}}
-		</h4>
-		<div class="ui attached segment">
-			<p>
-				{{.locale.Tr "admin.dashboard.statistic_info" .Stats.Counter.User .Stats.Counter.Org .Stats.Counter.PublicKey .Stats.Counter.Repo .Stats.Counter.Watch .Stats.Counter.Star .Stats.Counter.Action .Stats.Counter.Access .Stats.Counter.Issue .Stats.Counter.Comment .Stats.Counter.Oauth .Stats.Counter.Follow .Stats.Counter.Mirror .Stats.Counter.Release .Stats.Counter.AuthSource .Stats.Counter.Webhook .Stats.Counter.Milestone .Stats.Counter.Label .Stats.Counter.HookTask .Stats.Counter.Team .Stats.Counter.UpdateTask .Stats.Counter.Attachment | Str2html}}
-			</p>
-		</div>
 		<h4 class="ui top attached header">
 			{{.locale.Tr "admin.dashboard.operations"}}
 		</h4>

templates/admin/navbar.tmpl

@@ -53,6 +53,9 @@
 		<div class="item">
 			{{.locale.Tr "admin.monitor"}}
 			<div class="menu">
+				<a class="{{if .PageIsAdminMonitorStats}}active {{end}}item" href="{{AppSubUrl}}/admin/monitor/stats">
+					{{.locale.Tr "admin.monitor.stats"}}
+				</a>
 				<a class="{{if .PageIsAdminMonitorCron}}active {{end}}item" href="{{AppSubUrl}}/admin/monitor/cron">
 					{{.locale.Tr "admin.monitor.cron"}}
 				</a>

templates/admin/stats.tmpl

@@ -0,0 +1,17 @@
+{{template "admin/layout_head" (dict "ctxData" . "pageClass" "admin monitor")}}
+<div class="admin-setting-content">
+	<h4 class="ui top attached header">
+		{{.locale.Tr "admin.dashboard.statistic"}}
+	</h4>
+	<div class="ui attached table segment">
+		<table class="ui very basic striped table unstackable">
+			{{range $statsKey := .StatsKeys}}
+			<tr>
+				<td width="200">{{$statsKey}}</td>
+				<td>{{index $.StatsCounter $statsKey}}</td>
+			</tr>
+			{{end}}
+		</table>
+	</div>
+</div>
+{{template "admin/layout_footer" .}}