forgejo/routers/web/admin/admin.go

189 lines
6.3 KiB
Go
Raw Normal View History

2014-03-20 15:50:26 +04:00
// Copyright 2014 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
2014-03-20 15:50:26 +04:00
package admin
import (
2014-03-22 15:42:24 +04:00
"fmt"
"net/http"
2014-03-22 15:42:24 +04:00
"runtime"
"time"
2014-03-21 11:27:59 +04:00
activities_model "code.gitea.io/gitea/models/activities"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/process"
Graceful Queues: Issue Indexing and Tasks (#9363) * Queue: Add generic graceful queues with settings * Queue & Setting: Add worker pool implementation * Queue: Add worker settings * Queue: Make resizing worker pools * Queue: Add name variable to queues * Queue: Add monitoring * Queue: Improve logging * Issues: Gracefulise the issues indexer Remove the old now unused specific queues * Task: Move to generic queue and gracefulise * Issues: Standardise the issues indexer queue settings * Fix test * Queue: Allow Redis to connect to unix * Prevent deadlock during early shutdown of issue indexer * Add MaxWorker settings to queues * Merge branch 'master' into graceful-queues * Update modules/indexer/issues/indexer.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/indexer/issues/indexer.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/queue/queue_channel.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/queue/queue_disk.go * Update modules/queue/queue_disk_channel.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Rename queue.Description to queue.ManagedQueue as per @guillep2k * Cancel pool workers when removed * Remove dependency on queue from setting * Update modules/queue/queue_redis.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * As per @guillep2k add mutex locks on shutdown/terminate * move unlocking out of setInternal * Add warning if number of workers < 0 * Small changes as per @guillep2k * No redis host specified not found * Clean up documentation for queues * Update docs/content/doc/advanced/config-cheat-sheet.en-us.md * Update modules/indexer/issues/indexer_test.go * Ensure that persistable channel queue is added to manager * Rename QUEUE_NAME REDIS_QUEUE_NAME * Revert "Rename QUEUE_NAME REDIS_QUEUE_NAME" This reverts commit 1f83b4fc9b9dabda186257b38c265fe7012f90df. Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: techknowlogick <matti@mdranta.net> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2020-01-07 14:23:09 +03:00
"code.gitea.io/gitea/modules/queue"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/updatechecker"
Move macaron to chi (#14293) Use [chi](https://github.com/go-chi/chi) instead of the forked [macaron](https://gitea.com/macaron/macaron). Since macaron and chi have conflicts with session share, this big PR becomes a have-to thing. According my previous idea, we can replace macaron step by step but I'm wrong. :( Below is a list of big changes on this PR. - [x] Define `context.ResponseWriter` interface with an implementation `context.Response`. - [x] Use chi instead of macaron, and also a customize `Route` to wrap chi so that the router usage is similar as before. - [x] Create different routers for `web`, `api`, `internal` and `install` so that the codes will be more clear and no magic . - [x] Use https://github.com/unrolled/render instead of macaron's internal render - [x] Use https://github.com/NYTimes/gziphandler instead of https://gitea.com/macaron/gzip - [x] Use https://gitea.com/go-chi/session which is a modified version of https://gitea.com/macaron/session and removed `nodb` support since it will not be maintained. **BREAK** - [x] Use https://gitea.com/go-chi/captcha which is a modified version of https://gitea.com/macaron/captcha - [x] Use https://gitea.com/go-chi/cache which is a modified version of https://gitea.com/macaron/cache - [x] Use https://gitea.com/go-chi/binding which is a modified version of https://gitea.com/macaron/binding - [x] Use https://github.com/go-chi/cors instead of https://gitea.com/macaron/cors - [x] Dropped https://gitea.com/macaron/i18n and make a new one in `code.gitea.io/gitea/modules/translation` - [x] Move validation form structs from `code.gitea.io/gitea/modules/auth` to `code.gitea.io/gitea/modules/forms` to avoid dependency cycle. - [x] Removed macaron log service because it's not need any more. **BREAK** - [x] All form structs have to be get by `web.GetForm(ctx)` in the route function but not as a function parameter on routes definition. - [x] Move Git HTTP protocol implementation to use routers directly. - [x] Fix the problem that chi routes don't support trailing slash but macaron did. - [x] `/api/v1/swagger` now will be redirect to `/api/swagger` but not render directly so that `APIContext` will not create a html render. Notices: - Chi router don't support request with trailing slash - Integration test `TestUserHeatmap` maybe mysql version related. It's failed on my macOS(mysql 5.7.29 installed via brew) but succeed on CI. Co-authored-by: 6543 <6543@obermui.de>
2021-01-26 18:36:53 +03:00
"code.gitea.io/gitea/modules/web"
"code.gitea.io/gitea/services/cron"
"code.gitea.io/gitea/services/forms"
2014-03-20 15:50:26 +04:00
)
const (
Rewrite queue (#24505) # ⚠️ Breaking Many deprecated queue config options are removed (actually, they should have been removed in 1.18/1.19). If you see the fatal message when starting Gitea: "Please update your app.ini to remove deprecated config options", please follow the error messages to remove these options from your app.ini. Example: ``` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options ``` Many options in `[queue]` are are dropped, including: `WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`, `BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed from app.ini. # The problem The old queue package has some legacy problems: * complexity: I doubt few people could tell how it works. * maintainability: Too many channels and mutex/cond are mixed together, too many different structs/interfaces depends each other. * stability: due to the complexity & maintainability, sometimes there are strange bugs and difficult to debug, and some code doesn't have test (indeed some code is difficult to test because a lot of things are mixed together). * general applicability: although it is called "queue", its behavior is not a well-known queue. * scalability: it doesn't seem easy to make it work with a cluster without breaking its behaviors. It came from some very old code to "avoid breaking", however, its technical debt is too heavy now. It's a good time to introduce a better "queue" package. # The new queue package It keeps using old config and concept as much as possible. * It only contains two major kinds of concepts: * The "base queue": channel, levelqueue, redis * They have the same abstraction, the same interface, and they are tested by the same testing code. * The "WokerPoolQueue", it uses the "base queue" to provide "worker pool" function, calls the "handler" to process the data in the base queue. * The new code doesn't do "PushBack" * Think about a queue with many workers, the "PushBack" can't guarantee the order for re-queued unhandled items, so in new code it just does "normal push" * The new code doesn't do "pause/resume" * The "pause/resume" was designed to handle some handler's failure: eg: document indexer (elasticsearch) is down * If a queue is paused for long time, either the producers blocks or the new items are dropped. * The new code doesn't do such "pause/resume" trick, it's not a common queue's behavior and it doesn't help much. * If there are unhandled items, the "push" function just blocks for a few seconds and then re-queue them and retry. * The new code doesn't do "worker booster" * Gitea's queue's handlers are light functions, the cost is only the go-routine, so it doesn't make sense to "boost" them. * The new code only use "max worker number" to limit the concurrent workers. * The new "Push" never blocks forever * Instead of creating more and more blocking goroutines, return an error is more friendly to the server and to the end user. There are more details in code comments: eg: the "Flush" problem, the strange "code.index" hanging problem, the "immediate" queue problem. Almost ready for review. TODO: * [x] add some necessary comments during review * [x] add some more tests if necessary * [x] update documents and config options * [x] test max worker / active worker * [x] re-run the CI tasks to see whether any test is flaky * [x] improve the `handleOldLengthConfiguration` to provide more friendly messages * [x] fine tune default config values (eg: length?) ## Code coverage: ![image](https://user-images.githubusercontent.com/2114189/236620635-55576955-f95d-4810-b12f-879026a3afdf.png)
2023-05-08 14:49:59 +03:00
tplDashboard base.TplName = "admin/dashboard"
tplMonitor base.TplName = "admin/monitor"
tplStacktrace base.TplName = "admin/stacktrace"
tplQueueManage base.TplName = "admin/queue_manage"
)
2014-03-22 15:42:24 +04:00
var sysStatus struct {
Use auto-updating, natively hoverable, localized time elements (#23988) - Added [GitHub's `relative-time` element](https://github.com/github/relative-time-element) - Converted all formatted timestamps to use this element - No more flashes of unstyled content around time elements - These elements are localized using the `lang` property of the HTML file - Relative (e.g. the activities in the dashboard) and duration (e.g. server uptime in the admin page) time elements are auto-updated to keep up with the current time without refreshing the page - Code that is not needed anymore such as `formatting.js` and parts of `since.go` have been deleted Replaces #21440 Follows #22861 ## Screenshots ### Localized ![image](https://user-images.githubusercontent.com/20454870/230775041-f0af4fda-8f6b-46d3-b8e3-d340c791a50c.png) ![image](https://user-images.githubusercontent.com/20454870/230673393-931415a9-5729-4ac3-9a89-c0fb5fbeeeb7.png) ### Tooltips #### Native for dates ![image](https://user-images.githubusercontent.com/20454870/230797525-1fa0a854-83e3-484c-9da5-9425ab6528a3.png) #### Interactive for relative ![image](https://user-images.githubusercontent.com/115237/230796860-51e1d640-c820-4a34-ba2e-39087020626a.png) ### Auto-update ![rec](https://user-images.githubusercontent.com/20454870/230672159-37480d8f-435a-43e9-a2b0-44073351c805.gif) --------- Signed-off-by: Yarden Shoham <git@yardenshoham.com> Co-authored-by: wxiaoguang <wxiaoguang@gmail.com> Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: delvh <dev.lh@web.de>
2023-04-11 02:01:20 +03:00
StartTime string
2014-03-22 15:42:24 +04:00
NumGoroutine int
// General statistics.
MemAllocated string // bytes allocated and still in use
MemTotal string // bytes allocated (even if freed)
MemSys string // bytes obtained from system (sum of XxxSys below)
Lookups uint64 // number of pointer lookups
MemMallocs uint64 // number of mallocs
MemFrees uint64 // number of frees
// Main allocation heap statistics.
HeapAlloc string // bytes allocated and still in use
HeapSys string // bytes obtained from system
HeapIdle string // bytes in idle spans
HeapInuse string // bytes in non-idle span
HeapReleased string // bytes released to the OS
HeapObjects uint64 // total number of allocated objects
// Low-level fixed-size structure allocator statistics.
// Inuse is bytes used now.
// Sys is bytes obtained from system.
StackInuse string // bootstrap stacks
StackSys string
MSpanInuse string // mspan structures
MSpanSys string
MCacheInuse string // mcache structures
MCacheSys string
BuckHashSys string // profiling bucket hash table
GCSys string // GC metadata
OtherSys string // other system allocations
// Garbage collector statistics.
NextGC string // next run in HeapAlloc time (bytes)
LastGC string // last run in absolute time (ns)
PauseTotalNs string
PauseNs string // circular buffer of recent GC pause times, most recent at [(NumGC+255)%256]
NumGC uint32
}
func updateSystemStatus() {
Use auto-updating, natively hoverable, localized time elements (#23988) - Added [GitHub's `relative-time` element](https://github.com/github/relative-time-element) - Converted all formatted timestamps to use this element - No more flashes of unstyled content around time elements - These elements are localized using the `lang` property of the HTML file - Relative (e.g. the activities in the dashboard) and duration (e.g. server uptime in the admin page) time elements are auto-updated to keep up with the current time without refreshing the page - Code that is not needed anymore such as `formatting.js` and parts of `since.go` have been deleted Replaces #21440 Follows #22861 ## Screenshots ### Localized ![image](https://user-images.githubusercontent.com/20454870/230775041-f0af4fda-8f6b-46d3-b8e3-d340c791a50c.png) ![image](https://user-images.githubusercontent.com/20454870/230673393-931415a9-5729-4ac3-9a89-c0fb5fbeeeb7.png) ### Tooltips #### Native for dates ![image](https://user-images.githubusercontent.com/20454870/230797525-1fa0a854-83e3-484c-9da5-9425ab6528a3.png) #### Interactive for relative ![image](https://user-images.githubusercontent.com/115237/230796860-51e1d640-c820-4a34-ba2e-39087020626a.png) ### Auto-update ![rec](https://user-images.githubusercontent.com/20454870/230672159-37480d8f-435a-43e9-a2b0-44073351c805.gif) --------- Signed-off-by: Yarden Shoham <git@yardenshoham.com> Co-authored-by: wxiaoguang <wxiaoguang@gmail.com> Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: delvh <dev.lh@web.de>
2023-04-11 02:01:20 +03:00
sysStatus.StartTime = setting.AppStartTime.Format(time.RFC3339)
2014-03-22 17:21:57 +04:00
2014-03-22 15:42:24 +04:00
m := new(runtime.MemStats)
runtime.ReadMemStats(m)
sysStatus.NumGoroutine = runtime.NumGoroutine()
sysStatus.MemAllocated = base.FileSize(int64(m.Alloc))
sysStatus.MemTotal = base.FileSize(int64(m.TotalAlloc))
sysStatus.MemSys = base.FileSize(int64(m.Sys))
sysStatus.Lookups = m.Lookups
sysStatus.MemMallocs = m.Mallocs
sysStatus.MemFrees = m.Frees
sysStatus.HeapAlloc = base.FileSize(int64(m.HeapAlloc))
sysStatus.HeapSys = base.FileSize(int64(m.HeapSys))
sysStatus.HeapIdle = base.FileSize(int64(m.HeapIdle))
sysStatus.HeapInuse = base.FileSize(int64(m.HeapInuse))
sysStatus.HeapReleased = base.FileSize(int64(m.HeapReleased))
sysStatus.HeapObjects = m.HeapObjects
sysStatus.StackInuse = base.FileSize(int64(m.StackInuse))
sysStatus.StackSys = base.FileSize(int64(m.StackSys))
sysStatus.MSpanInuse = base.FileSize(int64(m.MSpanInuse))
sysStatus.MSpanSys = base.FileSize(int64(m.MSpanSys))
sysStatus.MCacheInuse = base.FileSize(int64(m.MCacheInuse))
sysStatus.MCacheSys = base.FileSize(int64(m.MCacheSys))
sysStatus.BuckHashSys = base.FileSize(int64(m.BuckHashSys))
sysStatus.GCSys = base.FileSize(int64(m.GCSys))
sysStatus.OtherSys = base.FileSize(int64(m.OtherSys))
sysStatus.NextGC = base.FileSize(int64(m.NextGC))
sysStatus.LastGC = fmt.Sprintf("%.1fs", float64(time.Now().UnixNano()-int64(m.LastGC))/1000/1000/1000)
2014-03-22 17:21:57 +04:00
sysStatus.PauseTotalNs = fmt.Sprintf("%.1fs", float64(m.PauseTotalNs)/1000/1000/1000)
sysStatus.PauseNs = fmt.Sprintf("%.3fs", float64(m.PauseNs[(m.NumGC+255)%256])/1000/1000/1000)
2014-03-22 15:42:24 +04:00
sysStatus.NumGC = m.NumGC
}
2016-11-21 06:21:24 +03:00
// Dashboard show admin panel dashboard
2016-03-11 19:56:52 +03:00
func Dashboard(ctx *context.Context) {
2014-08-28 18:29:00 +04:00
ctx.Data["Title"] = ctx.Tr("admin.dashboard")
ctx.Data["PageIsAdminDashboard"] = true
ctx.Data["Stats"] = activities_model.GetStatistic()
ctx.Data["NeedUpdate"] = updatechecker.GetNeedUpdate()
ctx.Data["RemoteVersion"] = updatechecker.GetRemoteVersion()
// FIXME: update periodically
updateSystemStatus()
ctx.Data["SysStatus"] = sysStatus
ctx.Data["SSH"] = setting.SSH
ctx.HTML(http.StatusOK, tplDashboard)
}
// DashboardPost run an admin operation
Move macaron to chi (#14293) Use [chi](https://github.com/go-chi/chi) instead of the forked [macaron](https://gitea.com/macaron/macaron). Since macaron and chi have conflicts with session share, this big PR becomes a have-to thing. According my previous idea, we can replace macaron step by step but I'm wrong. :( Below is a list of big changes on this PR. - [x] Define `context.ResponseWriter` interface with an implementation `context.Response`. - [x] Use chi instead of macaron, and also a customize `Route` to wrap chi so that the router usage is similar as before. - [x] Create different routers for `web`, `api`, `internal` and `install` so that the codes will be more clear and no magic . - [x] Use https://github.com/unrolled/render instead of macaron's internal render - [x] Use https://github.com/NYTimes/gziphandler instead of https://gitea.com/macaron/gzip - [x] Use https://gitea.com/go-chi/session which is a modified version of https://gitea.com/macaron/session and removed `nodb` support since it will not be maintained. **BREAK** - [x] Use https://gitea.com/go-chi/captcha which is a modified version of https://gitea.com/macaron/captcha - [x] Use https://gitea.com/go-chi/cache which is a modified version of https://gitea.com/macaron/cache - [x] Use https://gitea.com/go-chi/binding which is a modified version of https://gitea.com/macaron/binding - [x] Use https://github.com/go-chi/cors instead of https://gitea.com/macaron/cors - [x] Dropped https://gitea.com/macaron/i18n and make a new one in `code.gitea.io/gitea/modules/translation` - [x] Move validation form structs from `code.gitea.io/gitea/modules/auth` to `code.gitea.io/gitea/modules/forms` to avoid dependency cycle. - [x] Removed macaron log service because it's not need any more. **BREAK** - [x] All form structs have to be get by `web.GetForm(ctx)` in the route function but not as a function parameter on routes definition. - [x] Move Git HTTP protocol implementation to use routers directly. - [x] Fix the problem that chi routes don't support trailing slash but macaron did. - [x] `/api/v1/swagger` now will be redirect to `/api/swagger` but not render directly so that `APIContext` will not create a html render. Notices: - Chi router don't support request with trailing slash - Integration test `TestUserHeatmap` maybe mysql version related. It's failed on my macOS(mysql 5.7.29 installed via brew) but succeed on CI. Co-authored-by: 6543 <6543@obermui.de>
2021-01-26 18:36:53 +03:00
func DashboardPost(ctx *context.Context) {
form := web.GetForm(ctx).(*forms.AdminDashboardForm)
ctx.Data["Title"] = ctx.Tr("admin.dashboard")
ctx.Data["PageIsAdminDashboard"] = true
ctx.Data["Stats"] = activities_model.GetStatistic()
updateSystemStatus()
ctx.Data["SysStatus"] = sysStatus
2014-05-06 21:47:47 +04:00
// Run operation.
if form.Op != "" {
task := cron.GetTask(form.Op)
if task != nil {
go task.RunWithUser(ctx.Doer, nil)
ctx.Flash.Success(ctx.Tr("admin.dashboard.task.started", ctx.Tr("admin.dashboard."+form.Op)))
2014-05-06 21:47:47 +04:00
} else {
ctx.Flash.Error(ctx.Tr("admin.dashboard.task.unknown", form.Op))
2014-05-06 21:47:47 +04:00
}
}
if form.From == "monitor" {
ctx.Redirect(setting.AppSubURL + "/admin/monitor")
} else {
ctx.Redirect(setting.AppSubURL + "/admin")
}
2014-03-20 15:50:26 +04:00
}
2016-11-21 06:21:24 +03:00
// Monitor show admin monitor page
2016-03-11 19:56:52 +03:00
func Monitor(ctx *context.Context) {
2014-08-30 16:49:51 +04:00
ctx.Data["Title"] = ctx.Tr("admin.monitor")
ctx.Data["PageIsAdminMonitor"] = true
ctx.Data["Processes"], ctx.Data["ProcessCount"] = process.GetManager().Processes(false, true)
2015-08-17 21:19:29 +03:00
ctx.Data["Entries"] = cron.ListTasks()
Graceful Queues: Issue Indexing and Tasks (#9363) * Queue: Add generic graceful queues with settings * Queue & Setting: Add worker pool implementation * Queue: Add worker settings * Queue: Make resizing worker pools * Queue: Add name variable to queues * Queue: Add monitoring * Queue: Improve logging * Issues: Gracefulise the issues indexer Remove the old now unused specific queues * Task: Move to generic queue and gracefulise * Issues: Standardise the issues indexer queue settings * Fix test * Queue: Allow Redis to connect to unix * Prevent deadlock during early shutdown of issue indexer * Add MaxWorker settings to queues * Merge branch 'master' into graceful-queues * Update modules/indexer/issues/indexer.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/indexer/issues/indexer.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/queue/queue_channel.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/queue/queue_disk.go * Update modules/queue/queue_disk_channel.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Rename queue.Description to queue.ManagedQueue as per @guillep2k * Cancel pool workers when removed * Remove dependency on queue from setting * Update modules/queue/queue_redis.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * As per @guillep2k add mutex locks on shutdown/terminate * move unlocking out of setInternal * Add warning if number of workers < 0 * Small changes as per @guillep2k * No redis host specified not found * Clean up documentation for queues * Update docs/content/doc/advanced/config-cheat-sheet.en-us.md * Update modules/indexer/issues/indexer_test.go * Ensure that persistable channel queue is added to manager * Rename QUEUE_NAME REDIS_QUEUE_NAME * Revert "Rename QUEUE_NAME REDIS_QUEUE_NAME" This reverts commit 1f83b4fc9b9dabda186257b38c265fe7012f90df. Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: techknowlogick <matti@mdranta.net> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2020-01-07 14:23:09 +03:00
ctx.Data["Queues"] = queue.GetManager().ManagedQueues()
ctx.HTML(http.StatusOK, tplMonitor)
2014-06-13 21:01:52 +04:00
}
// GoroutineStacktrace show admin monitor goroutines page
func GoroutineStacktrace(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("admin.monitor")
ctx.Data["PageIsAdminMonitor"] = true
processStacks, processCount, goroutineCount, err := process.GetManager().ProcessStacktraces(false, false)
if err != nil {
ctx.ServerError("GoroutineStacktrace", err)
return
}
ctx.Data["ProcessStacks"] = processStacks
ctx.Data["GoroutineCount"] = goroutineCount
ctx.Data["ProcessCount"] = processCount
ctx.HTML(http.StatusOK, tplStacktrace)
}
// MonitorCancel cancels a process
func MonitorCancel(ctx *context.Context) {
pid := ctx.Params("pid")
process.GetManager().Cancel(process.IDType(pid))
ctx.JSON(http.StatusOK, map[string]interface{}{
"redirect": setting.AppSubURL + "/admin/monitor",
})
}