From 138cbe4d602838d0b4d431cb934d533cf5516ea9 Mon Sep 17 00:00:00 2001
From: tobi <31960611+tsmethurst@users.noreply.github.com>
Date: Mon, 22 Jan 2024 16:17:04 +0100
Subject: [PATCH] [feature] Ratelimit + serve emoji images on separate router
group (#2548)
* [feature] Serve + rate limit emoji files separately from attachments
* add a wee little warning about uploading loads of emojis
---
cmd/gotosocial/action/server/server.go | 21 +++-
cmd/gotosocial/action/testrig/testrig.go | 8 ++
internal/api/fileserver.go | 107 ++++++++++++++----
internal/api/fileserver/fileserver.go | 4 +-
.../settings/admin/emoji/local/overview.js | 5 +
5 files changed, 115 insertions(+), 30 deletions(-)
diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go
index 537f338b1..de9b3b3f1 100644
--- a/cmd/gotosocial/action/server/server.go
+++ b/cmd/gotosocial/action/server/server.go
@@ -104,6 +104,13 @@ var Start action.GTSAction = func(ctx context.Context) error {
return fmt.Errorf("error creating instance instance: %s", err)
}
+ // Get the instance account
+ // (we'll need this later).
+ instanceAccount, err := dbService.GetInstanceAccount(ctx, "")
+ if err != nil {
+ return fmt.Errorf("error retrieving instance account: %w", err)
+ }
+
// Open the storage backend
storage, err := gtsstorage.AutoConfig()
if err != nil {
@@ -311,16 +318,17 @@ var Start action.GTSAction = func(ctx context.Context) error {
// rate limiting
rlLimit := config.GetAdvancedRateLimitRequests()
rlExceptions := config.GetAdvancedRateLimitExceptions()
- clLimit := middleware.RateLimit(rlLimit, rlExceptions) // client api
- s2sLimit := middleware.RateLimit(rlLimit, rlExceptions) // server-to-server (AP)
- fsLimit := middleware.RateLimit(rlLimit, rlExceptions) // fileserver / web templates
+ clLimit := middleware.RateLimit(rlLimit, rlExceptions) // client api
+ s2sLimit := middleware.RateLimit(rlLimit, rlExceptions) // server-to-server (AP)
+ fsMainLimit := middleware.RateLimit(rlLimit, rlExceptions) // fileserver / web templates
+ fsEmojiLimit := middleware.RateLimit(rlLimit*2, rlExceptions) // fileserver (emojis only, use high limit)
// throttling
cpuMultiplier := config.GetAdvancedThrottlingMultiplier()
retryAfter := config.GetAdvancedThrottlingRetryAfter()
clThrottle := middleware.Throttle(cpuMultiplier, retryAfter) // client api
s2sThrottle := middleware.Throttle(cpuMultiplier, retryAfter) // server-to-server (AP)
- fsThrottle := middleware.Throttle(cpuMultiplier, retryAfter) // fileserver / web templates
+ fsThrottle := middleware.Throttle(cpuMultiplier, retryAfter) // fileserver / web templates / emojis
pkThrottle := middleware.Throttle(cpuMultiplier, retryAfter) // throttle public key endpoint separately
gzip := middleware.Gzip() // applied to all except fileserver
@@ -330,12 +338,13 @@ var Start action.GTSAction = func(ctx context.Context) error {
authModule.Route(router, clLimit, clThrottle, gzip)
clientModule.Route(router, clLimit, clThrottle, gzip)
metricsModule.Route(router, clLimit, clThrottle, gzip)
- fileserverModule.Route(router, fsLimit, fsThrottle)
+ fileserverModule.Route(router, fsMainLimit, fsThrottle)
+ fileserverModule.RouteEmojis(router, instanceAccount.ID, fsEmojiLimit, fsThrottle)
wellKnownModule.Route(router, gzip, s2sLimit, s2sThrottle)
nodeInfoModule.Route(router, s2sLimit, s2sThrottle, gzip)
activityPubModule.Route(router, s2sLimit, s2sThrottle, gzip)
activityPubModule.RoutePublicKey(router, s2sLimit, pkThrottle, gzip)
- webModule.Route(router, fsLimit, fsThrottle, gzip)
+ webModule.Route(router, fsMainLimit, fsThrottle, gzip)
// Start the GoToSocial server.
server := gotosocial.NewServer(dbService, router, cleaner)
diff --git a/cmd/gotosocial/action/testrig/testrig.go b/cmd/gotosocial/action/testrig/testrig.go
index d6bc92215..bf2c74f2f 100644
--- a/cmd/gotosocial/action/testrig/testrig.go
+++ b/cmd/gotosocial/action/testrig/testrig.go
@@ -83,6 +83,13 @@ var Start action.GTSAction = func(ctx context.Context) error {
testrig.StandardDBSetup(state.DB, nil)
+ // Get the instance account
+ // (we'll need this later).
+ instanceAccount, err := state.DB.GetInstanceAccount(ctx, "")
+ if err != nil {
+ return fmt.Errorf("error retrieving instance account: %w", err)
+ }
+
if os.Getenv("GTS_STORAGE_BACKEND") == "s3" {
var err error
state.Storage, err = storage.NewS3Storage()
@@ -225,6 +232,7 @@ var Start action.GTSAction = func(ctx context.Context) error {
clientModule.Route(router)
metricsModule.Route(router)
fileserverModule.Route(router)
+ fileserverModule.RouteEmojis(router, instanceAccount.ID)
wellKnownModule.Route(router)
nodeInfoModule.Route(router)
activityPubModule.Route(router)
diff --git a/internal/api/fileserver.go b/internal/api/fileserver.go
index 59f38c362..e0377b0e6 100644
--- a/internal/api/fileserver.go
+++ b/internal/api/fileserver.go
@@ -21,6 +21,7 @@ import (
"github.com/gin-gonic/gin"
"github.com/superseriousbusiness/gotosocial/internal/api/fileserver"
"github.com/superseriousbusiness/gotosocial/internal/config"
+ "github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/middleware"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/router"
@@ -30,36 +31,98 @@ type Fileserver struct {
fileserver *fileserver.Module
}
-func (f *Fileserver) Route(r *router.Router, m ...gin.HandlerFunc) {
- fileserverGroup := r.AttachGroup("fileserver")
-
- // Attach middlewares appropriate for this group.
- fileserverGroup.Use(m...)
- // If we're using local storage or proxying s3, we can set a
- // long max-age + immutable on all file requests to reflect
- // that we never host different files at the same URL (since
- // ULIDs are generated per piece of media), so we can
- // easily prevent clients having to fetch files repeatedly.
+// Attach cache middleware appropriate for file serving.
+func useFSCacheMiddleware(grp *gin.RouterGroup) {
+ // If we're using local storage or proxying s3 (ie., serving
+ // from here) we can set a long max-age + immutable on file
+ // requests to reflect that we never host different files at
+ // the same URL (since ULIDs are generated per piece of media),
+ // so we can prevent clients having to fetch files repeatedly.
//
- // If we *are* using non-proxying s3, however, the max age
- // must be set dynamically within the request handler,
- // based on how long the signed URL has left to live before
- // it expires. This ensures that clients won't cache expired
- // links. This is done within fileserver/servefile.go, so we
- // should not set the middleware here in that case.
+ // If we *are* using non-proxying s3, however (ie., not serving
+ // from here) the max age must be set dynamically within the
+ // request handler, based on how long the signed URL has left
+ // to live before it expires. This ensures that clients won't
+ // cache expired links. This is done within fileserver/servefile.go
+ // so we should not set the middleware here in that case.
//
// See:
//
// - https://developer.mozilla.org/en-US/docs/Web/HTTP/Caching#avoiding_revalidation
// - https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control#immutable
- if config.GetStorageBackend() == "local" || config.GetStorageS3Proxy() {
- fileserverGroup.Use(middleware.CacheControl(middleware.CacheControlConfig{
- Directives: []string{"private", "max-age=604800", "immutable"},
- Vary: []string{"Range"}, // Cache partial ranges separately.
- }))
+ servingFromHere := config.GetStorageBackend() == "local" || config.GetStorageS3Proxy()
+ if !servingFromHere {
+ return
}
- f.fileserver.Route(fileserverGroup.Handle)
+ grp.Use(middleware.CacheControl(middleware.CacheControlConfig{
+ Directives: []string{"private", "max-age=604800", "immutable"},
+ Vary: []string{"Range"}, // Cache partial ranges separately.
+ }))
+}
+
+// Route the "main" fileserver group
+// that handles everything except emojis.
+func (f *Fileserver) Route(
+ r *router.Router,
+ m ...gin.HandlerFunc,
+) {
+ const fsGroupPath = "fileserver" +
+ "/:" + fileserver.AccountIDKey +
+ "/:" + fileserver.MediaTypeKey
+ fsGroup := r.AttachGroup(fsGroupPath)
+
+ // Attach provided +
+ // cache middlewares.
+ fsGroup.Use(m...)
+ useFSCacheMiddleware(fsGroup)
+
+ f.fileserver.Route(fsGroup.Handle)
+}
+
+// Route the "emojis" fileserver
+// group to handle emojis specifically.
+//
+// instanceAccount ID is required because
+// that is the ID under which all emoji
+// files are stored, and from which all
+// emoji file requests are therefore served.
+func (f *Fileserver) RouteEmojis(
+ r *router.Router,
+ instanceAcctID string,
+ m ...gin.HandlerFunc,
+) {
+ var fsEmojiGroupPath = "fileserver" +
+ "/" + instanceAcctID +
+ "/" + string(media.TypeEmoji)
+ fsEmojiGroup := r.AttachGroup(fsEmojiGroupPath)
+
+ // Inject the instance account and emoji media
+ // type params into the gin context manually,
+ // since we know we're only going to be serving
+ // emojis (stored under the instance account ID)
+ // from this group. This allows us to use the
+ // same handler functions for both the "main"
+ // fileserver handler and the emojis handler.
+ fsEmojiGroup.Use(func(c *gin.Context) {
+ c.Params = append(c.Params, []gin.Param{
+ {
+ Key: fileserver.AccountIDKey,
+ Value: instanceAcctID,
+ },
+ {
+ Key: fileserver.MediaTypeKey,
+ Value: string(media.TypeEmoji),
+ },
+ }...)
+ })
+
+ // Attach provided +
+ // cache middlewares.
+ fsEmojiGroup.Use(m...)
+ useFSCacheMiddleware(fsEmojiGroup)
+
+ f.fileserver.Route(fsEmojiGroup.Handle)
}
func NewFileserver(p *processing.Processor) *Fileserver {
diff --git a/internal/api/fileserver/fileserver.go b/internal/api/fileserver/fileserver.go
index 3620ea63f..db15ce2e0 100644
--- a/internal/api/fileserver/fileserver.go
+++ b/internal/api/fileserver/fileserver.go
@@ -33,8 +33,8 @@ const (
MediaSizeKey = "media_size"
// FileNameKey is the actual filename being sought. Will usually be a UUID then something like .jpeg
FileNameKey = "file_name"
- // FileServePath is the fileserve path minus the 'fileserver' prefix.
- FileServePath = "/:" + AccountIDKey + "/:" + MediaTypeKey + "/:" + MediaSizeKey + "/:" + FileNameKey
+ // FileServePath is the fileserve path minus the 'fileserver/:account_id/:media_type' prefix.
+ FileServePath = "/:" + MediaSizeKey + "/:" + FileNameKey
)
type Module struct {
diff --git a/web/source/settings/admin/emoji/local/overview.js b/web/source/settings/admin/emoji/local/overview.js
index 757f07c43..44b11f584 100644
--- a/web/source/settings/admin/emoji/local/overview.js
+++ b/web/source/settings/admin/emoji/local/overview.js
@@ -64,6 +64,11 @@ module.exports = function EmojiOverview({ }) {
You can either upload them here directly, or copy from those already
present on other (known) instances through the Remote Emoji page.
+
+ Be warned! If you upload more than about 300-400 custom emojis in
+ total on your instance, this may lead to rate-limiting issues for users and clients
+ if they try to load all the emoji images at once (which is what many clients do).
+