[feature] Clean up/uncache remote media (#407)

* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully

README.md

@@ -197,6 +197,7 @@ The following libraries and frameworks are used by GoToSocial, with gratitude
 - [nfnt/resize](https://github.com/nfnt/resize); convenient image resizing. [ISC License](https://spdx.org/licenses/ISC.html).
 - [oklog/ulid](https://github.com/oklog/ulid); sequential, database-friendly ID generation. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html).
 - [ReneKroon/ttlcache](https://github.com/ReneKroon/ttlcache); in-memory caching. [MIT License](https://spdx.org/licenses/MIT.html).
+- [robfig/cron](https://github.com/robfig/cron); cron job scheduling. [MIT License](https://spdx.org/licenses/MIT.html).
 - [russross/blackfriday](https://github.com/russross/blackfriday); markdown parsing for statuses. [Simplified BSD License](https://spdx.org/licenses/BSD-2-Clause.html).
 - [sirupsen/logrus](https://github.com/sirupsen/logrus); logging. [MIT License](https://spdx.org/licenses/MIT.html).
 - [spf13/cobra](https://github.com/spf13/cobra); command-line tooling. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html).

cmd/gotosocial/flag/server.go

@@ -63,6 +63,7 @@ func Media(cmd *cobra.Command, values config.Values) {
 	cmd.Flags().Int(config.Keys.MediaVideoMaxSize, values.MediaVideoMaxSize, usage.MediaVideoMaxSize)
 	cmd.Flags().Int(config.Keys.MediaDescriptionMinChars, values.MediaDescriptionMinChars, usage.MediaDescriptionMinChars)
 	cmd.Flags().Int(config.Keys.MediaDescriptionMaxChars, values.MediaDescriptionMaxChars, usage.MediaDescriptionMaxChars)
+	cmd.Flags().Int(config.Keys.MediaRemoteCacheDays, values.MediaRemoteCacheDays, usage.MediaRemoteCacheDays)
 }
 
 // Storage attaches flags pertaining to storage config.

cmd/gotosocial/flag/usage.go

@@ -47,6 +47,7 @@ var usage = config.KeyNames{
 	MediaVideoMaxSize:          "Max size of accepted videos in bytes",
 	MediaDescriptionMinChars:   "Min required chars for an image description",
 	MediaDescriptionMaxChars:   "Max permitted chars for an image description",
+	MediaRemoteCacheDays:       "Number of days to locally cache media from remote instances. If set to 0, remote media will be kept indefinitely.",
 	StorageBackend:             "Storage backend to use for media attachments",
 	StorageLocalBasePath:       "Full path to an already-created directory where gts should store/retrieve media files. Subfolders will be created within this dir.",
 	StatusesMaxChars:           "Max permitted characters for posted statuses",

docs/configuration/media.md

@@ -28,4 +28,15 @@ media-description-min-chars: 0
 # Examples: [500, 1000, 1500]
 # Default: 500
 media-description-max-chars: 500
+
+# Int. Number of days to cache media from remote instances before they are removed from the cache.
+# A job will run every day at midnight to clean up any remote media older than the given amount of days.
+#
+# When remote media is removed from the cache, it is deleted from storage but the database entries for the media
+# are kept so that it can be fetched again if requested by a user.
+#
+# If this is set to 0, then media from remote instances will be cached indefinitely.
+# Examples: [30, 60, 7, 0]
+# Default: 30
+media-remote-cache-days: 30
 ```

example/config.yaml

@@ -205,6 +205,17 @@ media-description-min-chars: 0
 # Default: 500
 media-description-max-chars: 500
 
+# Int. Number of days to cache media from remote instances before they are removed from the cache.
+# A job will run every day at midnight to clean up any remote media older than the given amount of days.
+#
+# When remote media is removed from the cache, it is deleted from storage but the database entries for the media
+# are kept so that it can be fetched again if requested by a user.
+#
+# If this is set to 0, then media from remote instances will be cached indefinitely.
+# Examples: [30, 60, 7, 0]
+# Default: 30
+media-remote-cache-days: 30
+
 ##########################
 ##### STORAGE CONFIG #####
 ##########################

go.mod

@@ -18,12 +18,13 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/h2non/filetype v1.1.3
-	github.com/jackc/pgconn v1.10.1
+	github.com/jackc/pgconn v1.11.0
-	github.com/jackc/pgx/v4 v4.14.1
+	github.com/jackc/pgx/v4 v4.15.0
 	github.com/microcosm-cc/bluemonday v1.0.16
 	github.com/mitchellh/mapstructure v1.4.3
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/oklog/ulid v1.3.1
+	github.com/robfig/cron/v3 v3.0.1
 	github.com/russross/blackfriday/v2 v2.1.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.2.1
@@ -34,7 +35,7 @@ require (
 	github.com/superseriousbusiness/exif-terminator v0.1.0
 	github.com/superseriousbusiness/oauth2/v4 v4.3.2-SSB
 	github.com/tdewolff/minify/v2 v2.9.22
-	github.com/uptrace/bun v1.0.19
+	github.com/uptrace/bun v1.0.20
 	github.com/uptrace/bun/dialect/pgdialect v1.0.19
 	github.com/uptrace/bun/dialect/sqlitedialect v1.0.19
 	github.com/wagslane/go-password-validator v0.3.0
@@ -42,7 +43,7 @@ require (
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
 	golang.org/x/text v0.3.7
 	gopkg.in/mcuadros/go-syslog.v2 v2.3.0
-	modernc.org/sqlite v1.14.2
+	modernc.org/sqlite v1.14.6
 	mvdan.cc/xurls/v2 v2.3.0
 )
 
@@ -84,7 +85,7 @@ require (
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgproto3/v2 v2.2.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
-	github.com/jackc/pgtype v1.9.1 // indirect
+	github.com/jackc/pgtype v1.10.0 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
@@ -108,7 +109,7 @@ require (
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	golang.org/x/mod v0.5.1 // indirect
 	golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect
-	golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect
+	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
 	golang.org/x/tools v0.1.8 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
@@ -118,9 +119,9 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	lukechampine.com/uint128 v1.1.1 // indirect
-	modernc.org/cc/v3 v3.35.18 // indirect
+	modernc.org/cc/v3 v3.35.22 // indirect
-	modernc.org/ccgo/v3 v3.12.92 // indirect
+	modernc.org/ccgo/v3 v3.15.13 // indirect
-	modernc.org/libc v1.11.101 // indirect
+	modernc.org/libc v1.14.5 // indirect
 	modernc.org/mathutil v1.4.1 // indirect
 	modernc.org/memory v1.0.5 // indirect
 	modernc.org/opt v0.1.1 // indirect

go.sum

@@ -412,8 +412,8 @@ github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsU
 github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o=
 github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY=
 github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
-github.com/jackc/pgconn v1.10.1 h1:DzdIHIjG1AxGwoEEqS+mGsURyjt4enSmqzACXvVzOT8=
+github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ=
-github.com/jackc/pgconn v1.10.1/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
+github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
 github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
 github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
 github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
@@ -438,18 +438,18 @@ github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01C
 github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
 github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw=
 github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM=
-github.com/jackc/pgtype v1.9.1 h1:MJc2s0MFS8C3ok1wQTdQxWuXQcB6+HwAm5x1CzW7mf0=
+github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38=
-github.com/jackc/pgtype v1.9.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4=
+github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4=
 github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y=
 github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM=
 github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc=
 github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs=
-github.com/jackc/pgx/v4 v4.14.1 h1:71oo1KAGI6mXhLiTMn6iDFcp3e7+zon/capWjl2OEFU=
+github.com/jackc/pgx/v4 v4.15.0 h1:B7dTkXsdILD3MF987WGGCcg+tvLW6bZJdEcqVFeU//w=
-github.com/jackc/pgx/v4 v4.14.1/go.mod h1:RgDuE4Z34o7XE92RpLsvFiOEfrAUT0Xt2KxvX73W06M=
+github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw=
 github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
-github.com/jackc/puddle v1.2.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
@@ -518,8 +518,8 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA=
+github.com/mattn/go-sqlite3 v1.14.10 h1:MLn+5bFRlWMGoSRmJour3CL1w/qL96mvipqpwQW/Sfk=
-github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/mattn/go-sqlite3 v1.14.10/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/memcachier/mc v2.0.1+incompatible/go.mod h1:7bkvFE61leUBvXz+yxsOnGBQSZpBSPIMUQSmmSHvuXc=
 github.com/microcosm-cc/bluemonday v1.0.16 h1:kHmAq2t7WPWLjiGvzKa5o3HzSfahUKiOq7fAPUiMNIc=
@@ -590,6 +590,8 @@ github.com/quasoft/memstore v0.0.0-20191010062613-2bce066d2b0b h1:aUNXCGgukb4gtY
 github.com/quasoft/memstore v0.0.0-20191010062613-2bce066d2b0b/go.mod h1:wTPjTepVu7uJBYgZ0SdWHQlIas582j6cn2jgk4DDdlg=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
@@ -689,8 +691,9 @@ github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ=
 github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw=
-github.com/uptrace/bun v1.0.19 h1:Jl5GbcnLqeo4mHLVbF+LvzVXiXTrwQOdIVvuMO+5zXI=
 github.com/uptrace/bun v1.0.19/go.mod h1:Uv7z0z+7dXnUS9P5hMF0hdiM/4M+xOUHQCrZpyDrpRc=
+github.com/uptrace/bun v1.0.20 h1:/T4p9C9tEN75U3cFMBr5njlP+rBfs4An8BmlQQPbcfE=
+github.com/uptrace/bun v1.0.20/go.mod h1:Uv7z0z+7dXnUS9P5hMF0hdiM/4M+xOUHQCrZpyDrpRc=
 github.com/uptrace/bun/dialect/pgdialect v1.0.19 h1:8vdHpXRq3rbNDRn13llLxJ52wa+aBBn7QI0Q7IyW/d8=
 github.com/uptrace/bun/dialect/pgdialect v1.0.19/go.mod h1:VMmro4cCcjC7xAIeWYvRHv8V2nF+q2Tm36WK+poCeuo=
 github.com/uptrace/bun/dialect/sqlitedialect v1.0.19 h1:Oy0scO7sXbVBk9k6xc2yFF/X1maW5UU6uqO6coX/5/8=
@@ -970,8 +973,8 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211123173158-ef496fb156ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211210111614-af8b64212486 h1:5hpz5aRr+W1erYCL5JRhSUBJRph7l9XkNveoExlrKYk=
+golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
-golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1261,8 +1264,10 @@ modernc.org/cc/v3 v3.35.10/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g
 modernc.org/cc/v3 v3.35.15/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
 modernc.org/cc/v3 v3.35.16/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
 modernc.org/cc/v3 v3.35.17/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
-modernc.org/cc/v3 v3.35.18 h1:rMZhRcWrba0y3nVmdiQ7kxAgOOSq2m2f2VzjHLgEs6U=
 modernc.org/cc/v3 v3.35.18/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.20/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
+modernc.org/cc/v3 v3.35.22 h1:BzShpwCAP7TWzFppM4k2t03RhXhgYqaibROWkrWq7lE=
+modernc.org/cc/v3 v3.35.22/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
 modernc.org/ccgo/v3 v3.9.5/go.mod h1:umuo2EP2oDSBnD3ckjaVUXMrmeAw8C8OSICVa0iFf60=
 modernc.org/ccgo/v3 v3.10.0/go.mod h1:c0yBmkRFi7uW4J7fwx/JiijwOjeAeR2NoSaRVFPmjMw=
 modernc.org/ccgo/v3 v3.11.0/go.mod h1:dGNposbDp9TOZ/1KBxghxtUp/bzErD0/0QW4hhSaBMI=
@@ -1292,19 +1297,24 @@ modernc.org/ccgo/v3 v3.12.55/go.mod h1:rsXiIyJi9psOwiBkplOaHye5L4MOOaCjHg1Fxkj7I
 modernc.org/ccgo/v3 v3.12.56/go.mod h1:ljeFks3faDseCkr60JMpeDb2GSO3TKAmrzm7q9YOcMU=
 modernc.org/ccgo/v3 v3.12.57/go.mod h1:hNSF4DNVgBl8wYHpMvPqQWDQx8luqxDnNGCMM4NFNMc=
 modernc.org/ccgo/v3 v3.12.60/go.mod h1:k/Nn0zdO1xHVWjPYVshDeWKqbRWIfif5dtsIOCUVMqM=
-modernc.org/ccgo/v3 v3.12.65/go.mod h1:D6hQtKxPNZiY6wDBtehSGKFKmyXn53F8nGTpH+POmS4=
 modernc.org/ccgo/v3 v3.12.66/go.mod h1:jUuxlCFZTUZLMV08s7B1ekHX5+LIAurKTTaugUr/EhQ=
 modernc.org/ccgo/v3 v3.12.67/go.mod h1:Bll3KwKvGROizP2Xj17GEGOTrlvB1XcVaBrC90ORO84=
 modernc.org/ccgo/v3 v3.12.73/go.mod h1:hngkB+nUUqzOf3iqsM48Gf1FZhY599qzVg1iX+BT3cQ=
 modernc.org/ccgo/v3 v3.12.81/go.mod h1:p2A1duHoBBg1mFtYvnhAnQyI6vL0uw5PGYLSIgF6rYY=
-modernc.org/ccgo/v3 v3.12.82/go.mod h1:ApbflUfa5BKadjHynCficldU1ghjen84tuM5jRynB7w=
 modernc.org/ccgo/v3 v3.12.84/go.mod h1:ApbflUfa5BKadjHynCficldU1ghjen84tuM5jRynB7w=
 modernc.org/ccgo/v3 v3.12.86/go.mod h1:dN7S26DLTgVSni1PVA3KxxHTcykyDurf3OgUzNqTSrU=
 modernc.org/ccgo/v3 v3.12.90/go.mod h1:obhSc3CdivCRpYZmrvO88TXlW0NvoSVvdh/ccRjJYko=
-modernc.org/ccgo/v3 v3.12.92 h1:QT2o1iX5IMsKo0MQ9jKXKb5th+J1pDECtn0BXNB85bE=
 modernc.org/ccgo/v3 v3.12.92/go.mod h1:5yDdN7ti9KWPi5bRVWPl8UNhpEAtCjuEE7ayQnzzqHA=
-modernc.org/ccorpus v1.11.1 h1:K0qPfpVG1MJh5BYazccnmhywH4zHuOgJXgbjzyp6dWA=
+modernc.org/ccgo/v3 v3.13.1/go.mod h1:aBYVOUfIlcSnrsRVU8VRS35y2DIfpgkmVkYZ0tpIXi4=
+modernc.org/ccgo/v3 v3.15.1/go.mod h1:md59wBwDT2LznX/OTCPoVS6KIsdRgY8xqQwBV+hkTH0=
+modernc.org/ccgo/v3 v3.15.9/go.mod h1:md59wBwDT2LznX/OTCPoVS6KIsdRgY8xqQwBV+hkTH0=
+modernc.org/ccgo/v3 v3.15.10/go.mod h1:wQKxoFn0ynxMuCLfFD09c8XPUCc8obfchoVR9Cn0fI8=
+modernc.org/ccgo/v3 v3.15.12/go.mod h1:VFePOWoCd8uDGRJpq/zfJ29D0EVzMSyID8LCMWYbX6I=
+modernc.org/ccgo/v3 v3.15.13 h1:hqlCzNJTXLrhS70y1PqWckrF9x1btSQRC7JFuQcBg5c=
+modernc.org/ccgo/v3 v3.15.13/go.mod h1:QHtvdpeODlXjdK3tsbpyK+7U9JV4PQsrPGIbtmc0KfY=
 modernc.org/ccorpus v1.11.1/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
+modernc.org/ccorpus v1.11.4 h1:YOmQBBzE8GC/puUx76D5j/gJYIZQsydrh6VMJVfXF0M=
+modernc.org/ccorpus v1.11.4/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
 modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
 modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
 modernc.org/libc v1.9.8/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w=
@@ -1336,7 +1346,6 @@ modernc.org/libc v1.11.54/go.mod h1:S/FVnskbzVUrjfBqlGFIPA5m7UwB3n9fojHhCNfSsnw=
 modernc.org/libc v1.11.55/go.mod h1:j2A5YBRm6HjNkoSs/fzZrSxCuwWqcMYTDPLNx0URn3M=
 modernc.org/libc v1.11.56/go.mod h1:pakHkg5JdMLt2OgRadpPOTnyRXm/uzu+Yyg/LSLdi18=
 modernc.org/libc v1.11.58/go.mod h1:ns94Rxv0OWyoQrDqMFfWwka2BcaF6/61CqJRK9LP7S8=
-modernc.org/libc v1.11.70/go.mod h1:DUOmMYe+IvKi9n6Mycyx3DbjfzSKrdr/0Vgt3j7P5gw=
 modernc.org/libc v1.11.71/go.mod h1:DUOmMYe+IvKi9n6Mycyx3DbjfzSKrdr/0Vgt3j7P5gw=
 modernc.org/libc v1.11.75/go.mod h1:dGRVugT6edz361wmD9gk6ax1AbDSe0x5vji0dGJiPT0=
 modernc.org/libc v1.11.82/go.mod h1:NF+Ek1BOl2jeC7lw3a7Jj5PWyHPwWD4aq3wVKxqV1fI=
@@ -1344,8 +1353,13 @@ modernc.org/libc v1.11.86/go.mod h1:ePuYgoQLmvxdNT06RpGnaDKJmDNEkV7ZPKI2jnsvZoE=
 modernc.org/libc v1.11.87/go.mod h1:Qvd5iXTeLhI5PS0XSyqMY99282y+3euapQFxM7jYnpY=
 modernc.org/libc v1.11.88/go.mod h1:h3oIVe8dxmTcchcFuCcJ4nAWaoiwzKCdv82MM0oiIdQ=
 modernc.org/libc v1.11.98/go.mod h1:ynK5sbjsU77AP+nn61+k+wxUGRx9rOFcIqWYYMaDZ4c=
-modernc.org/libc v1.11.101 h1:mFp54enlBZiH0d26tRFoq2WL8KsjV1sefpOy4+gyTDQ=
 modernc.org/libc v1.11.101/go.mod h1:wLLYgEiY2D17NbBOEp+mIJJJBGSiy7fLL4ZrGGZ+8jI=
+modernc.org/libc v1.12.0/go.mod h1:2MH3DaF/gCU8i/UBiVE1VFRos4o523M7zipmwH8SIgQ=
+modernc.org/libc v1.14.1/go.mod h1:npFeGWjmZTjFeWALQLrvklVmAxv4m80jnG3+xI8FdJk=
+modernc.org/libc v1.14.2/go.mod h1:MX1GBLnRLNdvmK9azU9LCxZ5lMyhrbEMK8rG3X/Fe34=
+modernc.org/libc v1.14.3/go.mod h1:GPIvQVOVPizzlqyRX3l756/3ppsAgg1QgPxjr5Q4agQ=
+modernc.org/libc v1.14.5 h1:DAHvwGoVRDZs5iJXnX9RJrgXSsorupCWmJ2ac964Owk=
+modernc.org/libc v1.14.5/go.mod h1:2PJHINagVxO4QW/5OQdRrvMYo+bm5ClpUFfyXCYl9ak=
 modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
 modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
 modernc.org/mathutil v1.4.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
@@ -1356,16 +1370,16 @@ modernc.org/memory v1.0.5 h1:XRch8trV7GgvTec2i7jc33YlUI0RKVDBvZ5eZ5m8y14=
 modernc.org/memory v1.0.5/go.mod h1:B7OYswTRnfGg+4tDH1t1OeUNnsy2viGTdME4tzd+IjM=
 modernc.org/opt v0.1.1 h1:/0RX92k9vwVeDXj+Xn23DKp2VJubL7k8qNffND6qn3A=
 modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
-modernc.org/sqlite v1.14.2 h1:ohsW2+e+Qe2To1W6GNezzKGwjXwSax6R+CrhRxVaFbE=
+modernc.org/sqlite v1.14.6 h1:Jt5P3k80EtDBWaq1beAxnWW+5MdHXbZITujnRS7+zWg=
-modernc.org/sqlite v1.14.2/go.mod h1:yqfn85u8wVOE6ub5UT8VI9JjhrwBUUCNyTACN0h6Sx8=
+modernc.org/sqlite v1.14.6/go.mod h1:yiCvMv3HblGmzENNIaNtFhfaNIwcla4u2JQEwJPzfEc=
 modernc.org/strutil v1.1.1 h1:xv+J1BXY3Opl2ALrBwyfEikFAj8pmqcpnfmuwUwcozs=
 modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
-modernc.org/tcl v1.8.13 h1:V0sTNBw0Re86PvXZxuCub3oO9WrSTqALgrwNZNvLFGw=
+modernc.org/tcl v1.11.0 h1:B/zzEYjINeaki38KcIqdQRQx7W3WE7TkrlTwGnbm2II=
-modernc.org/tcl v1.8.13/go.mod h1:V+q/Ef0IJaNUSECieLU4o+8IScapxnMyFV6i/7uQlAY=
+modernc.org/tcl v1.11.0/go.mod h1:zsTUpbQ+NxQEjOjCUlImDLPv1sG8Ww0qp66ZvyOxCgw=
 modernc.org/token v1.0.0 h1:a0jaWiNMDhDUtqOj09wvjWWAqd3q7WpBulmL9H2egsk=
 modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
-modernc.org/z v1.2.19 h1:BGyRFWhDVn5LFS5OcX4Yd/MlpRTOc7hOPTdcIpCiUao=
+modernc.org/z v1.3.0 h1:4RWULo1Nvaq5ZBhbLe74u8p6tV4Mmm0ZrPBXYPm/xjM=
-modernc.org/z v1.2.19/go.mod h1:+ZpP0pc4zz97eukOzW3xagV/lS82IpPN9NGG5pNF9vY=
+modernc.org/z v1.3.0/go.mod h1:+mvgLH814oDjtATDdT3rs84JnUIpkvAF5B8AVkNlE2g=
 mvdan.cc/xurls/v2 v2.3.0 h1:59Olnbt67UKpxF1EwVBopJvkSUBmgtb468E4GVWIZ1I=
 mvdan.cc/xurls/v2 v2.3.0/go.mod h1:AjuTy7gEiUArFMjgBBDU4SMxlfUYsRokpJQgNWOt3e4=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

internal/api/client/fileserver/servefile.go

@@ -79,15 +79,15 @@ func (m *FileServer) ServeFile(c *gin.Context) {
 		return
 	}
 
-	content, err := m.processor.FileGet(c.Request.Context(), authed, &model.GetContentRequestForm{
+	content, errWithCode := m.processor.FileGet(c.Request.Context(), authed, &model.GetContentRequestForm{
 		AccountID: accountID,
 		MediaType: mediaType,
 		MediaSize: mediaSize,
 		FileName:  fileName,
 	})
-	if err != nil {
+	if errWithCode != nil {
-		l.Debug(err)
+		l.Errorf(errWithCode.Error())
-		c.String(http.StatusNotFound, "404 page not found")
+		c.JSON(errWithCode.Code(), gin.H{"error": errWithCode.Safe()})
 		return
 	}
 
@@ -104,7 +104,7 @@ func (m *FileServer) ServeFile(c *gin.Context) {
 	// This is mostly needed because when sharing a link to a gts-hosted file on something like mastodon, the masto servers will
 	// attempt to look up the content to provide a preview of the link, and they ask for text/html.
 	format, err := api.NegotiateAccept(c, api.Offer(content.ContentType))
-	if err != nil {
+	if errWithCode != nil {
 		c.JSON(http.StatusNotAcceptable, gin.H{"error": err.Error()})
 		return
 	}

internal/config/defaults.go

@@ -53,6 +53,7 @@ var Defaults = Values{
 	MediaVideoMaxSize:        10485760, // 10mb
 	MediaDescriptionMinChars: 0,
 	MediaDescriptionMaxChars: 500,
+	MediaRemoteCacheDays:     30,
 
 	StorageBackend:       "local",
 	StorageLocalBasePath: "/gotosocial/storage",

internal/config/keys.go

@@ -58,6 +58,7 @@ type KeyNames struct {
 	MediaVideoMaxSize        string
 	MediaDescriptionMinChars string
 	MediaDescriptionMaxChars string
+	MediaRemoteCacheDays     string
 
 	// storage
 	StorageBackend       string
@@ -138,6 +139,7 @@ var Keys = KeyNames{
 	MediaVideoMaxSize:        "media-video-max-size",
 	MediaDescriptionMinChars: "media-description-min-chars",
 	MediaDescriptionMaxChars: "media-description-max-chars",
+	MediaRemoteCacheDays:     "media-remote-cache-days",
 
 	StorageBackend:       "storage-backend",
 	StorageLocalBasePath: "storage-local-base-path",

internal/config/values.go

@@ -51,6 +51,7 @@ type Values struct {
 	MediaVideoMaxSize        int
 	MediaDescriptionMinChars int
 	MediaDescriptionMaxChars int
+	MediaRemoteCacheDays     int
 
 	StorageBackend       string
 	StorageLocalBasePath string

internal/db/bundb/basic_test.go

@@ -43,7 +43,7 @@ func (suite *BasicTestSuite) TestGetAllStatuses() {
 	s := []*gtsmodel.Status{}
 	err := suite.db.GetAll(context.Background(), &s)
 	suite.NoError(err)
-	suite.Len(s, 14)
+	suite.Len(s, 15)
 }
 
 func (suite *BasicTestSuite) TestGetAllNotNull() {

internal/db/bundb/bundb_test.go

@@ -55,9 +55,8 @@ func (suite *BunDBStandardTestSuite) SetupSuite() {
 }
 
 func (suite *BunDBStandardTestSuite) SetupTest() {
-	testrig.InitTestLog()
 	testrig.InitTestConfig()
-
+	testrig.InitTestLog()
 	suite.db = testrig.NewTestDB()
 	testrig.StandardDBSetup(suite.db, suite.testAccounts)
 }

internal/db/bundb/media.go

@@ -20,6 +20,7 @@ package bundb
 
 import (
 	"context"
+	"time"
 
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -48,3 +49,26 @@ func (m *mediaDB) GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.M
 	}
 	return attachment, nil
 }
+
+func (m *mediaDB) GetRemoteOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, db.Error) {
+	attachments := []*gtsmodel.MediaAttachment{}
+
+	q := m.conn.
+		NewSelect().
+		Model(&attachments).
+		Where("media_attachment.cached = true").
+		Where("media_attachment.avatar = false").
+		Where("media_attachment.header = false").
+		Where("media_attachment.created_at < ?", olderThan).
+		WhereGroup(" AND ", whereNotEmptyAndNotNull("media_attachment.remote_url")).
+		Order("media_attachment.created_at DESC")
+
+	if limit != 0 {
+		q = q.Limit(limit)
+	}
+
+	if err := q.Scan(ctx); err != nil {
+		return nil, m.conn.ProcessError(err)
+	}
+	return attachments, nil
+}

internal/db/bundb/media_test.go

@@ -0,0 +1,48 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package bundb_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/suite"
+)
+
+type MediaTestSuite struct {
+	BunDBStandardTestSuite
+}
+
+func (suite *MediaTestSuite) TestGetAttachmentByID() {
+	testAttachment := suite.testAttachments["admin_account_status_1_attachment_1"]
+	attachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID)
+	suite.NoError(err)
+	suite.NotNil(attachment)
+}
+
+func (suite *MediaTestSuite) TestGetOlder() {
+	attachments, err := suite.db.GetRemoteOlderThan(context.Background(), time.Now(), 20)
+	suite.NoError(err)
+	suite.Len(attachments, 1)
+}
+
+func TestMediaTestSuite(t *testing.T) {
+	suite.Run(t, new(MediaTestSuite))
+}

internal/db/bundb/migrations/20220214175650_media_cleanup.go

@@ -0,0 +1,172 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package migrations
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	previousgtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20211113114307_init"
+	newgtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20220214175650_media_cleanup"
+	"github.com/uptrace/bun"
+)
+
+func init() {
+	const batchSize = 100
+	up := func(ctx context.Context, db *bun.DB) error {
+		// we need to migrate media attachments into a new table
+		// see section 6 here: https://www.sqlite.org/lang_altertable.html
+
+		return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+			// create the new media attachments table
+			if _, err := tx.
+				NewCreateTable().
+				ModelTableExpr("new_media_attachments").
+				Model(&newgtsmodel.MediaAttachment{}).
+				IfNotExists().
+				Exec(ctx); err != nil {
+				return err
+			}
+
+			offset := time.Now()
+			// migrate existing media attachments into new table
+		migrateLoop:
+			for {
+				oldAttachments := []*previousgtsmodel.MediaAttachment{}
+				err := tx.
+					NewSelect().
+					Model(&oldAttachments).
+					// subtract a millisecond from the offset just to make sure we're not getting double entries (this happens sometimes)
+					Where("media_attachment.created_at < ?", offset.Add(-1*time.Millisecond)).
+					Order("media_attachment.created_at DESC").
+					Limit(batchSize).
+					Scan(ctx)
+				if err != nil && err != sql.ErrNoRows {
+					// there's been a real error
+					return err
+				}
+
+				if err == sql.ErrNoRows || len(oldAttachments) == 0 {
+					// we're finished migrating
+					break migrateLoop
+				}
+
+				// update the offset to the createdAt time of the oldest media attachment in the slice
+				offset = oldAttachments[len(oldAttachments)-1].CreatedAt
+
+				// for every old attachment, we need to make a new attachment out of it by taking the same values
+				newAttachments := []*newgtsmodel.MediaAttachment{}
+				for _, old := range oldAttachments {
+					new := &newgtsmodel.MediaAttachment{
+						ID:        old.ID,
+						CreatedAt: old.CreatedAt,
+						UpdatedAt: old.UpdatedAt,
+						StatusID:  old.StatusID,
+						URL:       old.URL,
+						RemoteURL: old.RemoteURL,
+						Type:      newgtsmodel.FileType(old.Type),
+						FileMeta: newgtsmodel.FileMeta{
+							Original: newgtsmodel.Original{
+								Width:  old.FileMeta.Original.Width,
+								Height: old.FileMeta.Original.Height,
+								Size:   old.FileMeta.Original.Size,
+								Aspect: old.FileMeta.Original.Aspect,
+							},
+							Small: newgtsmodel.Small{
+								Width:  old.FileMeta.Small.Width,
+								Height: old.FileMeta.Small.Height,
+								Size:   old.FileMeta.Small.Size,
+								Aspect: old.FileMeta.Small.Aspect,
+							},
+							Focus: newgtsmodel.Focus{
+								X: old.FileMeta.Focus.X,
+								Y: old.FileMeta.Focus.Y,
+							},
+						},
+						AccountID:         old.AccountID,
+						Description:       old.Description,
+						ScheduledStatusID: old.ScheduledStatusID,
+						Blurhash:          old.Blurhash,
+						Processing:        newgtsmodel.ProcessingStatus(old.Processing),
+						File: newgtsmodel.File{
+							Path:        old.File.Path,
+							ContentType: old.File.ContentType,
+							FileSize:    old.File.FileSize,
+							UpdatedAt:   old.File.UpdatedAt,
+						},
+						Thumbnail: newgtsmodel.Thumbnail{
+							Path:        old.Thumbnail.Path,
+							ContentType: old.Thumbnail.ContentType,
+							FileSize:    old.Thumbnail.FileSize,
+							UpdatedAt:   old.Thumbnail.UpdatedAt,
+							URL:         old.Thumbnail.URL,
+							RemoteURL:   old.Thumbnail.RemoteURL,
+						},
+						Avatar: old.Avatar,
+						Header: old.Header,
+						Cached: true,
+					}
+					newAttachments = append(newAttachments, new)
+				}
+
+				// insert this batch of new attachments, and then continue the loop
+				if _, err := tx.
+					NewInsert().
+					Model(&newAttachments).
+					ModelTableExpr("new_media_attachments").
+					Exec(ctx); err != nil {
+					return err
+				}
+			}
+
+			// we have all the data we need from the old table, so we can safely drop it now
+			if _, err := tx.NewDropTable().Model(&previousgtsmodel.MediaAttachment{}).Exec(ctx); err != nil {
+				return err
+			}
+
+			// rename the new table to the same name as the old table was
+			if _, err := tx.QueryContext(ctx, "ALTER TABLE new_media_attachments RENAME TO media_attachments;"); err != nil {
+				return err
+			}
+
+			// add an index to the new table
+			if _, err := tx.
+				NewCreateIndex().
+				Model(&newgtsmodel.MediaAttachment{}).
+				Index("media_attachments_id_idx").
+				Column("id").
+				Exec(ctx); err != nil {
+				return err
+			}
+
+			return nil
+		})
+	}
+
+	down := func(ctx context.Context, db *bun.DB) error {
+		return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+			return nil
+		})
+	}
+
+	if err := Migrations.Register(up, down); err != nil {
+		panic(err)
+	}
+}

internal/db/bundb/migrations/20220214175650_media_cleanup/account.go

@@ -0,0 +1,117 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+// Package gtsmodel contains types used *internally* by GoToSocial and added/removed/selected from the database.
+// These types should never be serialized and/or sent out via public APIs, as they contain sensitive information.
+// The annotation used on these structs is for handling them via the bun-db ORM.
+// See here for more info on bun model annotations: https://bun.uptrace.dev/guide/models.html
+package gtsmodel
+
+import (
+	"crypto/rsa"
+	"time"
+)
+
+// Account represents either a local or a remote fediverse account, gotosocial or otherwise (mastodon, pleroma, etc).
+type Account struct {
+	ID                      string           `validate:"required,ulid" bun:"type:CHAR(26),pk,nullzero,notnull,unique"`                                               // id of this item in the database
+	CreatedAt               time.Time        `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"`                                        // when was item created
+	UpdatedAt               time.Time        `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"`                                        // when was item last updated
+	Username                string           `validate:"required" bun:",nullzero,notnull,unique:userdomain"`                                                         // Username of the account, should just be a string of [a-zA-Z0-9_]. Can be added to domain to create the full username in the form ``[username]@[domain]`` eg., ``user_96@example.org``. Username and domain should be unique *with* each other
+	Domain                  string           `validate:"omitempty,fqdn" bun:",nullzero,unique:userdomain"`                                                           // Domain of the account, will be null if this is a local account, otherwise something like ``example.org``. Should be unique with username.
+	AvatarMediaAttachmentID string           `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"`                                                                // Database ID of the media attachment, if present
+	AvatarMediaAttachment   *MediaAttachment `validate:"-" bun:"rel:belongs-to"`                                                                                     // MediaAttachment corresponding to avatarMediaAttachmentID
+	AvatarRemoteURL         string           `validate:"omitempty,url" bun:",nullzero"`                                                                              // For a non-local account, where can the header be fetched?
+	HeaderMediaAttachmentID string           `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"`                                                                // Database ID of the media attachment, if present
+	HeaderMediaAttachment   *MediaAttachment `validate:"-" bun:"rel:belongs-to"`                                                                                     // MediaAttachment corresponding to headerMediaAttachmentID
+	HeaderRemoteURL         string           `validate:"omitempty,url" bun:",nullzero"`                                                                              // For a non-local account, where can the header be fetched?
+	DisplayName             string           `validate:"-" bun:""`                                                                                                   // DisplayName for this account. Can be empty, then just the Username will be used for display purposes.
+	Fields                  []Field          `validate:"-"`                                                                                                          // a key/value map of fields that this account has added to their profile
+	Note                    string           `validate:"-" bun:""`                                                                                                   // A note that this account has on their profile (ie., the account's bio/description of themselves)
+	Memorial                bool             `validate:"-" bun:",default:false"`                                                                                     // Is this a memorial account, ie., has the user passed away?
+	AlsoKnownAs             string           `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"`                                                                // This account is associated with x account id (TODO: migrate to be AlsoKnownAsID)
+	MovedToAccountID        string           `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"`                                                                // This account has moved this account id in the database
+	Bot                     bool             `validate:"-" bun:",default:false"`                                                                                     // Does this account identify itself as a bot?
+	Reason                  string           `validate:"-" bun:""`                                                                                                   // What reason was given for signing up when this account was created?
+	Locked                  bool             `validate:"-" bun:",default:true"`                                                                                      // Does this account need an approval for new followers?
+	Discoverable            bool             `validate:"-" bun:",default:false"`                                                                                     // Should this account be shown in the instance's profile directory?
+	Privacy                 Visibility       `validate:"required_without=Domain,omitempty,oneof=public unlocked followers_only mutuals_only direct" bun:",nullzero"` // Default post privacy for this account
+	Sensitive               bool             `validate:"-" bun:",default:false"`                                                                                     // Set posts from this account to sensitive by default?
+	Language                string           `validate:"omitempty,bcp47_language_tag" bun:",nullzero,notnull,default:'en'"`                                          // What language does this account post in?
+	URI                     string           `validate:"required,url" bun:",nullzero,notnull,unique"`                                                                // ActivityPub URI for this account.
+	URL                     string           `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"`                                               // Web URL for this account's profile
+	LastWebfingeredAt       time.Time        `validate:"required_with=Domain" bun:"type:timestamptz,nullzero"`                                                       // Last time this account was refreshed/located with webfinger.
+	InboxURI                string           `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"`                                               // Address of this account's ActivityPub inbox, for sending activity to
+	OutboxURI               string           `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"`                                               // Address of this account's activitypub outbox
+	FollowingURI            string           `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"`                                               // URI for getting the following list of this account
+	FollowersURI            string           `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"`                                               // URI for getting the followers list of this account
+	FeaturedCollectionURI   string           `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"`                                               // URL for getting the featured collection list of this account
+	ActorType               string           `validate:"oneof=Application Group Organization Person Service" bun:",nullzero,notnull"`                                // What type of activitypub actor is this account?
+	PrivateKey              *rsa.PrivateKey  `validate:"required_without=Domain"`                                                                                    // Privatekey for validating activitypub requests, will only be defined for local accounts
+	PublicKey               *rsa.PublicKey   `validate:"required"`                                                                                                   // Publickey for encoding activitypub requests, will be defined for both local and remote accounts
+	PublicKeyURI            string           `validate:"required,url" bun:",nullzero,notnull,unique"`                                                                // Web-reachable location of this account's public key
+	SensitizedAt            time.Time        `validate:"-" bun:"type:timestamptz,nullzero"`                                                                          // When was this account set to have all its media shown as sensitive?
+	SilencedAt              time.Time        `validate:"-" bun:"type:timestamptz,nullzero"`                                                                          // When was this account silenced (eg., statuses only visible to followers, not public)?
+	SuspendedAt             time.Time        `validate:"-" bun:"type:timestamptz,nullzero"`                                                                          // When was this account suspended (eg., don't allow it to log in/post, don't accept media/posts from this account)
+	HideCollections         bool             `validate:"-" bun:",default:false"`                                                                                     // Hide this account's collections
+	SuspensionOrigin        string           `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"`                                                                // id of the database entry that caused this account to become suspended -- can be an account ID or a domain block ID
+}
+
+// Field represents a key value field on an account, for things like pronouns, website, etc.
+// VerifiedAt is optional, to be used only if Value is a URL to a webpage that contains the
+// username of the user.
+type Field struct {
+	Name       string    `validate:"required"`          // Name of this field.
+	Value      string    `validate:"required"`          // Value of this field.
+	VerifiedAt time.Time `validate:"-" bun:",nullzero"` // This field was verified at (optional).
+}
+
+// Relationship describes a requester's relationship with another account.
+type Relationship struct {
+	ID                  string // The account id.
+	Following           bool   // Are you following this user?
+	ShowingReblogs      bool   // Are you receiving this user's boosts in your home timeline?
+	Notifying           bool   // Have you enabled notifications for this user?
+	FollowedBy          bool   // Are you followed by this user?
+	Blocking            bool   // Are you blocking this user?
+	BlockedBy           bool   // Is this user blocking you?
+	Muting              bool   // Are you muting this user?
+	MutingNotifications bool   // Are you muting notifications from this user?
+	Requested           bool   // Do you have a pending follow request for this user?
+	DomainBlocking      bool   // Are you blocking this user's domain?
+	Endorsed            bool   // Are you featuring this user on your profile?
+	Note                string // Your note on this account.
+}
+
+// Visibility represents the visibility granularity of a status.
+type Visibility string
+
+const (
+	// VisibilityPublic means this status will be visible to everyone on all timelines.
+	VisibilityPublic Visibility = "public"
+	// VisibilityUnlocked means this status will be visible to everyone, but will only show on home timeline to followers, and in lists.
+	VisibilityUnlocked Visibility = "unlocked"
+	// VisibilityFollowersOnly means this status is viewable to followers only.
+	VisibilityFollowersOnly Visibility = "followers_only"
+	// VisibilityMutualsOnly means this status is visible to mutual followers only.
+	VisibilityMutualsOnly Visibility = "mutuals_only"
+	// VisibilityDirect means this status is visible only to mentioned recipients.
+	VisibilityDirect Visibility = "direct"
+	// VisibilityDefault is used when no other setting can be found.
+	VisibilityDefault Visibility = VisibilityUnlocked
+)

internal/db/bundb/migrations/20220214175650_media_cleanup/mediaattachment.go

@@ -0,0 +1,116 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package gtsmodel
+
+import (
+	"time"
+)
+
+type MediaAttachment struct {
+	ID                string           `validate:"required,ulid" bun:"type:CHAR(26),pk,nullzero,notnull,unique"`                       // id of this item in the database
+	CreatedAt         time.Time        `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"`                // when was item created
+	UpdatedAt         time.Time        `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"`                // when was item last updated
+	StatusID          string           `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"`                                        // ID of the status to which this is attached
+	URL               string           `validate:"required_without=RemoteURL,omitempty,url" bun:",nullzero"`                           // Where can the attachment be retrieved on *this* server
+	RemoteURL         string           `validate:"required_without=URL,omitempty,url" bun:",nullzero"`                                 // Where can the attachment be retrieved on a remote server (empty for local media)
+	Type              FileType         `validate:"oneof=Image Gif Audio Video Unknown" bun:",nullzero,notnull"`                        // Type of file (image/gif/audio/video)
+	FileMeta          FileMeta         `validate:"required" bun:",embed:filemeta_,nullzero,notnull"`                                   // Metadata about the file
+	AccountID         string           `validate:"required,ulid" bun:"type:CHAR(26),nullzero,notnull"`                                 // To which account does this attachment belong
+	Account           *Account         `validate:"-" bun:"rel:has-one"`                                                                // Account corresponding to accountID
+	Description       string           `validate:"-" bun:""`                                                                           // Description of the attachment (for screenreaders)
+	ScheduledStatusID string           `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"`                                        // To which scheduled status does this attachment belong
+	Blurhash          string           `validate:"required_if=Type Image,required_if=Type Gif,required_if=Type Video" bun:",nullzero"` // What is the generated blurhash of this attachment
+	Processing        ProcessingStatus `validate:"oneof=0 1 2 666" bun:",notnull,default:2"`                                           // What is the processing status of this attachment
+	File              File             `validate:"required" bun:",embed:file_,notnull,nullzero"`                                       // metadata for the whole file
+	Thumbnail         Thumbnail        `validate:"required" bun:",embed:thumbnail_,notnull,nullzero"`                                  // small image thumbnail derived from a larger image, video, or audio file.
+	Avatar            bool             `validate:"-" bun:",notnull,default:false"`                                                     // Is this attachment being used as an avatar?
+	Header            bool             `validate:"-" bun:",notnull,default:false"`                                                     // Is this attachment being used as a header?
+	Cached            bool             `validate:"-" bun:",notnull"`                                                                   // Is this attachment currently cached by our instance?
+}
+
+// File refers to the metadata for the whole file
+type File struct {
+	Path        string    `validate:"required,file" bun:",nullzero,notnull"`                               // Path of the file in storage.
+	ContentType string    `validate:"required" bun:",nullzero,notnull"`                                    // MIME content type of the file.
+	FileSize    int       `validate:"required" bun:",notnull"`                                             // File size in bytes
+	UpdatedAt   time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // When was the file last updated.
+}
+
+// Thumbnail refers to a small image thumbnail derived from a larger image, video, or audio file.
+type Thumbnail struct {
+	Path        string    `validate:"required,file" bun:",nullzero,notnull"`                               // Path of the file in storage.
+	ContentType string    `validate:"required" bun:",nullzero,notnull"`                                    // MIME content type of the file.
+	FileSize    int       `validate:"required" bun:",notnull"`                                             // File size in bytes
+	UpdatedAt   time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // When was the file last updated.
+	URL         string    `validate:"required_without=RemoteURL,omitempty,url" bun:",nullzero"`            // What is the URL of the thumbnail on the local server
+	RemoteURL   string    `validate:"required_without=URL,omitempty,url" bun:",nullzero"`                  // What is the remote URL of the thumbnail (empty for local media)
+}
+
+// ProcessingStatus refers to how far along in the processing stage the attachment is.
+type ProcessingStatus int
+
+// MediaAttachment processing states.
+const (
+	ProcessingStatusReceived   ProcessingStatus = 0   // ProcessingStatusReceived indicates the attachment has been received and is awaiting processing. No thumbnail available yet.
+	ProcessingStatusProcessing ProcessingStatus = 1   // ProcessingStatusProcessing indicates the attachment is currently being processed. Thumbnail is available but full media is not.
+	ProcessingStatusProcessed  ProcessingStatus = 2   // ProcessingStatusProcessed indicates the attachment has been fully processed and is ready to be served.
+	ProcessingStatusError      ProcessingStatus = 666 // ProcessingStatusError indicates something went wrong processing the attachment and it won't be tried again--these can be deleted.
+)
+
+// FileType refers to the file type of the media attaachment.
+type FileType string
+
+// MediaAttachment file types.
+const (
+	FileTypeImage   FileType = "Image"   // FileTypeImage is for jpegs and pngs
+	FileTypeGif     FileType = "Gif"     // FileTypeGif is for native gifs and soundless videos that have been converted to gifs
+	FileTypeAudio   FileType = "Audio"   // FileTypeAudio is for audio-only files (no video)
+	FileTypeVideo   FileType = "Video"   // FileTypeVideo is for files with audio + visual
+	FileTypeUnknown FileType = "Unknown" // FileTypeUnknown is for unknown file types (surprise surprise!)
+)
+
+// FileMeta describes metadata about the actual contents of the file.
+type FileMeta struct {
+	Original Original `validate:"required" bun:"embed:original_"`
+	Small    Small    `bun:"embed:small_"`
+	Focus    Focus    `bun:"embed:focus_"`
+}
+
+// Small can be used for a thumbnail of any media type
+type Small struct {
+	Width  int     `validate:"required_with=Height Size Aspect"`  // width in pixels
+	Height int     `validate:"required_with=Width Size Aspect"`   // height in pixels
+	Size   int     `validate:"required_with=Width Height Aspect"` // size in pixels (width * height)
+	Aspect float64 `validate:"required_with=Widhth Height Size"`  // aspect ratio (width / height)
+}
+
+// Original can be used for original metadata for any media type
+type Original struct {
+	Width  int     `validate:"required_with=Height Size Aspect"`  // width in pixels
+	Height int     `validate:"required_with=Width Size Aspect"`   // height in pixels
+	Size   int     `validate:"required_with=Width Height Aspect"` // size in pixels (width * height)
+	Aspect float64 `validate:"required_with=Widhth Height Size"`  // aspect ratio (width / height)
+}
+
+// Focus describes the 'center' of the image for display purposes.
+// X and Y should each be between -1 and 1
+type Focus struct {
+	X float32 `validate:"omitempty,max=1,min=-1"`
+	Y float32 `validate:"omitempty,max=1,min=-1"`
+}

internal/db/bundb/util.go

@@ -37,6 +37,20 @@ func whereEmptyOrNull(column string) func(*bun.SelectQuery) *bun.SelectQuery {
 	}
 }
 
+// whereNotEmptyAndNotNull is a convenience function to return a bun WhereGroup that specifies
+// that the given column should be NEITHER an empty string NOR null.
+//
+// Use it as follows:
+//
+//   q = q.WhereGroup(" AND ", whereNotEmptyAndNotNull("whatever_column"))
+func whereNotEmptyAndNotNull(column string) func(*bun.SelectQuery) *bun.SelectQuery {
+	return func(q *bun.SelectQuery) *bun.SelectQuery {
+		return q.
+			Where("? IS NOT NULL", bun.Ident(column)).
+			Where("? != ''", bun.Ident(column))
+	}
+}
+
 // updateWhere parses []db.Where and adds it to the given update query.
 func updateWhere(q *bun.UpdateQuery, where []db.Where) {
 	for _, w := range where {

internal/db/media.go

@@ -20,6 +20,7 @@ package db
 
 import (
 	"context"
+	"time"
 
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
@@ -28,4 +29,10 @@ import (
 type Media interface {
 	// GetAttachmentByID gets a single attachment by its ID
 	GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.MediaAttachment, Error)
+	// GetRemoteOlderThan gets limit n remote media attachments older than the given olderThan time.
+	// These will be returned in order of attachment.created_at descending (newest to oldest in other words).
+	//
+	// The selected media attachments will be those with both a URL and a RemoteURL filled in.
+	// In other words, media attachments that originated remotely, and that we currently have cached locally.
+	GetRemoteOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, Error)
 }

internal/federation/federator.go

@@ -39,6 +39,8 @@ type Federator interface {
 	FederatingActor() pub.FederatingActor
 	// FederatingDB returns the underlying FederatingDB interface.
 	FederatingDB() federatingdb.DB
+	// TransportController returns the underlying transport controller.
+	TransportController() transport.Controller
 
 	// AuthenticateFederatedRequest can be used to check the authenticity of incoming http-signed requests for federating resources.
 	// The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments.
@@ -107,3 +109,7 @@ func (f *federator) FederatingActor() pub.FederatingActor {
 func (f *federator) FederatingDB() federatingdb.DB {
 	return f.federatingDB
 }
+
+func (f *federator) TransportController() transport.Controller {
+	return f.transportController
+}

internal/gtsmodel/mediaattachment.go

@@ -32,17 +32,18 @@ type MediaAttachment struct {
 	URL               string           `validate:"required_without=RemoteURL,omitempty,url" bun:",nullzero"`                           // Where can the attachment be retrieved on *this* server
 	RemoteURL         string           `validate:"required_without=URL,omitempty,url" bun:",nullzero"`                                 // Where can the attachment be retrieved on a remote server (empty for local media)
 	Type              FileType         `validate:"oneof=Image Gif Audio Video Unknown" bun:",nullzero,notnull"`                        // Type of file (image/gif/audio/video)
-	FileMeta          FileMeta         `validate:"required" bun:",nullzero,notnull"`                                                   // Metadata about the file
+	FileMeta          FileMeta         `validate:"required" bun:",embed:filemeta_,nullzero,notnull"`                                   // Metadata about the file
 	AccountID         string           `validate:"required,ulid" bun:"type:CHAR(26),nullzero,notnull"`                                 // To which account does this attachment belong
 	Account           *Account         `validate:"-" bun:"rel:has-one"`                                                                // Account corresponding to accountID
 	Description       string           `validate:"-" bun:""`                                                                           // Description of the attachment (for screenreaders)
 	ScheduledStatusID string           `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"`                                        // To which scheduled status does this attachment belong
 	Blurhash          string           `validate:"required_if=Type Image,required_if=Type Gif,required_if=Type Video" bun:",nullzero"` // What is the generated blurhash of this attachment
 	Processing        ProcessingStatus `validate:"oneof=0 1 2 666" bun:",notnull,default:2"`                                           // What is the processing status of this attachment
-	File              File             `validate:"required" bun:",notnull,nullzero"`                                                   // metadata for the whole file
+	File              File             `validate:"required" bun:",embed:file_,notnull,nullzero"`                                       // metadata for the whole file
-	Thumbnail         Thumbnail        `validate:"required" bun:",notnull,nullzero"`                                                   // small image thumbnail derived from a larger image, video, or audio file.
+	Thumbnail         Thumbnail        `validate:"required" bun:",embed:thumbnail_,notnull,nullzero"`                                  // small image thumbnail derived from a larger image, video, or audio file.
 	Avatar            bool             `validate:"-" bun:",notnull,default:false"`                                                     // Is this attachment being used as an avatar?
 	Header            bool             `validate:"-" bun:",notnull,default:false"`                                                     // Is this attachment being used as a header?
+	Cached            bool             `validate:"-" bun:",notnull"`                                                                   // Is this attachment currently cached by our instance?
 }
 
 // File refers to the metadata for the whole file
@@ -88,9 +89,9 @@ const (
 
 // FileMeta describes metadata about the actual contents of the file.
 type FileMeta struct {
-	Original Original `validate:"required"`
+	Original Original `validate:"required" bun:"embed:original_"`
-	Small    Small
+	Small    Small    `bun:"embed:small_"`
-	Focus    Focus
+	Focus    Focus    `bun:"embed:focus_"`
 }
 
 // Small can be used for a thumbnail of any media type

internal/media/manager.go

@@ -21,11 +21,16 @@ package media
 import (
 	"context"
 	"errors"
+	"fmt"
 	"runtime"
+	"time"
 
 	"codeberg.org/gruf/go-runners"
 	"codeberg.org/gruf/go-store/kv"
+	"github.com/robfig/cron/v3"
 	"github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 )
 
@@ -61,6 +66,12 @@ type Manager interface {
 	//
 	// ai is optional and can be nil. Any additional information about the emoji provided will be put in the database.
 	ProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error)
+	// RecacheMedia refetches, reprocesses, and recaches an existing attachment that has been uncached via pruneRemote.
+	RecacheMedia(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, attachmentID string) (*ProcessingMedia, error)
+	// PruneRemote prunes all remote media cached on this instance that's older than the given amount of days.
+	// 'Pruning' in this context means removing the locally stored data of the attachment (both thumbnail and full size),
+	// and setting 'cached' to false on the associated attachment.
+	PruneRemote(ctx context.Context, olderThanDays int) (int, error)
 	// NumWorkers returns the total number of workers available to this manager.
 	NumWorkers() int
 	// QueueSize returns the total capacity of the queue.
@@ -79,6 +90,7 @@ type manager struct {
 	db           db.DB
 	storage      *kv.KVStore
 	pool         runners.WorkerPool
+	stopCronJobs func() error
 	numWorkers   int
 	queueSize    int
 }
@@ -97,8 +109,10 @@ type manager struct {
 // For a 4 core machine, this will be 2 workers, and a queue length of 20.
 // For a single or 2-core machine, the media manager will get 1 worker, and a queue of length 10.
 func NewManager(database db.DB, storage *kv.KVStore) (Manager, error) {
-	numWorkers := runtime.NumCPU() / 2
+
+	// configure the worker pool
 	// make sure we always have at least 1 worker even on single-core machines
+	numWorkers := runtime.NumCPU() / 2
 	if numWorkers == 0 {
 		numWorkers = 1
 	}
@@ -112,11 +126,61 @@ func NewManager(database db.DB, storage *kv.KVStore) (Manager, error) {
 		queueSize:  queueSize,
 	}
 
+	// start the worker pool
 	if start := m.pool.Start(); !start {
 		return nil, errors.New("could not start worker pool")
 	}
 	logrus.Debugf("started media manager worker pool with %d workers and queue capacity of %d", numWorkers, queueSize)
 
+	// start remote cache cleanup cronjob if configured
+	cacheCleanupDays := viper.GetInt(config.Keys.MediaRemoteCacheDays)
+	if cacheCleanupDays != 0 {
+		// we need a way of cancelling running jobs if the media manager is told to stop
+		pruneCtx, pruneCancel := context.WithCancel(context.Background())
+
+		// create a new cron instance and add a function to it
+		c := cron.New(cron.WithLogger(&logrusWrapper{}))
+
+		pruneFunc := func() {
+			begin := time.Now()
+			pruned, err := m.PruneRemote(pruneCtx, cacheCleanupDays)
+			if err != nil {
+				logrus.Errorf("media manager: error pruning remote cache: %s", err)
+				return
+			}
+			logrus.Infof("media manager: pruned %d remote cache entries in %s", pruned, time.Since(begin))
+		}
+
+		// run every night
+		entryID, err := c.AddFunc("@midnight", pruneFunc)
+		if err != nil {
+			pruneCancel()
+			return nil, fmt.Errorf("error starting media manager remote cache cleanup job: %s", err)
+		}
+
+		// since we're running a cron job, we should define how the manager should stop them
+		m.stopCronJobs = func() error {
+			// try to stop any jobs gracefully by waiting til they're finished
+			cronCtx := c.Stop()
+
+			select {
+			case <-cronCtx.Done():
+				logrus.Infof("media manager: cron finished jobs and stopped gracefully")
+			case <-time.After(1 * time.Minute):
+				logrus.Infof("media manager: cron didn't stop after 60 seconds, will force close")
+				break
+			}
+
+			// whether the job is finished neatly or we had to wait a minute, cancel the context on the prune job
+			pruneCancel()
+			return nil
+		}
+
+		// now start all the cron stuff we've lined up
+		c.Start()
+		logrus.Infof("started media manager remote cache cleanup job: will run next at %s", c.Entry(entryID).Next)
+	}
+
 	return m, nil
 }
 
@@ -168,6 +232,30 @@ func (m *manager) ProcessEmoji(ctx context.Context, data DataFunc, postData Post
 	return processingEmoji, nil
 }
 
+func (m *manager) RecacheMedia(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, attachmentID string) (*ProcessingMedia, error) {
+	processingRecache, err := m.preProcessRecache(ctx, data, postData, attachmentID)
+	if err != nil {
+		return nil, err
+	}
+
+	logrus.Tracef("RecacheMedia: about to enqueue recache with attachmentID %s, queue length is %d", processingRecache.AttachmentID(), m.pool.Queue())
+	m.pool.Enqueue(func(innerCtx context.Context) {
+		select {
+		case <-innerCtx.Done():
+			// if the inner context is done that means the worker pool is closing, so we should just return
+			return
+		default:
+			// start loading the media already for the caller's convenience
+			if _, err := processingRecache.LoadAttachment(innerCtx); err != nil {
+				logrus.Errorf("RecacheMedia: error processing recache with attachmentID %s: %s", processingRecache.AttachmentID(), err)
+			}
+		}
+	})
+	logrus.Tracef("RecacheMedia: succesfully queued recache with attachmentID %s, queue length is %d", processingRecache.AttachmentID(), m.pool.Queue())
+
+	return processingRecache, nil
+}
+
 func (m *manager) NumWorkers() int {
 	return m.numWorkers
 }
@@ -186,10 +274,15 @@ func (m *manager) ActiveWorkers() int {
 
 func (m *manager) Stop() error {
 	logrus.Info("stopping media manager worker pool")
-
 	stopped := m.pool.Stop()
 	if !stopped {
 		return errors.New("could not stop media manager worker pool")
 	}
+
+	if m.stopCronJobs != nil { // only defined if cron jobs are actually running
+		logrus.Info("stopping media manager cache cleanup jobs")
+		return m.stopCronJobs()
+	}
+
 	return nil
 }

internal/media/manager_test.go

@@ -31,7 +31,7 @@ import (
 	"codeberg.org/gruf/go-store/kv"
 	"codeberg.org/gruf/go-store/storage"
 	"github.com/stretchr/testify/suite"
-	gtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20211113114307_init"
+	gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 )
 

internal/media/media_test.go

@@ -22,6 +22,7 @@ import (
 	"codeberg.org/gruf/go-store/kv"
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
@@ -32,6 +33,7 @@ type MediaStandardTestSuite struct {
 	db              db.DB
 	storage         *kv.KVStore
 	manager         media.Manager
+	testAttachments map[string]*gtsmodel.MediaAttachment
 }
 
 func (suite *MediaStandardTestSuite) SetupSuite() {
@@ -45,6 +47,7 @@ func (suite *MediaStandardTestSuite) SetupSuite() {
 func (suite *MediaStandardTestSuite) SetupTest() {
 	testrig.StandardStorageSetup(suite.storage, "../../testrig/media")
 	testrig.StandardDBSetup(suite.db, nil)
+	suite.testAttachments = testrig.NewTestAttachments()
 	suite.manager = testrig.NewTestMediaManager(suite.db, suite.storage)
 }
 

internal/media/processingmedia.go

@@ -66,6 +66,9 @@ type ProcessingMedia struct {
 
 	// track whether this media has already been put in the databse
 	insertedInDB bool
+
+	// true if this is a recache, false if it's brand new media
+	recache bool
 }
 
 // AttachmentID returns the ID of the underlying media attachment without blocking processing.
@@ -93,9 +96,17 @@ func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAt
 
 	// store the result in the database before returning it
 	if !p.insertedInDB {
+		if p.recache {
+			// if it's a recache we should only need to update
+			if err := p.database.UpdateByPrimaryKey(ctx, p.attachment); err != nil {
+				return nil, err
+			}
+		} else {
+			// otherwise we need to really PUT it
 			if err := p.database.Put(ctx, p.attachment); err != nil {
 				return nil, err
 			}
+		}
 		p.insertedInDB = true
 	}
 
@@ -305,6 +316,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 	if err := p.storage.PutStream(p.attachment.File.Path, clean); err != nil {
 		return fmt.Errorf("store: error storing stream: %s", err)
 	}
+	p.attachment.Cached = true
 
 	// if the original reader is a readcloser, close it since we're done with it now
 	if rc, ok := reader.(io.ReadCloser); ok {
@@ -360,6 +372,7 @@ func (m *manager) preProcessMedia(ctx context.Context, data DataFunc, postData P
 		Thumbnail:         thumbnail,
 		Avatar:            false,
 		Header:            false,
+		Cached:            false,
 	}
 
 	// check if we have additional info to add to the attachment,
@@ -418,3 +431,24 @@ func (m *manager) preProcessMedia(ctx context.Context, data DataFunc, postData P
 
 	return processingMedia, nil
 }
+
+func (m *manager) preProcessRecache(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, attachmentID string) (*ProcessingMedia, error) {
+	// get the existing attachment
+	attachment, err := m.db.GetAttachmentByID(ctx, attachmentID)
+	if err != nil {
+		return nil, err
+	}
+
+	processingMedia := &ProcessingMedia{
+		attachment:    attachment,
+		data:          data,
+		postData:      postData,
+		thumbState:    int32(received),
+		fullSizeState: int32(received),
+		database:      m.db,
+		storage:       m.storage,
+		recache:       true, // indicate it's a recache
+	}
+
+	return processingMedia, nil
+}

internal/media/pruneremote.go

@@ -0,0 +1,96 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package media
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"codeberg.org/gruf/go-store/storage"
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+// amount of media attachments to select at a time from the db when pruning
+const selectPruneLimit = 20
+
+func (m *manager) PruneRemote(ctx context.Context, olderThanDays int) (int, error) {
+	var totalPruned int
+
+	// convert days into a duration string
+	olderThanHoursString := fmt.Sprintf("%dh", olderThanDays*24)
+	// parse the duration string into a duration
+	olderThanHours, err := time.ParseDuration(olderThanHoursString)
+	if err != nil {
+		return totalPruned, fmt.Errorf("PruneRemote: %d", err)
+	}
+	// 'subtract' that from the time now to give our threshold
+	olderThan := time.Now().Add(-olderThanHours)
+	logrus.Infof("PruneRemote: pruning media older than %s", olderThan)
+
+	// select 20 attachments at a time and prune them
+	for attachments, err := m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit) {
+
+		// use the age of the oldest attachment (the last one in the slice) as the next 'older than' value
+		l := len(attachments)
+		logrus.Tracef("PruneRemote: got %d attachments older than %s", l, olderThan)
+		olderThan = attachments[l-1].CreatedAt
+
+		// prune each attachment
+		for _, attachment := range attachments {
+			if err := m.PruneOne(ctx, attachment); err != nil {
+				return totalPruned, err
+			}
+			totalPruned++
+		}
+	}
+
+	// make sure we don't have a real error when we leave the loop
+	if err != nil && err != db.ErrNoEntries {
+		return totalPruned, err
+	}
+
+	logrus.Infof("PruneRemote: finished pruning remote media: pruned %d entries", totalPruned)
+	return totalPruned, nil
+}
+
+func (m *manager) PruneOne(ctx context.Context, attachment *gtsmodel.MediaAttachment) error {
+	if attachment.File.Path != "" {
+		// delete the full size attachment from storage
+		logrus.Tracef("PruneOne: deleting %s", attachment.File.Path)
+		if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound {
+			return err
+		}
+		attachment.Cached = false
+	}
+
+	if attachment.Thumbnail.Path != "" {
+		// delete the thumbnail from storage
+		logrus.Tracef("PruneOne: deleting %s", attachment.Thumbnail.Path)
+		if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
+			return err
+		}
+		attachment.Cached = false
+	}
+
+	// update the attachment to reflect that we no longer have it cached
+	return m.db.UpdateByPrimaryKey(ctx, attachment)
+}

internal/media/pruneremote_test.go

@@ -0,0 +1,111 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package media_test
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"os"
+	"testing"
+
+	"codeberg.org/gruf/go-store/storage"
+	"github.com/stretchr/testify/suite"
+)
+
+type PruneRemoteTestSuite struct {
+	MediaStandardTestSuite
+}
+
+func (suite *PruneRemoteTestSuite) TestPruneRemote() {
+	testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
+	suite.True(testAttachment.Cached)
+
+	totalPruned, err := suite.manager.PruneRemote(context.Background(), 1)
+	suite.NoError(err)
+	suite.Equal(1, totalPruned)
+
+	prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID)
+	suite.NoError(err)
+
+	// the media should no longer be cached
+	suite.False(prunedAttachment.Cached)
+}
+
+func (suite *PruneRemoteTestSuite) TestPruneRemoteTwice() {
+	totalPruned, err := suite.manager.PruneRemote(context.Background(), 1)
+	suite.NoError(err)
+	suite.Equal(1, totalPruned)
+
+	// final prune should prune nothing, since the first prune already happened
+	totalPrunedAgain, err := suite.manager.PruneRemote(context.Background(), 1)
+	suite.NoError(err)
+	suite.Equal(0, totalPrunedAgain)
+}
+
+func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
+	ctx := context.Background()
+	testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
+
+	totalPruned, err := suite.manager.PruneRemote(ctx, 1)
+	suite.NoError(err)
+	suite.Equal(1, totalPruned)
+
+	// media should no longer be stored
+	_, err = suite.storage.Get(testAttachment.File.Path)
+	suite.Error(err)
+	suite.ErrorIs(err, storage.ErrNotFound)
+	_, err = suite.storage.Get(testAttachment.Thumbnail.Path)
+	suite.Error(err)
+	suite.ErrorIs(err, storage.ErrNotFound)
+
+	// now recache the image....
+	data := func(_ context.Context) (io.Reader, int, error) {
+		// load bytes from a test image
+		b, err := os.ReadFile("../../testrig/media/thoughtsofdog-original.jpeg")
+		if err != nil {
+			panic(err)
+		}
+		return bytes.NewBuffer(b), len(b), nil
+	}
+	processingRecache, err := suite.manager.RecacheMedia(ctx, data, nil, testAttachment.ID)
+	suite.NoError(err)
+
+	// synchronously load the recached attachment
+	recachedAttachment, err := processingRecache.LoadAttachment(ctx)
+	suite.NoError(err)
+	suite.NotNil(recachedAttachment)
+
+	// recachedAttachment should be basically the same as the old attachment
+	suite.True(recachedAttachment.Cached)
+	suite.Equal(testAttachment.ID, recachedAttachment.ID)
+	suite.Equal(testAttachment.File.Path, recachedAttachment.File.Path)           // file should be stored in the same place
+	suite.Equal(testAttachment.Thumbnail.Path, recachedAttachment.Thumbnail.Path) // as should the thumbnail
+	suite.EqualValues(testAttachment.FileMeta, recachedAttachment.FileMeta)       // and the filemeta should be the same
+
+	// recached files should be back in storage
+	_, err = suite.storage.Get(recachedAttachment.File.Path)
+	suite.NoError(err)
+	_, err = suite.storage.Get(recachedAttachment.Thumbnail.Path)
+	suite.NoError(err)
+}
+
+func TestPruneRemoteTestSuite(t *testing.T) {
+	suite.Run(t, &PruneRemoteTestSuite{})
+}

internal/media/util.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 
 	"github.com/h2non/filetype"
+	"github.com/sirupsen/logrus"
 )
 
 // parseContentType parses the MIME content type from a file, returning it as a string in the form (eg., "image/jpeg").
@@ -103,3 +104,17 @@ func ParseMediaSize(s string) (Size, error) {
 	}
 	return "", fmt.Errorf("%s not a recognized MediaSize", s)
 }
+
+// logrusWrapper is just a util for passing the logrus logger into the cron logging system.
+type logrusWrapper struct {
+}
+
+// Info logs routine messages about cron's operation.
+func (l *logrusWrapper) Info(msg string, keysAndValues ...interface{}) {
+	logrus.Info("media manager cron logger: ", msg, keysAndValues)
+}
+
+// Error logs an error condition.
+func (l *logrusWrapper) Error(err error, msg string, keysAndValues ...interface{}) {
+	logrus.Error("media manager cron logger: ", err, msg, keysAndValues)
+}

internal/processing/media.go

@@ -38,6 +38,6 @@ func (p *processor) MediaUpdate(ctx context.Context, authed *oauth.Auth, mediaAt
 	return p.mediaProcessor.Update(ctx, authed.Account, mediaAttachmentID, form)
 }
 
-func (p *processor) FileGet(ctx context.Context, authed *oauth.Auth, form *apimodel.GetContentRequestForm) (*apimodel.Content, error) {
+func (p *processor) FileGet(ctx context.Context, authed *oauth.Auth, form *apimodel.GetContentRequestForm) (*apimodel.Content, gtserror.WithCode) {
 	return p.mediaProcessor.GetFile(ctx, authed.Account, form)
 }

internal/processing/media/getfile.go

@@ -19,8 +19,11 @@
 package media
 
 import (
+	"bufio"
 	"context"
 	"fmt"
+	"io"
+	"net/url"
 	"strings"
 
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
@@ -29,7 +32,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 )
 
-func (p *processor) GetFile(ctx context.Context, account *gtsmodel.Account, form *apimodel.GetContentRequestForm) (*apimodel.Content, error) {
+func (p *processor) GetFile(ctx context.Context, account *gtsmodel.Account, form *apimodel.GetContentRequestForm) (*apimodel.Content, gtserror.WithCode) {
 	// parse the form fields
 	mediaSize, err := media.ParseMediaSize(form.MediaSize)
 	if err != nil {
@@ -46,74 +49,208 @@ func (p *processor) GetFile(ctx context.Context, account *gtsmodel.Account, form
 		return nil, gtserror.NewErrorNotFound(fmt.Errorf("file name %s not parseable", form.FileName))
 	}
 	wantedMediaID := spl[0]
+	expectedAccountID := form.AccountID
 
 	// get the account that owns the media and make sure it's not suspended
-	acct, err := p.db.GetAccountByID(ctx, form.AccountID)
+	acct, err := p.db.GetAccountByID(ctx, expectedAccountID)
 	if err != nil {
-		return nil, gtserror.NewErrorNotFound(fmt.Errorf("account with id %s could not be selected from the db: %s", form.AccountID, err))
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("account with id %s could not be selected from the db: %s", expectedAccountID, err))
 	}
 	if !acct.SuspendedAt.IsZero() {
-		return nil, gtserror.NewErrorNotFound(fmt.Errorf("account with id %s is suspended", form.AccountID))
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("account with id %s is suspended", expectedAccountID))
 	}
 
 	// make sure the requesting account and the media account don't block each other
 	if account != nil {
-		blocked, err := p.db.IsBlocked(ctx, account.ID, form.AccountID, true)
+		blocked, err := p.db.IsBlocked(ctx, account.ID, expectedAccountID, true)
 		if err != nil {
-			return nil, gtserror.NewErrorNotFound(fmt.Errorf("block status could not be established between accounts %s and %s: %s", form.AccountID, account.ID, err))
+			return nil, gtserror.NewErrorNotFound(fmt.Errorf("block status could not be established between accounts %s and %s: %s", expectedAccountID, account.ID, err))
 		}
 		if blocked {
-			return nil, gtserror.NewErrorNotFound(fmt.Errorf("block exists between accounts %s and %s", form.AccountID, account.ID))
+			return nil, gtserror.NewErrorNotFound(fmt.Errorf("block exists between accounts %s and %s", expectedAccountID, account.ID))
 		}
 	}
 
 	// the way we store emojis is a little different from the way we store other attachments,
 	// so we need to take different steps depending on the media type being requested
-	content := &apimodel.Content{}
-	var storagePath string
 	switch mediaType {
 	case media.TypeEmoji:
-		e := &gtsmodel.Emoji{}
+		return p.getEmojiContent(ctx, wantedMediaID, mediaSize)
-		if err := p.db.GetByID(ctx, wantedMediaID, e); err != nil {
-			return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s could not be taken from the db: %s", wantedMediaID, err))
-		}
-		if e.Disabled {
-			return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s has been disabled", wantedMediaID))
-		}
-		switch mediaSize {
-		case media.SizeOriginal:
-			content.ContentType = e.ImageContentType
-			content.ContentLength = int64(e.ImageFileSize)
-			storagePath = e.ImagePath
-		case media.SizeStatic:
-			content.ContentType = e.ImageStaticContentType
-			content.ContentLength = int64(e.ImageStaticFileSize)
-			storagePath = e.ImageStaticPath
-		default:
-			return nil, gtserror.NewErrorNotFound(fmt.Errorf("media size %s not recognized for emoji", mediaSize))
-		}
 	case media.TypeAttachment, media.TypeHeader, media.TypeAvatar:
+		return p.getAttachmentContent(ctx, account, wantedMediaID, expectedAccountID, mediaSize)
+	default:
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("media type %s not recognized", mediaType))
+	}
+}
+
+func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount *gtsmodel.Account, wantedMediaID string, expectedAccountID string, mediaSize media.Size) (*apimodel.Content, gtserror.WithCode) {
+	attachmentContent := &apimodel.Content{}
+	var storagePath string
+
+	// retrieve attachment from the database and do basic checks on it
 	a, err := p.db.GetAttachmentByID(ctx, wantedMediaID)
 	if err != nil {
 		return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s could not be taken from the db: %s", wantedMediaID, err))
 	}
-		if a.AccountID != form.AccountID {
+
-			return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s is not owned by %s", wantedMediaID, form.AccountID))
+	if a.AccountID != expectedAccountID {
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s is not owned by %s", wantedMediaID, expectedAccountID))
 	}
+
+	// get file information from the attachment depending on the requested media size
 	switch mediaSize {
 	case media.SizeOriginal:
-			content.ContentType = a.File.ContentType
+		attachmentContent.ContentType = a.File.ContentType
-			content.ContentLength = int64(a.File.FileSize)
+		attachmentContent.ContentLength = int64(a.File.FileSize)
 		storagePath = a.File.Path
 	case media.SizeSmall:
-			content.ContentType = a.Thumbnail.ContentType
+		attachmentContent.ContentType = a.Thumbnail.ContentType
-			content.ContentLength = int64(a.Thumbnail.FileSize)
+		attachmentContent.ContentLength = int64(a.Thumbnail.FileSize)
 		storagePath = a.Thumbnail.Path
 	default:
 		return nil, gtserror.NewErrorNotFound(fmt.Errorf("media size %s not recognized for attachment", mediaSize))
 	}
+
+	// if we have the media cached on our server already, we can now simply return it from storage
+	if a.Cached {
+		return p.streamFromStorage(storagePath, attachmentContent)
 	}
 
+	// if we don't have it cached, then we can assume two things:
+	// 1. this is remote media, since local media should never be uncached
+	// 2. we need to fetch it again using a transport and the media manager
+	remoteMediaIRI, err := url.Parse(a.RemoteURL)
+	if err != nil {
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("error parsing remote media iri %s: %s", a.RemoteURL, err))
+	}
+
+	// use an empty string as requestingUsername to use the instance account, unless the request for this
+	// media has been http signed, then use the requesting account to make the request to remote server
+	var requestingUsername string
+	if requestingAccount != nil {
+		requestingUsername = requestingAccount.Username
+	}
+
+	var data media.DataFunc
+	var postDataCallback media.PostDataCallbackFunc
+
+	if mediaSize == media.SizeSmall {
+		// if it's the thumbnail that's requested then the user will have to wait a bit while we process the
+		// large version and derive a thumbnail from it, so use the normal recaching procedure: fetch the media,
+		// process it, then return the thumbnail data
+		data = func(innerCtx context.Context) (io.Reader, int, error) {
+			transport, err := p.transportController.NewTransportForUsername(innerCtx, requestingUsername)
+			if err != nil {
+				return nil, 0, err
+			}
+			return transport.DereferenceMedia(innerCtx, remoteMediaIRI)
+		}
+	} else {
+		// if it's the full-sized version being requested, we can cheat a bit by streaming data to the user as
+		// it's retrieved from the remote server, using tee; this saves the user from having to wait while
+		// we process the media on our side
+		//
+		// this looks a bit like this:
+		//
+		//                http fetch                   buffered pipe
+		// remote server ------------> data function ----------------> api caller
+		//                                   |
+		//                                   | tee
+		//                                   |
+		//                                   ▼
+		//                            instance storage
+
+		// Buffer each end of the pipe, so that if the caller drops the connection during the flow, the tee
+		// reader can continue without having to worry about tee-ing into a closed or blocked pipe.
+		pipeReader, pipeWriter := io.Pipe()
+		bufferedWriter := bufio.NewWriterSize(pipeWriter, int(attachmentContent.ContentLength))
+		bufferedReader := bufio.NewReaderSize(pipeReader, int(attachmentContent.ContentLength))
+
+		// the caller will read from the buffered reader, so it doesn't matter if they drop out without reading everything
+		attachmentContent.Content = bufferedReader
+
+		data = func(innerCtx context.Context) (io.Reader, int, error) {
+			transport, err := p.transportController.NewTransportForUsername(innerCtx, requestingUsername)
+			if err != nil {
+				return nil, 0, err
+			}
+
+			readCloser, fileSize, err := transport.DereferenceMedia(innerCtx, remoteMediaIRI)
+			if err != nil {
+				return nil, 0, err
+			}
+
+			// everything read from the readCloser by the media manager will be written into the bufferedWriter
+			teeReader := io.TeeReader(readCloser, bufferedWriter)
+			return teeReader, fileSize, nil
+		}
+
+		// close the pipewriter after data has been piped into it, so the reader on the other side doesn't block;
+		// we don't need to close the reader here because that's the caller's responsibility
+		postDataCallback = func(innerCtx context.Context) error {
+			// flush the buffered writer into the buffer of the reader...
+			if err := bufferedWriter.Flush(); err != nil {
+				return err
+			}
+
+			// and close the underlying pipe writer
+			if err := pipeWriter.Close(); err != nil {
+				return err
+			}
+
+			return nil
+		}
+	}
+
+	// put the media recached in the queue
+	processingMedia, err := p.mediaManager.RecacheMedia(ctx, data, postDataCallback, wantedMediaID)
+	if err != nil {
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("error recaching media: %s", err))
+	}
+
+	// if it's the thumbnail, stream the processed thumbnail from storage, after waiting for processing to finish
+	if mediaSize == media.SizeSmall {
+		// below function call blocks until all processing on the attachment has finished...
+		if _, err := processingMedia.LoadAttachment(ctx); err != nil {
+			return nil, gtserror.NewErrorNotFound(fmt.Errorf("error loading recached attachment: %s", err))
+		}
+		// ... so now we can safely return it
+		return p.streamFromStorage(storagePath, attachmentContent)
+	}
+
+	return attachmentContent, nil
+}
+
+func (p *processor) getEmojiContent(ctx context.Context, wantedEmojiID string, emojiSize media.Size) (*apimodel.Content, gtserror.WithCode) {
+	emojiContent := &apimodel.Content{}
+	var storagePath string
+
+	e := &gtsmodel.Emoji{}
+	if err := p.db.GetByID(ctx, wantedEmojiID, e); err != nil {
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s could not be taken from the db: %s", wantedEmojiID, err))
+	}
+
+	if e.Disabled {
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s has been disabled", wantedEmojiID))
+	}
+
+	switch emojiSize {
+	case media.SizeOriginal:
+		emojiContent.ContentType = e.ImageContentType
+		emojiContent.ContentLength = int64(e.ImageFileSize)
+		storagePath = e.ImagePath
+	case media.SizeStatic:
+		emojiContent.ContentType = e.ImageStaticContentType
+		emojiContent.ContentLength = int64(e.ImageStaticFileSize)
+		storagePath = e.ImageStaticPath
+	default:
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("media size %s not recognized for emoji", emojiSize))
+	}
+
+	return p.streamFromStorage(storagePath, emojiContent)
+}
+
+func (p *processor) streamFromStorage(storagePath string, content *apimodel.Content) (*apimodel.Content, gtserror.WithCode) {
 	reader, err := p.storage.GetStream(storagePath)
 	if err != nil {
 		return nil, gtserror.NewErrorNotFound(fmt.Errorf("error retrieving from storage: %s", err))

internal/processing/media/getfile_test.go

@@ -0,0 +1,208 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package media_test
+
+import (
+	"context"
+	"io"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/suite"
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+)
+
+type GetFileTestSuite struct {
+	MediaStandardTestSuite
+}
+
+func (suite *GetFileTestSuite) TestGetRemoteFileCached() {
+	ctx := context.Background()
+
+	testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
+	fileName := path.Base(testAttachment.File.Path)
+	requestingAccount := suite.testAccounts["local_account_1"]
+
+	content, errWithCode := suite.mediaProcessor.GetFile(ctx, requestingAccount, &apimodel.GetContentRequestForm{
+		AccountID: testAttachment.AccountID,
+		MediaType: string(media.TypeAttachment),
+		MediaSize: string(media.SizeOriginal),
+		FileName:  fileName,
+	})
+
+	suite.NoError(errWithCode)
+	suite.NotNil(content)
+	b, err := io.ReadAll(content.Content)
+	suite.NoError(err)
+
+	if closer, ok := content.Content.(io.Closer); ok {
+		suite.NoError(closer.Close())
+	}
+
+	suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, b)
+	suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].ContentType, content.ContentType)
+	suite.EqualValues(len(suite.testRemoteAttachments[testAttachment.RemoteURL].Data), content.ContentLength)
+}
+
+func (suite *GetFileTestSuite) TestGetRemoteFileUncached() {
+	ctx := context.Background()
+
+	// uncache the file from local
+	testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
+	testAttachment.Cached = false
+	err := suite.db.UpdateByPrimaryKey(ctx, testAttachment)
+	suite.NoError(err)
+	err = suite.storage.Delete(testAttachment.File.Path)
+	suite.NoError(err)
+	err = suite.storage.Delete(testAttachment.Thumbnail.Path)
+	suite.NoError(err)
+
+	// now fetch it
+	fileName := path.Base(testAttachment.File.Path)
+	requestingAccount := suite.testAccounts["local_account_1"]
+
+	content, errWithCode := suite.mediaProcessor.GetFile(ctx, requestingAccount, &apimodel.GetContentRequestForm{
+		AccountID: testAttachment.AccountID,
+		MediaType: string(media.TypeAttachment),
+		MediaSize: string(media.SizeOriginal),
+		FileName:  fileName,
+	})
+
+	suite.NoError(errWithCode)
+	suite.NotNil(content)
+	b, err := io.ReadAll(content.Content)
+	suite.NoError(err)
+
+	if closer, ok := content.Content.(io.Closer); ok {
+		suite.NoError(closer.Close())
+	}
+
+	suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, b)
+	suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].ContentType, content.ContentType)
+	suite.EqualValues(len(suite.testRemoteAttachments[testAttachment.RemoteURL].Data), content.ContentLength)
+	time.Sleep(2 * time.Second) // wait a few seconds for the media manager to finish doing stuff
+
+	// the attachment should be updated in the database
+	dbAttachment, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID)
+	suite.NoError(err)
+	suite.True(dbAttachment.Cached)
+
+	// the file should be back in storage at the same path as before
+	refreshedBytes, err := suite.storage.Get(testAttachment.File.Path)
+	suite.NoError(err)
+	suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, refreshedBytes)
+}
+
+func (suite *GetFileTestSuite) TestGetRemoteFileUncachedInterrupted() {
+	ctx := context.Background()
+
+	// uncache the file from local
+	testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
+	testAttachment.Cached = false
+	err := suite.db.UpdateByPrimaryKey(ctx, testAttachment)
+	suite.NoError(err)
+	err = suite.storage.Delete(testAttachment.File.Path)
+	suite.NoError(err)
+	err = suite.storage.Delete(testAttachment.Thumbnail.Path)
+	suite.NoError(err)
+
+	// now fetch it
+	fileName := path.Base(testAttachment.File.Path)
+	requestingAccount := suite.testAccounts["local_account_1"]
+
+	content, errWithCode := suite.mediaProcessor.GetFile(ctx, requestingAccount, &apimodel.GetContentRequestForm{
+		AccountID: testAttachment.AccountID,
+		MediaType: string(media.TypeAttachment),
+		MediaSize: string(media.SizeOriginal),
+		FileName:  fileName,
+	})
+
+	suite.NoError(errWithCode)
+	suite.NotNil(content)
+
+	// only read the first kilobyte and then stop
+	b := make([]byte, 1024)
+	_, err = content.Content.Read(b)
+	suite.NoError(err)
+
+	// close the reader
+	if closer, ok := content.Content.(io.Closer); ok {
+		suite.NoError(closer.Close())
+	}
+
+	time.Sleep(2 * time.Second) // wait a few seconds for the media manager to finish doing stuff
+
+	// the attachment should still be updated in the database even though the caller hung up
+	dbAttachment, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID)
+	suite.NoError(err)
+	suite.True(dbAttachment.Cached)
+
+	// the file should be back in storage at the same path as before
+	refreshedBytes, err := suite.storage.Get(testAttachment.File.Path)
+	suite.NoError(err)
+	suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, refreshedBytes)
+}
+
+func (suite *GetFileTestSuite) TestGetRemoteFileThumbnailUncached() {
+	ctx := context.Background()
+	testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
+
+	// fetch the existing thumbnail bytes from storage first
+	thumbnailBytes, err := suite.storage.Get(testAttachment.Thumbnail.Path)
+	suite.NoError(err)
+
+	// uncache the file from local
+	testAttachment.Cached = false
+	err = suite.db.UpdateByPrimaryKey(ctx, testAttachment)
+	suite.NoError(err)
+	err = suite.storage.Delete(testAttachment.File.Path)
+	suite.NoError(err)
+	err = suite.storage.Delete(testAttachment.Thumbnail.Path)
+	suite.NoError(err)
+
+	// now fetch the thumbnail
+	fileName := path.Base(testAttachment.File.Path)
+	requestingAccount := suite.testAccounts["local_account_1"]
+
+	content, errWithCode := suite.mediaProcessor.GetFile(ctx, requestingAccount, &apimodel.GetContentRequestForm{
+		AccountID: testAttachment.AccountID,
+		MediaType: string(media.TypeAttachment),
+		MediaSize: string(media.SizeSmall),
+		FileName:  fileName,
+	})
+
+	suite.NoError(errWithCode)
+	suite.NotNil(content)
+	b, err := io.ReadAll(content.Content)
+	suite.NoError(err)
+
+	if closer, ok := content.Content.(io.Closer); ok {
+		suite.NoError(closer.Close())
+	}
+
+	suite.Equal(thumbnailBytes, b)
+	suite.Equal("image/jpeg", content.ContentType)
+	suite.EqualValues(testAttachment.Thumbnail.FileSize, content.ContentLength)
+}
+
+func TestGetFileTestSuite(t *testing.T) {
+	suite.Run(t, &GetFileTestSuite{})
+}

internal/processing/media/media.go

@@ -27,6 +27,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
 
@@ -36,7 +37,8 @@ type Processor interface {
 	Create(ctx context.Context, account *gtsmodel.Account, form *apimodel.AttachmentRequest) (*apimodel.Attachment, error)
 	// Delete deletes the media attachment with the given ID, including all files pertaining to that attachment.
 	Delete(ctx context.Context, mediaAttachmentID string) gtserror.WithCode
-	GetFile(ctx context.Context, account *gtsmodel.Account, form *apimodel.GetContentRequestForm) (*apimodel.Content, error)
+	// GetFile retrieves a file from storage and streams it back to the caller via an io.reader embedded in *apimodel.Content.
+	GetFile(ctx context.Context, account *gtsmodel.Account, form *apimodel.GetContentRequestForm) (*apimodel.Content, gtserror.WithCode)
 	GetMedia(ctx context.Context, account *gtsmodel.Account, mediaAttachmentID string) (*apimodel.Attachment, gtserror.WithCode)
 	Update(ctx context.Context, account *gtsmodel.Account, mediaAttachmentID string, form *apimodel.AttachmentUpdateRequest) (*apimodel.Attachment, gtserror.WithCode)
 }
@@ -44,15 +46,17 @@ type Processor interface {
 type processor struct {
 	tc                  typeutils.TypeConverter
 	mediaManager        media.Manager
+	transportController transport.Controller
 	storage             *kv.KVStore
 	db                  db.DB
 }
 
 // New returns a new media processor.
-func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, storage *kv.KVStore) Processor {
+func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, transportController transport.Controller, storage *kv.KVStore) Processor {
 	return &processor{
 		tc:                  tc,
 		mediaManager:        mediaManager,
+		transportController: transportController,
 		storage:             storage,
 		db:                  db,
 	}

internal/processing/media/media_test.go

@@ -0,0 +1,125 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package media_test
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+
+	"codeberg.org/gruf/go-store/kv"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+	mediaprocessing "github.com/superseriousbusiness/gotosocial/internal/processing/media"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
+	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+type MediaStandardTestSuite struct {
+	// standard suite interfaces
+	suite.Suite
+	db                  db.DB
+	tc                  typeutils.TypeConverter
+	storage             *kv.KVStore
+	mediaManager        media.Manager
+	transportController transport.Controller
+
+	// standard suite models
+	testTokens            map[string]*gtsmodel.Token
+	testClients           map[string]*gtsmodel.Client
+	testApplications      map[string]*gtsmodel.Application
+	testUsers             map[string]*gtsmodel.User
+	testAccounts          map[string]*gtsmodel.Account
+	testAttachments       map[string]*gtsmodel.MediaAttachment
+	testStatuses          map[string]*gtsmodel.Status
+	testRemoteAttachments map[string]testrig.RemoteAttachmentFile
+
+	// module being tested
+	mediaProcessor mediaprocessing.Processor
+}
+
+func (suite *MediaStandardTestSuite) SetupSuite() {
+	suite.testTokens = testrig.NewTestTokens()
+	suite.testClients = testrig.NewTestClients()
+	suite.testApplications = testrig.NewTestApplications()
+	suite.testUsers = testrig.NewTestUsers()
+	suite.testAccounts = testrig.NewTestAccounts()
+	suite.testAttachments = testrig.NewTestAttachments()
+	suite.testStatuses = testrig.NewTestStatuses()
+	suite.testRemoteAttachments = testrig.NewTestFediAttachments("../../../testrig/media")
+}
+
+func (suite *MediaStandardTestSuite) SetupTest() {
+	testrig.InitTestConfig()
+	testrig.InitTestLog()
+
+	suite.db = testrig.NewTestDB()
+	suite.tc = testrig.NewTestTypeConverter(suite.db)
+	suite.storage = testrig.NewTestStorage()
+	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
+	suite.transportController = suite.mockTransportController()
+	suite.mediaProcessor = mediaprocessing.New(suite.db, suite.tc, suite.mediaManager, suite.transportController, suite.storage)
+	testrig.StandardDBSetup(suite.db, nil)
+	testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")
+}
+
+func (suite *MediaStandardTestSuite) TearDownTest() {
+	testrig.StandardDBTeardown(suite.db)
+	testrig.StandardStorageTeardown(suite.storage)
+}
+
+func (suite *MediaStandardTestSuite) mockTransportController() transport.Controller {
+	do := func(req *http.Request) (*http.Response, error) {
+		logrus.Debugf("received request for %s", req.URL)
+
+		responseBytes := []byte{}
+		responseType := ""
+		responseLength := 0
+
+		if attachment, ok := suite.testRemoteAttachments[req.URL.String()]; ok {
+			responseBytes = attachment.Data
+			responseType = attachment.ContentType
+		}
+
+		if len(responseBytes) != 0 {
+			// we found something, so print what we're going to return
+			logrus.Debugf("returning response %s", string(responseBytes))
+		}
+		responseLength = len(responseBytes)
+
+		reader := bytes.NewReader(responseBytes)
+		readCloser := io.NopCloser(reader)
+		response := &http.Response{
+			StatusCode:    200,
+			Body:          readCloser,
+			ContentLength: int64(responseLength),
+			Header: http.Header{
+				"content-type": {responseType},
+			},
+		}
+
+		return response, nil
+	}
+	mockClient := testrig.NewMockHTTPClient(do)
+	return testrig.NewTestTransportController(mockClient, suite.db)
+}

internal/processing/processor.go

@@ -115,7 +115,7 @@ type Processor interface {
 	BlocksGet(ctx context.Context, authed *oauth.Auth, maxID string, sinceID string, limit int) (*apimodel.BlocksResponse, gtserror.WithCode)
 
 	// FileGet handles the fetching of a media attachment file via the fileserver.
-	FileGet(ctx context.Context, authed *oauth.Auth, form *apimodel.GetContentRequestForm) (*apimodel.Content, error)
+	FileGet(ctx context.Context, authed *oauth.Auth, form *apimodel.GetContentRequestForm) (*apimodel.Content, gtserror.WithCode)
 
 	// FollowRequestsGet handles the getting of the authed account's incoming follow requests
 	FollowRequestsGet(ctx context.Context, auth *oauth.Auth) ([]apimodel.Account, gtserror.WithCode)
@@ -270,7 +270,7 @@ func NewProcessor(
 	streamingProcessor := streaming.New(db, oauthServer)
 	accountProcessor := account.New(db, tc, mediaManager, oauthServer, fromClientAPI, federator)
 	adminProcessor := admin.New(db, tc, mediaManager, fromClientAPI)
-	mediaProcessor := mediaProcessor.New(db, tc, mediaManager, storage)
+	mediaProcessor := mediaProcessor.New(db, tc, mediaManager, federator.TransportController(), storage)
 	userProcessor := user.New(db, emailSender)
 	federationProcessor := federationProcessor.New(db, tc, federator, fromFederator)
 	filter := visibility.NewFilter(db)

internal/processing/streaming/notification_test.go

@@ -52,7 +52,7 @@ func (suite *NotificationTestSuite) TestStreamNotification() {
 	suite.NoError(err)
 
 	msg := <-openStream.Messages
-	suite.Equal(`{"id":"01FH57SJCMDWQGEAJ0X08CE3WV","type":"follow","created_at":"2021-10-04T10:52:36+02:00","account":{"id":"01F8MH5ZK5VRH73AKHQM6Y9VNX","username":"foss_satan","acct":"foss_satan@fossbros-anonymous.io","display_name":"big gerald","locked":false,"bot":false,"created_at":"2021-09-26T12:52:36+02:00","note":"i post about like, i dunno, stuff, or whatever!!!!","url":"http://fossbros-anonymous.io/@foss_satan","avatar":"","avatar_static":"","header":"","header_static":"","followers_count":0,"following_count":0,"statuses_count":0,"last_status_at":"","emojis":[],"fields":[]}}`, msg.Payload)
+	suite.Equal(`{"id":"01FH57SJCMDWQGEAJ0X08CE3WV","type":"follow","created_at":"2021-10-04T10:52:36+02:00","account":{"id":"01F8MH5ZK5VRH73AKHQM6Y9VNX","username":"foss_satan","acct":"foss_satan@fossbros-anonymous.io","display_name":"big gerald","locked":false,"bot":false,"created_at":"2021-09-26T12:52:36+02:00","note":"i post about like, i dunno, stuff, or whatever!!!!","url":"http://fossbros-anonymous.io/@foss_satan","avatar":"","avatar_static":"","header":"","header_static":"","followers_count":0,"following_count":0,"statuses_count":1,"last_status_at":"2021-09-20T10:40:37Z","emojis":[],"fields":[]}}`, msg.Payload)
 }
 
 func TestNotificationTestSuite(t *testing.T) {

internal/timeline/get_test.go

@@ -84,8 +84,8 @@ func (suite *GetTestSuite) TestGetDefault() {
 		suite.FailNow(err.Error())
 	}
 
-	// we only have 14 statuses in the test suite
+	// we only have 15 statuses in the test suite
-	suite.Len(statuses, 14)
+	suite.Len(statuses, 15)
 
 	// statuses should be sorted highest to lowest ID
 	var highest string
@@ -177,8 +177,8 @@ func (suite *GetTestSuite) TestGetMinID() {
 		suite.FailNow(err.Error())
 	}
 
-	// we should only get 7 statuses back, since we asked for a min ID that excludes some of our entries
+	// we should only get 8 statuses back, since we asked for a min ID that excludes some of our entries
-	suite.Len(statuses, 7)
+	suite.Len(statuses, 8)
 
 	// statuses should be sorted highest to lowest ID
 	var highest string
@@ -199,8 +199,8 @@ func (suite *GetTestSuite) TestGetSinceID() {
 		suite.FailNow(err.Error())
 	}
 
-	// we should only get 7 statuses back, since we asked for a since ID that excludes some of our entries
+	// we should only get 8 statuses back, since we asked for a since ID that excludes some of our entries
-	suite.Len(statuses, 7)
+	suite.Len(statuses, 8)
 
 	// statuses should be sorted highest to lowest ID
 	var highest string
@@ -221,8 +221,8 @@ func (suite *GetTestSuite) TestGetSinceIDPrepareNext() {
 		suite.FailNow(err.Error())
 	}
 
-	// we should only get 7 statuses back, since we asked for a since ID that excludes some of our entries
+	// we should only get 8 statuses back, since we asked for a since ID that excludes some of our entries
-	suite.Len(statuses, 7)
+	suite.Len(statuses, 8)
 
 	// statuses should be sorted highest to lowest ID
 	var highest string

test/cliparsing.sh

@@ -5,7 +5,7 @@ set -e
 echo "STARTING CLI TESTS"
 
 echo "TEST_1 Make sure defaults are set correctly."
-TEST_1_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","help":false,"host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_1_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","help":false,"host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_1="$(go run ./cmd/gotosocial/... debug config)"
 if [ "${TEST_1}" != "${TEST_1_EXPECTED}" ]; then
     echo "TEST_1 not equal TEST_1_EXPECTED"
@@ -15,7 +15,7 @@ else
 fi
 
 echo "TEST_2 Override db-address from default using cli flag."
-TEST_2_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","help":false,"host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_2_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","help":false,"host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_2="$(go run ./cmd/gotosocial/... --db-address some.db.address debug config)"
 if [ "${TEST_2}" != "${TEST_2_EXPECTED}" ]; then
     echo "TEST_2 not equal TEST_2_EXPECTED"
@@ -25,7 +25,7 @@ else
 fi
 
 echo "TEST_3 Override db-address from default using env var."
-TEST_3_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","help":false,"host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_3_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","help":false,"host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_3="$(GTS_DB_ADDRESS=some.db.address go run ./cmd/gotosocial/... debug config)"
 if [ "${TEST_3}" != "${TEST_3_EXPECTED}" ]; then
     echo "TEST_3 not equal TEST_3_EXPECTED"
@@ -35,7 +35,7 @@ else
 fi
 
 echo "TEST_4 Override db-address from default using both env var and cli flag. The cli flag should take priority."
-TEST_4_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.other.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","help":false,"host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_4_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.other.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","help":false,"host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_4="$(GTS_DB_ADDRESS=some.db.address go run ./cmd/gotosocial/... --db-address some.other.db.address debug config)"
 if [ "${TEST_4}" != "${TEST_4_EXPECTED}" ]; then
     echo "TEST_4 not equal TEST_4_EXPECTED"
@@ -45,7 +45,7 @@ else
 fi
 
 echo "TEST_5 Test loading a config file by passing an env var."
-TEST_5_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_5_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_5="$(GTS_CONFIG_PATH=./test/test.yaml go run ./cmd/gotosocial/... debug config)"
 if [ "${TEST_5}" != "${TEST_5_EXPECTED}" ]; then
     echo "TEST_5 not equal TEST_5_EXPECTED"
@@ -55,7 +55,7 @@ else
 fi
 
 echo "TEST_6 Test loading a config file by passing cli flag."
-TEST_6_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_6_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_6="$(go run ./cmd/gotosocial/... --config-path ./test/test.yaml debug config)"
 if [ "${TEST_6}" != "${TEST_6_EXPECTED}" ]; then
     echo "TEST_6 not equal TEST_6_EXPECTED"
@@ -65,7 +65,7 @@ else
 fi
 
 echo "TEST_7 Test loading a config file and overriding one of the variables with a cli flag."
-TEST_7_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_7_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_7="$(go run ./cmd/gotosocial/... --config-path ./test/test.yaml --account-domain '' debug config)"
 if [ "${TEST_7}" != "${TEST_7_EXPECTED}" ]; then
     echo "TEST_7 not equal TEST_7_EXPECTED"
@@ -75,7 +75,7 @@ else
 fi
 
 echo "TEST_8 Test loading a config file and overriding one of the variables with an env var."
-TEST_8_EXPECTED='{"account-domain":"peepee","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_8_EXPECTED='{"account-domain":"peepee","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_8="$(GTS_ACCOUNT_DOMAIN='peepee' go run ./cmd/gotosocial/... --config-path ./test/test.yaml debug config)"
 if [ "${TEST_8}" != "${TEST_8_EXPECTED}" ]; then
     echo "TEST_8 not equal TEST_8_EXPECTED"
@@ -85,7 +85,7 @@ else
 fi
 
 echo "TEST_9 Test loading a config file and overriding one of the variables with both an env var and a cli flag. The cli flag should have priority."
-TEST_9_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_9_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_9="$(GTS_ACCOUNT_DOMAIN='peepee' go run ./cmd/gotosocial/... --config-path ./test/test.yaml --account-domain '' debug config)"
 if [ "${TEST_9}" != "${TEST_9_EXPECTED}" ]; then
     echo "TEST_9 not equal TEST_9_EXPECTED"
@@ -95,7 +95,7 @@ else
 fi
 
 echo "TEST_10 Test loading a config file from json."
-TEST_10_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.json","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_10_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.json","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_10="$(go run ./cmd/gotosocial/... --config-path ./test/test.json debug config)"
 if [ "${TEST_10}" != "${TEST_10_EXPECTED}" ]; then
     echo "TEST_10 not equal TEST_10_EXPECTED"
@@ -105,7 +105,7 @@ else
 fi
 
 echo "TEST_11 Test loading a partial config file. Default values should be used apart from those set in the config file."
-TEST_11_EXPECTED='{"account-domain":"peepee.poopoo","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test2.yaml","db-address":"","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","help":false,"host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-level":"trace","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
+TEST_11_EXPECTED='{"account-domain":"peepee.poopoo","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test2.yaml","db-address":"","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","help":false,"host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-level":"trace","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
 TEST_11="$(go run ./cmd/gotosocial/... --config-path ./test/test2.yaml debug config)"
 if [ "${TEST_11}" != "${TEST_11_EXPECTED}" ]; then
     echo "TEST_11 not equal TEST_11_EXPECTED"

testrig/config.go

@@ -86,6 +86,7 @@ var TestDefaults = config.Values{
 	MediaVideoMaxSize:        5242880, // 5mb
 	MediaDescriptionMinChars: 0,
 	MediaDescriptionMaxChars: 500,
+	MediaRemoteCacheDays:     30,
 
 	StorageBackend:       "local",
 	StorageLocalBasePath: "/gotosocial/storage",

testrig/testmodels.go

@@ -567,13 +567,13 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 			Blurhash:          "LNJRdVM{00Rj%Mayt7j[4nWBofRj",
 			Processing:        2,
 			File: gtsmodel.File{
-				Path:        "/gotosocial/storage/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg",
+				Path:        "01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg",
 				ContentType: "image/jpeg",
 				FileSize:    62529,
 				UpdatedAt:   time.Now().Add(-71 * time.Hour),
 			},
 			Thumbnail: gtsmodel.Thumbnail{
-				Path:        "/gotosocial/storage/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpeg",
+				Path:        "01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpeg",
 				ContentType: "image/jpeg",
 				FileSize:    6872,
 				UpdatedAt:   time.Now().Add(-71 * time.Hour),
@@ -582,6 +582,7 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 			},
 			Avatar: false,
 			Header: false,
+			Cached: true,
 		},
 		"local_account_1_status_4_attachment_1": {
 			ID:        "01F8MH7TDVANYKWVE8VVKFPJTJ",
@@ -615,13 +616,13 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 			Blurhash:          "LEDara58O=t5EMSOENEN9]}?aK%0",
 			Processing:        2,
 			File: gtsmodel.File{
-				Path:        "/gotosocial/storage/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH7TDVANYKWVE8VVKFPJTJ.gif",
+				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH7TDVANYKWVE8VVKFPJTJ.gif",
 				ContentType: "image/gif",
 				FileSize:    1109138,
 				UpdatedAt:   time.Now().Add(-1 * time.Hour),
 			},
 			Thumbnail: gtsmodel.Thumbnail{
-				Path:        "/gotosocial/storage/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH7TDVANYKWVE8VVKFPJTJ.jpeg",
+				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH7TDVANYKWVE8VVKFPJTJ.jpeg",
 				ContentType: "image/jpeg",
 				FileSize:    8803,
 				UpdatedAt:   time.Now().Add(-1 * time.Hour),
@@ -630,6 +631,7 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 			},
 			Avatar: false,
 			Header: false,
+			Cached: true,
 		},
 		"local_account_1_unattached_1": {
 			ID:        "01F8MH8RMYQ6MSNY3JM2XT1CQ5",
@@ -663,13 +665,13 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 			Blurhash:          "LSAd]9ogDge-R:M|j=xWIto0xXWX",
 			Processing:        2,
 			File: gtsmodel.File{
-				Path:        "/gotosocial/storage/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpeg",
+				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpeg",
 				ContentType: "image/jpeg",
 				FileSize:    27759,
 				UpdatedAt:   time.Now().Add(30 * time.Second),
 			},
 			Thumbnail: gtsmodel.Thumbnail{
-				Path:        "/gotosocial/storage/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpeg",
+				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpeg",
 				ContentType: "image/jpeg",
 				FileSize:    6177,
 				UpdatedAt:   time.Now().Add(30 * time.Second),
@@ -678,6 +680,7 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 			},
 			Avatar: false,
 			Header: false,
+			Cached: true,
 		},
 		"local_account_1_avatar": {
 			ID:        "01F8MH58A357CV5K7R7TJMSH6S",
@@ -711,13 +714,13 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 			Blurhash:          "LKK9MT,p|YSNDkJ-5rsmvnwcOoe:",
 			Processing:        2,
 			File: gtsmodel.File{
-				Path:        "/gotosocial/storage/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg",
+				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg",
 				ContentType: "image/jpeg",
 				FileSize:    457680,
 				UpdatedAt:   time.Now().Add(-47 * time.Hour),
 			},
 			Thumbnail: gtsmodel.Thumbnail{
-				Path:        "/gotosocial/storage/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg",
+				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg",
 				ContentType: "image/jpeg",
 				FileSize:    15374,
 				UpdatedAt:   time.Now().Add(-47 * time.Hour),
@@ -726,6 +729,7 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 			},
 			Avatar: true,
 			Header: false,
+			Cached: true,
 		},
 		"local_account_1_header": {
 			ID:        "01PFPMWK2FF0D9WMHEJHR07C3Q",
@@ -759,13 +763,13 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 			Blurhash:          "L26j{^WCs+R-N}jsxWj@4;WWxDoK",
 			Processing:        2,
 			File: gtsmodel.File{
-				Path:        "/gotosocial/storage/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg",
+				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg",
 				ContentType: "image/jpeg",
 				FileSize:    517226,
 				UpdatedAt:   time.Now().Add(-47 * time.Hour),
 			},
 			Thumbnail: gtsmodel.Thumbnail{
-				Path:        "/gotosocial/storage/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg",
+				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg",
 				ContentType: "image/jpeg",
 				FileSize:    42308,
 				UpdatedAt:   time.Now().Add(-47 * time.Hour),
@@ -774,6 +778,56 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 			},
 			Avatar: false,
 			Header: true,
+			Cached: true,
+		},
+		"remote_account_1_status_1_attachment_1": {
+			ID:        "01FVW7RXPQ8YJHTEXYPE7Q8ZY0",
+			StatusID:  "01FVW7JHQFSFK166WWKR8CBA6M",
+			URL:       "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
+			RemoteURL: "http://fossbros-anonymous.io/attachments/original/13bbc3f8-2b5e-46ea-9531-40b4974d9912.jpeg",
+			CreatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"),
+			UpdatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"),
+			Type:      gtsmodel.FileTypeImage,
+			FileMeta: gtsmodel.FileMeta{
+				Original: gtsmodel.Original{
+					Width:  472,
+					Height: 291,
+					Size:   137352,
+					Aspect: 1.6219931271477663,
+				},
+				Small: gtsmodel.Small{
+					Width:  472,
+					Height: 291,
+					Size:   137352,
+					Aspect: 1.6219931271477663,
+				},
+				Focus: gtsmodel.Focus{
+					X: 0,
+					Y: 0,
+				},
+			},
+			AccountID:         "01F8MH1H7YV1Z7D2C8K2730QBF",
+			Description:       "tweet from thoughts of dog: i drank. all the water. in my bowl. earlier. but just now. i returned. to the same bowl. and it was. full again.. the bowl. is haunted",
+			ScheduledStatusID: "",
+			Blurhash:          "LARysgM_IU_3~pD%M_Rj_39FIAt6",
+			Processing:        2,
+			File: gtsmodel.File{
+				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
+				ContentType: "image/jpeg",
+				FileSize:    19310,
+				UpdatedAt:   TimeMustParse("2021-09-20T12:40:37+02:00"),
+			},
+			Thumbnail: gtsmodel.Thumbnail{
+				Path:        "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
+				ContentType: "image/jpeg",
+				FileSize:    20395,
+				UpdatedAt:   TimeMustParse("2021-09-20T12:40:37+02:00"),
+				URL:         "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
+				RemoteURL:   "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpeg",
+			},
+			Avatar: false,
+			Header: false,
+			Cached: true,
 		},
 	}
 }
@@ -848,6 +902,10 @@ func newTestStoredAttachments() map[string]filenames {
 			Original: "team-fortress-original.jpeg",
 			Small:    "team-fortress-small.jpeg",
 		},
+		"remote_account_1_status_1_attachment_1": {
+			Original: "thoughtsofdog-original.jpeg",
+			Small:    "thoughtsofdog-small.jpeg",
+		},
 	}
 }
 
@@ -1202,6 +1260,33 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Likeable:                 true,
 			ActivityStreamsType:      ap.ObjectNote,
 		},
+		"remote_account_1_status_1": {
+			ID:                       "01FVW7JHQFSFK166WWKR8CBA6M",
+			URI:                      "http://fossbros-anonymous.io/users/foss_satan/statuses/01FVW7JHQFSFK166WWKR8CBA6M",
+			URL:                      "http://fossbros-anonymous.io/@foss_satan/statuses/01FVW7JHQFSFK166WWKR8CBA6M",
+			Content:                  "dark souls status bot: \"thoughts of dog\"",
+			AttachmentIDs:            []string{"01FVW7RXPQ8YJHTEXYPE7Q8ZY0"},
+			CreatedAt:                TimeMustParse("2021-09-20T12:40:37+02:00"),
+			UpdatedAt:                TimeMustParse("2021-09-20T12:40:37+02:00"),
+			Local:                    false,
+			AccountURI:               "http://fossbros-anonymous.io/users/foss_satan",
+			MentionIDs:               []string{},
+			AccountID:                "01F8MH5ZK5VRH73AKHQM6Y9VNX",
+			InReplyToID:              "",
+			InReplyToAccountID:       "",
+			InReplyToURI:             "",
+			BoostOfID:                "",
+			ContentWarning:           "",
+			Visibility:               gtsmodel.VisibilityUnlocked,
+			Sensitive:                false,
+			Language:                 "en",
+			CreatedWithApplicationID: "",
+			Federated:                true,
+			Boostable:                true,
+			Replyable:                true,
+			Likeable:                 true,
+			ActivityStreamsType:      ap.ObjectNote,
+		},
 	}
 }
 
@@ -1501,11 +1586,21 @@ func NewTestFediAttachments(relativePath string) map[string]RemoteAttachmentFile
 	if err != nil {
 		panic(err)
 	}
+
+	thoughtsOfDogBytes, err := os.ReadFile(fmt.Sprintf("%s/thoughtsofdog-original.jpeg", relativePath))
+	if err != nil {
+		panic(err)
+	}
+
 	return map[string]RemoteAttachmentFile{
 		"https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg": {
 			Data:        beeBytes,
 			ContentType: "image/jpeg",
 		},
+		"http://fossbros-anonymous.io/attachments/original/13bbc3f8-2b5e-46ea-9531-40b4974d9912.jpeg": {
+			Data:        thoughtsOfDogBytes,
+			ContentType: "image/jpeg",
+		},
 	}
 }
 

vendor/github.com/jackc/pgconn/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 1.11.0 (February 7, 2022)
+
+* Support port in ip from LookupFunc to override config (James Hartig)
+* Fix TLS connection timeout (Blake Embrey)
+* Add support for read-only, primary, standby, prefer-standby target_session_attributes (Oscar)
+* Fix connect when receiving NoticeResponse
+
 # 1.10.1 (November 20, 2021)
 
 * Close without waiting for response (Kei Kamikawa)

vendor/github.com/jackc/pgconn/config.go

@@ -248,21 +248,21 @@ func ParseConfig(connString string) (*Config, error) {
 	config.LookupFunc = makeDefaultResolver().LookupHost
 
 	notRuntimeParams := map[string]struct{}{
-		"host":                 struct{}{},
+		"host":                 {},
-		"port":                 struct{}{},
+		"port":                 {},
-		"database":             struct{}{},
+		"database":             {},
-		"user":                 struct{}{},
+		"user":                 {},
-		"password":             struct{}{},
+		"password":             {},
-		"passfile":             struct{}{},
+		"passfile":             {},
-		"connect_timeout":      struct{}{},
+		"connect_timeout":      {},
-		"sslmode":              struct{}{},
+		"sslmode":              {},
-		"sslkey":               struct{}{},
+		"sslkey":               {},
-		"sslcert":              struct{}{},
+		"sslcert":              {},
-		"sslrootcert":          struct{}{},
+		"sslrootcert":          {},
-		"target_session_attrs": struct{}{},
+		"target_session_attrs": {},
-		"min_read_buffer_size": struct{}{},
+		"min_read_buffer_size": {},
-		"service":              struct{}{},
+		"service":              {},
-		"servicefile":          struct{}{},
+		"servicefile":          {},
 	}
 
 	for k, v := range settings {
@@ -329,10 +329,19 @@ func ParseConfig(connString string) (*Config, error) {
 		}
 	}
 
-	if settings["target_session_attrs"] == "read-write" {
+	switch tsa := settings["target_session_attrs"]; tsa {
+	case "read-write":
 		config.ValidateConnect = ValidateConnectTargetSessionAttrsReadWrite
-	} else if settings["target_session_attrs"] != "any" {
+	case "read-only":
-		return nil, &parseConfigError{connString: connString, msg: fmt.Sprintf("unknown target_session_attrs value: %v", settings["target_session_attrs"])}
+		config.ValidateConnect = ValidateConnectTargetSessionAttrsReadOnly
+	case "primary":
+		config.ValidateConnect = ValidateConnectTargetSessionAttrsPrimary
+	case "standby":
+		config.ValidateConnect = ValidateConnectTargetSessionAttrsStandby
+	case "any", "prefer-standby":
+		// do nothing
+	default:
+		return nil, &parseConfigError{connString: connString, msg: fmt.Sprintf("unknown target_session_attrs value: %v", tsa)}
 	}
 
 	return config, nil
@@ -727,3 +736,48 @@ func ValidateConnectTargetSessionAttrsReadWrite(ctx context.Context, pgConn *PgC
 
 	return nil
 }
+
+// ValidateConnectTargetSessionAttrsReadOnly is an ValidateConnectFunc that implements libpq compatible
+// target_session_attrs=read-only.
+func ValidateConnectTargetSessionAttrsReadOnly(ctx context.Context, pgConn *PgConn) error {
+	result := pgConn.ExecParams(ctx, "show transaction_read_only", nil, nil, nil, nil).Read()
+	if result.Err != nil {
+		return result.Err
+	}
+
+	if string(result.Rows[0][0]) != "on" {
+		return errors.New("connection is not read only")
+	}
+
+	return nil
+}
+
+// ValidateConnectTargetSessionAttrsStandby is an ValidateConnectFunc that implements libpq compatible
+// target_session_attrs=standby.
+func ValidateConnectTargetSessionAttrsStandby(ctx context.Context, pgConn *PgConn) error {
+	result := pgConn.ExecParams(ctx, "select pg_is_in_recovery()", nil, nil, nil, nil).Read()
+	if result.Err != nil {
+		return result.Err
+	}
+
+	if string(result.Rows[0][0]) != "t" {
+		return errors.New("server is not in hot standby mode")
+	}
+
+	return nil
+}
+
+// ValidateConnectTargetSessionAttrsPrimary is an ValidateConnectFunc that implements libpq compatible
+// target_session_attrs=primary.
+func ValidateConnectTargetSessionAttrsPrimary(ctx context.Context, pgConn *PgConn) error {
+	result := pgConn.ExecParams(ctx, "select pg_is_in_recovery()", nil, nil, nil, nil).Read()
+	if result.Err != nil {
+		return result.Err
+	}
+
+	if string(result.Rows[0][0]) == "t" {
+		return errors.New("server is in standby mode")
+	}
+
+	return nil
+}

vendor/github.com/jackc/pgconn/pgconn.go

@@ -11,6 +11,7 @@ import (
 	"io"
 	"math"
 	"net"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -44,7 +45,8 @@ type Notification struct {
 // DialFunc is a function that can be used to connect to a PostgreSQL server.
 type DialFunc func(ctx context.Context, network, addr string) (net.Conn, error)
 
-// LookupFunc is a function that can be used to lookup IPs addrs from host.
+// LookupFunc is a function that can be used to lookup IPs addrs from host. Optionally an ip:port combination can be
+// returned in order to override the connection string's port.
 type LookupFunc func(ctx context.Context, host string) (addrs []string, err error)
 
 // BuildFrontendFunc is a function that can be used to create Frontend implementation for connection.
@@ -196,6 +198,18 @@ func expandWithIPs(ctx context.Context, lookupFn LookupFunc, fallbacks []*Fallba
 		}
 
 		for _, ip := range ips {
+			splitIP, splitPort, err := net.SplitHostPort(ip)
+			if err == nil {
+				port, err := strconv.ParseUint(splitPort, 10, 16)
+				if err != nil {
+					return nil, fmt.Errorf("error parsing port (%s) from lookup: %w", splitPort, err)
+				}
+				configs = append(configs, &FallbackConfig{
+					Host:      splitIP,
+					Port:      uint16(port),
+					TLSConfig: fb.TLSConfig,
+				})
+			} else {
 				configs = append(configs, &FallbackConfig{
 					Host:      ip,
 					Port:      fb.Port,
@@ -203,6 +217,7 @@ func expandWithIPs(ctx context.Context, lookupFn LookupFunc, fallbacks []*Fallba
 				})
 			}
 		}
+	}
 
 	return configs, nil
 }
@@ -215,7 +230,7 @@ func connect(ctx context.Context, config *Config, fallbackConfig *FallbackConfig
 
 	var err error
 	network, address := NetworkAddress(fallbackConfig.Host, fallbackConfig.Port)
-	pgConn.conn, err = config.DialFunc(ctx, network, address)
+	netConn, err := config.DialFunc(ctx, network, address)
 	if err != nil {
 		var netErr net.Error
 		if errors.As(err, &netErr) && netErr.Timeout() {
@@ -224,24 +239,27 @@ func connect(ctx context.Context, config *Config, fallbackConfig *FallbackConfig
 		return nil, &connectError{config: config, msg: "dial error", err: err}
 	}
 
-	pgConn.parameterStatuses = make(map[string]string)
+	pgConn.conn = netConn
+	pgConn.contextWatcher = newContextWatcher(netConn)
+	pgConn.contextWatcher.Watch(ctx)
 
 	if fallbackConfig.TLSConfig != nil {
-		if err := pgConn.startTLS(fallbackConfig.TLSConfig); err != nil {
+		tlsConn, err := startTLS(netConn, fallbackConfig.TLSConfig)
-			pgConn.conn.Close()
+		pgConn.contextWatcher.Unwatch() // Always unwatch `netConn` after TLS.
+		if err != nil {
+			netConn.Close()
 			return nil, &connectError{config: config, msg: "tls error", err: err}
 		}
+
+		pgConn.conn = tlsConn
+		pgConn.contextWatcher = newContextWatcher(tlsConn)
+		pgConn.contextWatcher.Watch(ctx)
 	}
 
-	pgConn.status = connStatusConnecting
-	pgConn.contextWatcher = ctxwatch.NewContextWatcher(
-		func() { pgConn.conn.SetDeadline(time.Date(1, 1, 1, 1, 1, 1, 1, time.UTC)) },
-		func() { pgConn.conn.SetDeadline(time.Time{}) },
-	)
-
-	pgConn.contextWatcher.Watch(ctx)
 	defer pgConn.contextWatcher.Unwatch()
 
+	pgConn.parameterStatuses = make(map[string]string)
+	pgConn.status = connStatusConnecting
 	pgConn.frontend = config.BuildFrontend(pgConn.conn, pgConn.conn)
 
 	startupMsg := pgproto3.StartupMessage{
@@ -317,7 +335,7 @@ func connect(ctx context.Context, config *Config, fallbackConfig *FallbackConfig
 				}
 			}
 			return pgConn, nil
-		case *pgproto3.ParameterStatus:
+		case *pgproto3.ParameterStatus, *pgproto3.NoticeResponse:
 			// handled by ReceiveMessage
 		case *pgproto3.ErrorResponse:
 			pgConn.conn.Close()
@@ -329,24 +347,29 @@ func connect(ctx context.Context, config *Config, fallbackConfig *FallbackConfig
 	}
 }
 
-func (pgConn *PgConn) startTLS(tlsConfig *tls.Config) (err error) {
+func newContextWatcher(conn net.Conn) *ctxwatch.ContextWatcher {
-	err = binary.Write(pgConn.conn, binary.BigEndian, []int32{8, 80877103})
+	return ctxwatch.NewContextWatcher(
+		func() { conn.SetDeadline(time.Date(1, 1, 1, 1, 1, 1, 1, time.UTC)) },
+		func() { conn.SetDeadline(time.Time{}) },
+	)
+}
+
+func startTLS(conn net.Conn, tlsConfig *tls.Config) (net.Conn, error) {
+	err := binary.Write(conn, binary.BigEndian, []int32{8, 80877103})
 	if err != nil {
-		return
+		return nil, err
 	}
 
 	response := make([]byte, 1)
-	if _, err = io.ReadFull(pgConn.conn, response); err != nil {
+	if _, err = io.ReadFull(conn, response); err != nil {
-		return
+		return nil, err
 	}
 
 	if response[0] != 'S' {
-		return errors.New("server refused TLS connection")
+		return nil, errors.New("server refused TLS connection")
 	}
 
-	pgConn.conn = tls.Client(pgConn.conn, tlsConfig)
+	return tls.Client(conn, tlsConfig), nil
-
-	return nil
 }
 
 func (pgConn *PgConn) txPasswordMessage(password string) (err error) {
@@ -1694,10 +1717,7 @@ func Construct(hc *HijackedConn) (*PgConn, error) {
 		cleanupDone: make(chan struct{}),
 	}
 
-	pgConn.contextWatcher = ctxwatch.NewContextWatcher(
+	pgConn.contextWatcher = newContextWatcher(pgConn.conn)
-		func() { pgConn.conn.SetDeadline(time.Date(1, 1, 1, 1, 1, 1, 1, time.UTC)) },
-		func() { pgConn.conn.SetDeadline(time.Time{}) },
-	)
 
 	return pgConn, nil
 }

vendor/github.com/jackc/pgtype/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 1.10.0 (February 7, 2022)
+
+* Normalize UTC timestamps to comply with stdlib (Torkel Rogstad)
+* Assign Numeric to *big.Rat (Oleg Lomaka)
+* Fix typo in float8 error message (Pinank Solanki)
+* Scan type aliases for floating point types (Collin Forsyth)
+
 # 1.9.1 (November 28, 2021)
 
 * Fix: binary timestamp is assumed to be in UTC (restored behavior changed in v1.9.0)

vendor/github.com/jackc/pgtype/convert.go

@@ -337,6 +337,10 @@ func float64AssignTo(srcVal float64, srcStatus Status, dst interface{}) error {
 			if v := reflect.ValueOf(dst); v.Kind() == reflect.Ptr {
 				el := v.Elem()
 				switch el.Kind() {
+				// if dst is a type alias of a float32 or 64, set dst val
+				case reflect.Float32, reflect.Float64:
+					el.SetFloat(srcVal)
+					return nil
 				// if dst is a pointer to pointer, strip the pointer and try again
 				case reflect.Ptr:
 					if el.IsNil() {

vendor/github.com/jackc/pgtype/float8.go

@@ -204,7 +204,7 @@ func (dst *Float8) DecodeBinary(ci *ConnInfo, src []byte) error {
 	}
 
 	if len(src) != 8 {
-		return fmt.Errorf("invalid length for float4: %v", len(src))
+		return fmt.Errorf("invalid length for float8: %v", len(src))
 	}
 
 	n := int64(binary.BigEndian.Uint64(src))

vendor/github.com/jackc/pgtype/numeric.go

@@ -369,6 +369,12 @@ func (src *Numeric) AssignTo(dst interface{}) error {
 				return fmt.Errorf("%d is greater than maximum value for %T", normalizedInt, *v)
 			}
 			*v = normalizedInt.Uint64()
+		case *big.Rat:
+			rat, err := src.toBigRat()
+			if err != nil {
+				return err
+			}
+			v.Set(rat)
 		default:
 			if nextDst, retry := GetAssignToDstType(dst); retry {
 				return src.AssignTo(nextDst)
@@ -406,6 +412,26 @@ func (dst *Numeric) toBigInt() (*big.Int, error) {
 	return num, nil
 }
 
+func (dst *Numeric) toBigRat() (*big.Rat, error) {
+	if dst.NaN {
+		return nil, fmt.Errorf("%v is not a number", dst)
+	} else if dst.InfinityModifier == Infinity {
+		return nil, fmt.Errorf("%v is infinity", dst)
+	} else if dst.InfinityModifier == NegativeInfinity {
+		return nil, fmt.Errorf("%v is -infinity", dst)
+	}
+
+	num := new(big.Rat).SetInt(dst.Int)
+	if dst.Exp > 0 {
+		mul := new(big.Int).Exp(big10, big.NewInt(int64(dst.Exp)), nil)
+		num.Mul(num, new(big.Rat).SetInt(mul))
+	} else if dst.Exp < 0 {
+		mul := new(big.Int).Exp(big10, big.NewInt(int64(-dst.Exp)), nil)
+		num.Quo(num, new(big.Rat).SetInt(mul))
+	}
+	return num, nil
+}
+
 func (src *Numeric) toFloat64() (float64, error) {
 	if src.NaN {
 		return math.NaN(), nil

vendor/github.com/jackc/pgtype/timestamptz.go

@@ -124,7 +124,7 @@ func (dst *Timestamptz) DecodeText(ci *ConnInfo, src []byte) error {
 			return err
 		}
 
-		*dst = Timestamptz{Time: tim, Status: Present}
+		*dst = Timestamptz{Time: normalizePotentialUTC(tim), Status: Present}
 	}
 
 	return nil
@@ -231,6 +231,9 @@ func (src Timestamptz) Value() (driver.Value, error) {
 		if src.InfinityModifier != None {
 			return src.InfinityModifier.String(), nil
 		}
+		if src.Time.Location().String() == time.UTC.String() {
+			return src.Time.UTC(), nil
+		}
 		return src.Time, nil
 	case Null:
 		return nil, nil
@@ -289,8 +292,23 @@ func (dst *Timestamptz) UnmarshalJSON(b []byte) error {
 			return err
 		}
 
-		*dst = Timestamptz{Time: tim, Status: Present}
+		*dst = Timestamptz{Time: normalizePotentialUTC(tim), Status: Present}
 	}
 
 	return nil
 }
+
+// Normalize timestamps in UTC location to behave similarly to how the Golang
+// standard library does it: UTC timestamps lack a .loc value.
+//
+// Reason for this: when comparing two timestamps with reflect.DeepEqual (generally
+// speaking not a good idea, but several testing libraries (for example testify)
+// does this), their location data needs to be equal for them to be considered
+// equal.
+func normalizePotentialUTC(timestamp time.Time) time.Time {
+	if timestamp.Location().String() != time.UTC.String() {
+		return timestamp
+	}
+
+	return timestamp.UTC()
+}

vendor/github.com/jackc/pgx/v4/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 4.15.0 (February 7, 2022)
+
+* Upgrade to pgconn v1.11.0
+* Upgrade to pgtype v1.10.0
+* Upgrade puddle to v1.2.1
+* Make BatchResults.Close safe to be called multiple times
+
 # 4.14.1 (November 28, 2021)
 
 * Upgrade pgtype to v1.9.1 (fixes unintentional change to timestamp binary decoding)

vendor/github.com/jackc/pgx/v4/batch.go

@@ -3,6 +3,7 @@ package pgx
 import (
 	"context"
 	"errors"
+	"fmt"
 
 	"github.com/jackc/pgconn"
 )
@@ -46,7 +47,7 @@ type BatchResults interface {
 
 	// Close closes the batch operation. This must be called before the underlying connection can be used again. Any error
 	// that occurred during a batch operation may have made it impossible to resyncronize the connection with the server.
-	// In this case the underlying connection will have been closed.
+	// In this case the underlying connection will have been closed. Close is safe to call multiple times.
 	Close() error
 }
 
@@ -57,6 +58,7 @@ type batchResults struct {
 	err    error
 	b      *Batch
 	ix     int
+	closed bool
 }
 
 // Exec reads the results from the next query in the batch as if the query has been sent with Exec.
@@ -64,6 +66,9 @@ func (br *batchResults) Exec() (pgconn.CommandTag, error) {
 	if br.err != nil {
 		return nil, br.err
 	}
+	if br.closed {
+		return nil, fmt.Errorf("batch already closed")
+	}
 
 	query, arguments, _ := br.nextQueryAndArgs()
 
@@ -114,6 +119,11 @@ func (br *batchResults) Query() (Rows, error) {
 		return &connRows{err: br.err, closed: true}, br.err
 	}
 
+	if br.closed {
+		alreadyClosedErr := fmt.Errorf("batch already closed")
+		return &connRows{err: alreadyClosedErr, closed: true}, alreadyClosedErr
+	}
+
 	rows := br.conn.getRows(br.ctx, query, arguments)
 
 	if !br.mrr.NextResult() {
@@ -140,6 +150,10 @@ func (br *batchResults) Query() (Rows, error) {
 
 // QueryFunc reads the results from the next query in the batch as if the query has been sent with Conn.QueryFunc.
 func (br *batchResults) QueryFunc(scans []interface{}, f func(QueryFuncRow) error) (pgconn.CommandTag, error) {
+	if br.closed {
+		return nil, fmt.Errorf("batch already closed")
+	}
+
 	rows, err := br.Query()
 	if err != nil {
 		return nil, err
@@ -179,6 +193,11 @@ func (br *batchResults) Close() error {
 		return br.err
 	}
 
+	if br.closed {
+		return nil
+	}
+	br.closed = true
+
 	// log any queries that haven't yet been logged by Exec or Query
 	for {
 		query, args, ok := br.nextQueryAndArgs()

vendor/github.com/jackc/pgx/v4/rows.go

@@ -41,10 +41,13 @@ type Rows interface {
 
 	// Scan reads the values from the current row into dest values positionally.
 	// dest can include pointers to core types, values implementing the Scanner
-	// interface, and nil. nil will skip the value entirely.
+	// interface, and nil. nil will skip the value entirely. It is an error to
+	// call Scan without first calling Next() and checking that it returned true.
 	Scan(dest ...interface{}) error
 
-	// Values returns the decoded row values.
+	// Values returns the decoded row values. As with Scan(), it is an error to
+	// call Values without first calling Next() and checking that it returned
+	// true.
 	Values() ([]interface{}, error)
 
 	// RawValues returns the unparsed bytes of the row values. The returned [][]byte is only valid until the next Next

vendor/github.com/robfig/cron/v3/.gitignore

@@ -0,0 +1,22 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe

vendor/github.com/robfig/cron/v3/.travis.yml

@@ -0,0 +1 @@
+language: go

vendor/github.com/robfig/cron/v3/LICENSE

@@ -0,0 +1,21 @@
+Copyright (C) 2012 Rob Figueiredo
+All Rights Reserved.
+
+MIT LICENSE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/robfig/cron/v3/README.md

@@ -0,0 +1,125 @@
+[![GoDoc](http://godoc.org/github.com/robfig/cron?status.png)](http://godoc.org/github.com/robfig/cron)
+[![Build Status](https://travis-ci.org/robfig/cron.svg?branch=master)](https://travis-ci.org/robfig/cron)
+
+# cron
+
+Cron V3 has been released!
+
+To download the specific tagged release, run:
+
+	go get github.com/robfig/cron/v3@v3.0.0
+
+Import it in your program as:
+
+	import "github.com/robfig/cron/v3"
+
+It requires Go 1.11 or later due to usage of Go Modules.
+
+Refer to the documentation here:
+http://godoc.org/github.com/robfig/cron
+
+The rest of this document describes the the advances in v3 and a list of
+breaking changes for users that wish to upgrade from an earlier version.
+
+## Upgrading to v3 (June 2019)
+
+cron v3 is a major upgrade to the library that addresses all outstanding bugs,
+feature requests, and rough edges. It is based on a merge of master which
+contains various fixes to issues found over the years and the v2 branch which
+contains some backwards-incompatible features like the ability to remove cron
+jobs. In addition, v3 adds support for Go Modules, cleans up rough edges like
+the timezone support, and fixes a number of bugs.
+
+New features:
+
+- Support for Go modules. Callers must now import this library as
+  `github.com/robfig/cron/v3`, instead of `gopkg.in/...`
+
+- Fixed bugs:
+  - 0f01e6b parser: fix combining of Dow and Dom (#70)
+  - dbf3220 adjust times when rolling the clock forward to handle non-existent midnight (#157)
+  - eeecf15 spec_test.go: ensure an error is returned on 0 increment (#144)
+  - 70971dc cron.Entries(): update request for snapshot to include a reply channel (#97)
+  - 1cba5e6 cron: fix: removing a job causes the next scheduled job to run too late (#206)
+
+- Standard cron spec parsing by default (first field is "minute"), with an easy
+  way to opt into the seconds field (quartz-compatible). Although, note that the
+  year field (optional in Quartz) is not supported.
+
+- Extensible, key/value logging via an interface that complies with
+  the https://github.com/go-logr/logr project.
+
+- The new Chain & JobWrapper types allow you to install "interceptors" to add
+  cross-cutting behavior like the following:
+  - Recover any panics from jobs
+  - Delay a job's execution if the previous run hasn't completed yet
+  - Skip a job's execution if the previous run hasn't completed yet
+  - Log each job's invocations
+  - Notification when jobs are completed
+
+It is backwards incompatible with both v1 and v2. These updates are required:
+
+- The v1 branch accepted an optional seconds field at the beginning of the cron
+  spec. This is non-standard and has led to a lot of confusion. The new default
+  parser conforms to the standard as described by [the Cron wikipedia page].
+
+  UPDATING: To retain the old behavior, construct your Cron with a custom
+  parser:
+
+      // Seconds field, required
+      cron.New(cron.WithSeconds())
+
+      // Seconds field, optional
+      cron.New(
+          cron.WithParser(
+              cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor))
+
+- The Cron type now accepts functional options on construction rather than the
+  previous ad-hoc behavior modification mechanisms (setting a field, calling a setter).
+
+  UPDATING: Code that sets Cron.ErrorLogger or calls Cron.SetLocation must be
+  updated to provide those values on construction.
+
+- CRON_TZ is now the recommended way to specify the timezone of a single
+  schedule, which is sanctioned by the specification. The legacy "TZ=" prefix
+  will continue to be supported since it is unambiguous and easy to do so.
+
+  UPDATING: No update is required.
+
+- By default, cron will no longer recover panics in jobs that it runs.
+  Recovering can be surprising (see issue #192) and seems to be at odds with
+  typical behavior of libraries. Relatedly, the `cron.WithPanicLogger` option
+  has been removed to accommodate the more general JobWrapper type.
+
+  UPDATING: To opt into panic recovery and configure the panic logger:
+
+      cron.New(cron.WithChain(
+          cron.Recover(logger),  // or use cron.DefaultLogger
+      ))
+
+- In adding support for https://github.com/go-logr/logr, `cron.WithVerboseLogger` was
+  removed, since it is duplicative with the leveled logging.
+
+  UPDATING: Callers should use `WithLogger` and specify a logger that does not
+  discard `Info` logs. For convenience, one is provided that wraps `*log.Logger`:
+
+      cron.New(
+          cron.WithLogger(cron.VerbosePrintfLogger(logger)))
+
+
+### Background - Cron spec format
+
+There are two cron spec formats in common usage:
+
+- The "standard" cron format, described on [the Cron wikipedia page] and used by
+  the cron Linux system utility.
+
+- The cron format used by [the Quartz Scheduler], commonly used for scheduled
+  jobs in Java software
+
+[the Cron wikipedia page]: https://en.wikipedia.org/wiki/Cron
+[the Quartz Scheduler]: http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/tutorial-lesson-06.html
+
+The original version of this package included an optional "seconds" field, which
+made it incompatible with both of these formats. Now, the "standard" format is
+the default format accepted, and the Quartz format is opt-in.

vendor/github.com/robfig/cron/v3/chain.go

@@ -0,0 +1,92 @@
+package cron
+
+import (
+	"fmt"
+	"runtime"
+	"sync"
+	"time"
+)
+
+// JobWrapper decorates the given Job with some behavior.
+type JobWrapper func(Job) Job
+
+// Chain is a sequence of JobWrappers that decorates submitted jobs with
+// cross-cutting behaviors like logging or synchronization.
+type Chain struct {
+	wrappers []JobWrapper
+}
+
+// NewChain returns a Chain consisting of the given JobWrappers.
+func NewChain(c ...JobWrapper) Chain {
+	return Chain{c}
+}
+
+// Then decorates the given job with all JobWrappers in the chain.
+//
+// This:
+//     NewChain(m1, m2, m3).Then(job)
+// is equivalent to:
+//     m1(m2(m3(job)))
+func (c Chain) Then(j Job) Job {
+	for i := range c.wrappers {
+		j = c.wrappers[len(c.wrappers)-i-1](j)
+	}
+	return j
+}
+
+// Recover panics in wrapped jobs and log them with the provided logger.
+func Recover(logger Logger) JobWrapper {
+	return func(j Job) Job {
+		return FuncJob(func() {
+			defer func() {
+				if r := recover(); r != nil {
+					const size = 64 << 10
+					buf := make([]byte, size)
+					buf = buf[:runtime.Stack(buf, false)]
+					err, ok := r.(error)
+					if !ok {
+						err = fmt.Errorf("%v", r)
+					}
+					logger.Error(err, "panic", "stack", "...\n"+string(buf))
+				}
+			}()
+			j.Run()
+		})
+	}
+}
+
+// DelayIfStillRunning serializes jobs, delaying subsequent runs until the
+// previous one is complete. Jobs running after a delay of more than a minute
+// have the delay logged at Info.
+func DelayIfStillRunning(logger Logger) JobWrapper {
+	return func(j Job) Job {
+		var mu sync.Mutex
+		return FuncJob(func() {
+			start := time.Now()
+			mu.Lock()
+			defer mu.Unlock()
+			if dur := time.Since(start); dur > time.Minute {
+				logger.Info("delay", "duration", dur)
+			}
+			j.Run()
+		})
+	}
+}
+
+// SkipIfStillRunning skips an invocation of the Job if a previous invocation is
+// still running. It logs skips to the given logger at Info level.
+func SkipIfStillRunning(logger Logger) JobWrapper {
+	return func(j Job) Job {
+		var ch = make(chan struct{}, 1)
+		ch <- struct{}{}
+		return FuncJob(func() {
+			select {
+			case v := <-ch:
+				j.Run()
+				ch <- v
+			default:
+				logger.Info("skip")
+			}
+		})
+	}
+}

vendor/github.com/robfig/cron/v3/constantdelay.go

@@ -0,0 +1,27 @@
+package cron
+
+import "time"
+
+// ConstantDelaySchedule represents a simple recurring duty cycle, e.g. "Every 5 minutes".
+// It does not support jobs more frequent than once a second.
+type ConstantDelaySchedule struct {
+	Delay time.Duration
+}
+
+// Every returns a crontab Schedule that activates once every duration.
+// Delays of less than a second are not supported (will round up to 1 second).
+// Any fields less than a Second are truncated.
+func Every(duration time.Duration) ConstantDelaySchedule {
+	if duration < time.Second {
+		duration = time.Second
+	}
+	return ConstantDelaySchedule{
+		Delay: duration - time.Duration(duration.Nanoseconds())%time.Second,
+	}
+}
+
+// Next returns the next time this should be run.
+// This rounds so that the next activation time will be on the second.
+func (schedule ConstantDelaySchedule) Next(t time.Time) time.Time {
+	return t.Add(schedule.Delay - time.Duration(t.Nanosecond())*time.Nanosecond)
+}

vendor/github.com/robfig/cron/v3/cron.go

@@ -0,0 +1,355 @@
+package cron
+
+import (
+	"context"
+	"sort"
+	"sync"
+	"time"
+)
+
+// Cron keeps track of any number of entries, invoking the associated func as
+// specified by the schedule. It may be started, stopped, and the entries may
+// be inspected while running.
+type Cron struct {
+	entries   []*Entry
+	chain     Chain
+	stop      chan struct{}
+	add       chan *Entry
+	remove    chan EntryID
+	snapshot  chan chan []Entry
+	running   bool
+	logger    Logger
+	runningMu sync.Mutex
+	location  *time.Location
+	parser    ScheduleParser
+	nextID    EntryID
+	jobWaiter sync.WaitGroup
+}
+
+// ScheduleParser is an interface for schedule spec parsers that return a Schedule
+type ScheduleParser interface {
+	Parse(spec string) (Schedule, error)
+}
+
+// Job is an interface for submitted cron jobs.
+type Job interface {
+	Run()
+}
+
+// Schedule describes a job's duty cycle.
+type Schedule interface {
+	// Next returns the next activation time, later than the given time.
+	// Next is invoked initially, and then each time the job is run.
+	Next(time.Time) time.Time
+}
+
+// EntryID identifies an entry within a Cron instance
+type EntryID int
+
+// Entry consists of a schedule and the func to execute on that schedule.
+type Entry struct {
+	// ID is the cron-assigned ID of this entry, which may be used to look up a
+	// snapshot or remove it.
+	ID EntryID
+
+	// Schedule on which this job should be run.
+	Schedule Schedule
+
+	// Next time the job will run, or the zero time if Cron has not been
+	// started or this entry's schedule is unsatisfiable
+	Next time.Time
+
+	// Prev is the last time this job was run, or the zero time if never.
+	Prev time.Time
+
+	// WrappedJob is the thing to run when the Schedule is activated.
+	WrappedJob Job
+
+	// Job is the thing that was submitted to cron.
+	// It is kept around so that user code that needs to get at the job later,
+	// e.g. via Entries() can do so.
+	Job Job
+}
+
+// Valid returns true if this is not the zero entry.
+func (e Entry) Valid() bool { return e.ID != 0 }
+
+// byTime is a wrapper for sorting the entry array by time
+// (with zero time at the end).
+type byTime []*Entry
+
+func (s byTime) Len() int      { return len(s) }
+func (s byTime) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s byTime) Less(i, j int) bool {
+	// Two zero times should return false.
+	// Otherwise, zero is "greater" than any other time.
+	// (To sort it at the end of the list.)
+	if s[i].Next.IsZero() {
+		return false
+	}
+	if s[j].Next.IsZero() {
+		return true
+	}
+	return s[i].Next.Before(s[j].Next)
+}
+
+// New returns a new Cron job runner, modified by the given options.
+//
+// Available Settings
+//
+//   Time Zone
+//     Description: The time zone in which schedules are interpreted
+//     Default:     time.Local
+//
+//   Parser
+//     Description: Parser converts cron spec strings into cron.Schedules.
+//     Default:     Accepts this spec: https://en.wikipedia.org/wiki/Cron
+//
+//   Chain
+//     Description: Wrap submitted jobs to customize behavior.
+//     Default:     A chain that recovers panics and logs them to stderr.
+//
+// See "cron.With*" to modify the default behavior.
+func New(opts ...Option) *Cron {
+	c := &Cron{
+		entries:   nil,
+		chain:     NewChain(),
+		add:       make(chan *Entry),
+		stop:      make(chan struct{}),
+		snapshot:  make(chan chan []Entry),
+		remove:    make(chan EntryID),
+		running:   false,
+		runningMu: sync.Mutex{},
+		logger:    DefaultLogger,
+		location:  time.Local,
+		parser:    standardParser,
+	}
+	for _, opt := range opts {
+		opt(c)
+	}
+	return c
+}
+
+// FuncJob is a wrapper that turns a func() into a cron.Job
+type FuncJob func()
+
+func (f FuncJob) Run() { f() }
+
+// AddFunc adds a func to the Cron to be run on the given schedule.
+// The spec is parsed using the time zone of this Cron instance as the default.
+// An opaque ID is returned that can be used to later remove it.
+func (c *Cron) AddFunc(spec string, cmd func()) (EntryID, error) {
+	return c.AddJob(spec, FuncJob(cmd))
+}
+
+// AddJob adds a Job to the Cron to be run on the given schedule.
+// The spec is parsed using the time zone of this Cron instance as the default.
+// An opaque ID is returned that can be used to later remove it.
+func (c *Cron) AddJob(spec string, cmd Job) (EntryID, error) {
+	schedule, err := c.parser.Parse(spec)
+	if err != nil {
+		return 0, err
+	}
+	return c.Schedule(schedule, cmd), nil
+}
+
+// Schedule adds a Job to the Cron to be run on the given schedule.
+// The job is wrapped with the configured Chain.
+func (c *Cron) Schedule(schedule Schedule, cmd Job) EntryID {
+	c.runningMu.Lock()
+	defer c.runningMu.Unlock()
+	c.nextID++
+	entry := &Entry{
+		ID:         c.nextID,
+		Schedule:   schedule,
+		WrappedJob: c.chain.Then(cmd),
+		Job:        cmd,
+	}
+	if !c.running {
+		c.entries = append(c.entries, entry)
+	} else {
+		c.add <- entry
+	}
+	return entry.ID
+}
+
+// Entries returns a snapshot of the cron entries.
+func (c *Cron) Entries() []Entry {
+	c.runningMu.Lock()
+	defer c.runningMu.Unlock()
+	if c.running {
+		replyChan := make(chan []Entry, 1)
+		c.snapshot <- replyChan
+		return <-replyChan
+	}
+	return c.entrySnapshot()
+}
+
+// Location gets the time zone location
+func (c *Cron) Location() *time.Location {
+	return c.location
+}
+
+// Entry returns a snapshot of the given entry, or nil if it couldn't be found.
+func (c *Cron) Entry(id EntryID) Entry {
+	for _, entry := range c.Entries() {
+		if id == entry.ID {
+			return entry
+		}
+	}
+	return Entry{}
+}
+
+// Remove an entry from being run in the future.
+func (c *Cron) Remove(id EntryID) {
+	c.runningMu.Lock()
+	defer c.runningMu.Unlock()
+	if c.running {
+		c.remove <- id
+	} else {
+		c.removeEntry(id)
+	}
+}
+
+// Start the cron scheduler in its own goroutine, or no-op if already started.
+func (c *Cron) Start() {
+	c.runningMu.Lock()
+	defer c.runningMu.Unlock()
+	if c.running {
+		return
+	}
+	c.running = true
+	go c.run()
+}
+
+// Run the cron scheduler, or no-op if already running.
+func (c *Cron) Run() {
+	c.runningMu.Lock()
+	if c.running {
+		c.runningMu.Unlock()
+		return
+	}
+	c.running = true
+	c.runningMu.Unlock()
+	c.run()
+}
+
+// run the scheduler.. this is private just due to the need to synchronize
+// access to the 'running' state variable.
+func (c *Cron) run() {
+	c.logger.Info("start")
+
+	// Figure out the next activation times for each entry.
+	now := c.now()
+	for _, entry := range c.entries {
+		entry.Next = entry.Schedule.Next(now)
+		c.logger.Info("schedule", "now", now, "entry", entry.ID, "next", entry.Next)
+	}
+
+	for {
+		// Determine the next entry to run.
+		sort.Sort(byTime(c.entries))
+
+		var timer *time.Timer
+		if len(c.entries) == 0 || c.entries[0].Next.IsZero() {
+			// If there are no entries yet, just sleep - it still handles new entries
+			// and stop requests.
+			timer = time.NewTimer(100000 * time.Hour)
+		} else {
+			timer = time.NewTimer(c.entries[0].Next.Sub(now))
+		}
+
+		for {
+			select {
+			case now = <-timer.C:
+				now = now.In(c.location)
+				c.logger.Info("wake", "now", now)
+
+				// Run every entry whose next time was less than now
+				for _, e := range c.entries {
+					if e.Next.After(now) || e.Next.IsZero() {
+						break
+					}
+					c.startJob(e.WrappedJob)
+					e.Prev = e.Next
+					e.Next = e.Schedule.Next(now)
+					c.logger.Info("run", "now", now, "entry", e.ID, "next", e.Next)
+				}
+
+			case newEntry := <-c.add:
+				timer.Stop()
+				now = c.now()
+				newEntry.Next = newEntry.Schedule.Next(now)
+				c.entries = append(c.entries, newEntry)
+				c.logger.Info("added", "now", now, "entry", newEntry.ID, "next", newEntry.Next)
+
+			case replyChan := <-c.snapshot:
+				replyChan <- c.entrySnapshot()
+				continue
+
+			case <-c.stop:
+				timer.Stop()
+				c.logger.Info("stop")
+				return
+
+			case id := <-c.remove:
+				timer.Stop()
+				now = c.now()
+				c.removeEntry(id)
+				c.logger.Info("removed", "entry", id)
+			}
+
+			break
+		}
+	}
+}
+
+// startJob runs the given job in a new goroutine.
+func (c *Cron) startJob(j Job) {
+	c.jobWaiter.Add(1)
+	go func() {
+		defer c.jobWaiter.Done()
+		j.Run()
+	}()
+}
+
+// now returns current time in c location
+func (c *Cron) now() time.Time {
+	return time.Now().In(c.location)
+}
+
+// Stop stops the cron scheduler if it is running; otherwise it does nothing.
+// A context is returned so the caller can wait for running jobs to complete.
+func (c *Cron) Stop() context.Context {
+	c.runningMu.Lock()
+	defer c.runningMu.Unlock()
+	if c.running {
+		c.stop <- struct{}{}
+		c.running = false
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		c.jobWaiter.Wait()
+		cancel()
+	}()
+	return ctx
+}
+
+// entrySnapshot returns a copy of the current cron entry list.
+func (c *Cron) entrySnapshot() []Entry {
+	var entries = make([]Entry, len(c.entries))
+	for i, e := range c.entries {
+		entries[i] = *e
+	}
+	return entries
+}
+
+func (c *Cron) removeEntry(id EntryID) {
+	var entries []*Entry
+	for _, e := range c.entries {
+		if e.ID != id {
+			entries = append(entries, e)
+		}
+	}
+	c.entries = entries
+}

vendor/github.com/robfig/cron/v3/doc.go

@@ -0,0 +1,231 @@
+/*
+Package cron implements a cron spec parser and job runner.
+
+Installation
+
+To download the specific tagged release, run:
+
+	go get github.com/robfig/cron/v3@v3.0.0
+
+Import it in your program as:
+
+	import "github.com/robfig/cron/v3"
+
+It requires Go 1.11 or later due to usage of Go Modules.
+
+Usage
+
+Callers may register Funcs to be invoked on a given schedule.  Cron will run
+them in their own goroutines.
+
+	c := cron.New()
+	c.AddFunc("30 * * * *", func() { fmt.Println("Every hour on the half hour") })
+	c.AddFunc("30 3-6,20-23 * * *", func() { fmt.Println(".. in the range 3-6am, 8-11pm") })
+	c.AddFunc("CRON_TZ=Asia/Tokyo 30 04 * * *", func() { fmt.Println("Runs at 04:30 Tokyo time every day") })
+	c.AddFunc("@hourly",      func() { fmt.Println("Every hour, starting an hour from now") })
+	c.AddFunc("@every 1h30m", func() { fmt.Println("Every hour thirty, starting an hour thirty from now") })
+	c.Start()
+	..
+	// Funcs are invoked in their own goroutine, asynchronously.
+	...
+	// Funcs may also be added to a running Cron
+	c.AddFunc("@daily", func() { fmt.Println("Every day") })
+	..
+	// Inspect the cron job entries' next and previous run times.
+	inspect(c.Entries())
+	..
+	c.Stop()  // Stop the scheduler (does not stop any jobs already running).
+
+CRON Expression Format
+
+A cron expression represents a set of times, using 5 space-separated fields.
+
+	Field name   | Mandatory? | Allowed values  | Allowed special characters
+	----------   | ---------- | --------------  | --------------------------
+	Minutes      | Yes        | 0-59            | * / , -
+	Hours        | Yes        | 0-23            | * / , -
+	Day of month | Yes        | 1-31            | * / , - ?
+	Month        | Yes        | 1-12 or JAN-DEC | * / , -
+	Day of week  | Yes        | 0-6 or SUN-SAT  | * / , - ?
+
+Month and Day-of-week field values are case insensitive.  "SUN", "Sun", and
+"sun" are equally accepted.
+
+The specific interpretation of the format is based on the Cron Wikipedia page:
+https://en.wikipedia.org/wiki/Cron
+
+Alternative Formats
+
+Alternative Cron expression formats support other fields like seconds. You can
+implement that by creating a custom Parser as follows.
+
+	cron.New(
+		cron.WithParser(
+			cron.NewParser(
+				cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor)))
+
+Since adding Seconds is the most common modification to the standard cron spec,
+cron provides a builtin function to do that, which is equivalent to the custom
+parser you saw earlier, except that its seconds field is REQUIRED:
+
+	cron.New(cron.WithSeconds())
+
+That emulates Quartz, the most popular alternative Cron schedule format:
+http://www.quartz-scheduler.org/documentation/quartz-2.x/tutorials/crontrigger.html
+
+Special Characters
+
+Asterisk ( * )
+
+The asterisk indicates that the cron expression will match for all values of the
+field; e.g., using an asterisk in the 5th field (month) would indicate every
+month.
+
+Slash ( / )
+
+Slashes are used to describe increments of ranges. For example 3-59/15 in the
+1st field (minutes) would indicate the 3rd minute of the hour and every 15
+minutes thereafter. The form "*\/..." is equivalent to the form "first-last/...",
+that is, an increment over the largest possible range of the field.  The form
+"N/..." is accepted as meaning "N-MAX/...", that is, starting at N, use the
+increment until the end of that specific range.  It does not wrap around.
+
+Comma ( , )
+
+Commas are used to separate items of a list. For example, using "MON,WED,FRI" in
+the 5th field (day of week) would mean Mondays, Wednesdays and Fridays.
+
+Hyphen ( - )
+
+Hyphens are used to define ranges. For example, 9-17 would indicate every
+hour between 9am and 5pm inclusive.
+
+Question mark ( ? )
+
+Question mark may be used instead of '*' for leaving either day-of-month or
+day-of-week blank.
+
+Predefined schedules
+
+You may use one of several pre-defined schedules in place of a cron expression.
+
+	Entry                  | Description                                | Equivalent To
+	-----                  | -----------                                | -------------
+	@yearly (or @annually) | Run once a year, midnight, Jan. 1st        | 0 0 1 1 *
+	@monthly               | Run once a month, midnight, first of month | 0 0 1 * *
+	@weekly                | Run once a week, midnight between Sat/Sun  | 0 0 * * 0
+	@daily (or @midnight)  | Run once a day, midnight                   | 0 0 * * *
+	@hourly                | Run once an hour, beginning of hour        | 0 * * * *
+
+Intervals
+
+You may also schedule a job to execute at fixed intervals, starting at the time it's added
+or cron is run. This is supported by formatting the cron spec like this:
+
+    @every <duration>
+
+where "duration" is a string accepted by time.ParseDuration
+(http://golang.org/pkg/time/#ParseDuration).
+
+For example, "@every 1h30m10s" would indicate a schedule that activates after
+1 hour, 30 minutes, 10 seconds, and then every interval after that.
+
+Note: The interval does not take the job runtime into account.  For example,
+if a job takes 3 minutes to run, and it is scheduled to run every 5 minutes,
+it will have only 2 minutes of idle time between each run.
+
+Time zones
+
+By default, all interpretation and scheduling is done in the machine's local
+time zone (time.Local). You can specify a different time zone on construction:
+
+      cron.New(
+          cron.WithLocation(time.UTC))
+
+Individual cron schedules may also override the time zone they are to be
+interpreted in by providing an additional space-separated field at the beginning
+of the cron spec, of the form "CRON_TZ=Asia/Tokyo".
+
+For example:
+
+	# Runs at 6am in time.Local
+	cron.New().AddFunc("0 6 * * ?", ...)
+
+	# Runs at 6am in America/New_York
+	nyc, _ := time.LoadLocation("America/New_York")
+	c := cron.New(cron.WithLocation(nyc))
+	c.AddFunc("0 6 * * ?", ...)
+
+	# Runs at 6am in Asia/Tokyo
+	cron.New().AddFunc("CRON_TZ=Asia/Tokyo 0 6 * * ?", ...)
+
+	# Runs at 6am in Asia/Tokyo
+	c := cron.New(cron.WithLocation(nyc))
+	c.SetLocation("America/New_York")
+	c.AddFunc("CRON_TZ=Asia/Tokyo 0 6 * * ?", ...)
+
+The prefix "TZ=(TIME ZONE)" is also supported for legacy compatibility.
+
+Be aware that jobs scheduled during daylight-savings leap-ahead transitions will
+not be run!
+
+Job Wrappers
+
+A Cron runner may be configured with a chain of job wrappers to add
+cross-cutting functionality to all submitted jobs. For example, they may be used
+to achieve the following effects:
+
+  - Recover any panics from jobs (activated by default)
+  - Delay a job's execution if the previous run hasn't completed yet
+  - Skip a job's execution if the previous run hasn't completed yet
+  - Log each job's invocations
+
+Install wrappers for all jobs added to a cron using the `cron.WithChain` option:
+
+	cron.New(cron.WithChain(
+		cron.SkipIfStillRunning(logger),
+	))
+
+Install wrappers for individual jobs by explicitly wrapping them:
+
+	job = cron.NewChain(
+		cron.SkipIfStillRunning(logger),
+	).Then(job)
+
+Thread safety
+
+Since the Cron service runs concurrently with the calling code, some amount of
+care must be taken to ensure proper synchronization.
+
+All cron methods are designed to be correctly synchronized as long as the caller
+ensures that invocations have a clear happens-before ordering between them.
+
+Logging
+
+Cron defines a Logger interface that is a subset of the one defined in
+github.com/go-logr/logr. It has two logging levels (Info and Error), and
+parameters are key/value pairs. This makes it possible for cron logging to plug
+into structured logging systems. An adapter, [Verbose]PrintfLogger, is provided
+to wrap the standard library *log.Logger.
+
+For additional insight into Cron operations, verbose logging may be activated
+which will record job runs, scheduling decisions, and added or removed jobs.
+Activate it with a one-off logger as follows:
+
+	cron.New(
+		cron.WithLogger(
+			cron.VerbosePrintfLogger(log.New(os.Stdout, "cron: ", log.LstdFlags))))
+
+
+Implementation
+
+Cron entries are stored in an array, sorted by their next activation time.  Cron
+sleeps until the next job is due to be run.
+
+Upon waking:
+ - it runs each entry that is active on that second
+ - it calculates the next run times for the jobs that were run
+ - it re-sorts the array of entries by next activation time.
+ - it goes to sleep until the soonest job.
+*/
+package cron

vendor/github.com/robfig/cron/v3/logger.go

@@ -0,0 +1,86 @@
+package cron
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+	"time"
+)
+
+// DefaultLogger is used by Cron if none is specified.
+var DefaultLogger Logger = PrintfLogger(log.New(os.Stdout, "cron: ", log.LstdFlags))
+
+// DiscardLogger can be used by callers to discard all log messages.
+var DiscardLogger Logger = PrintfLogger(log.New(ioutil.Discard, "", 0))
+
+// Logger is the interface used in this package for logging, so that any backend
+// can be plugged in. It is a subset of the github.com/go-logr/logr interface.
+type Logger interface {
+	// Info logs routine messages about cron's operation.
+	Info(msg string, keysAndValues ...interface{})
+	// Error logs an error condition.
+	Error(err error, msg string, keysAndValues ...interface{})
+}
+
+// PrintfLogger wraps a Printf-based logger (such as the standard library "log")
+// into an implementation of the Logger interface which logs errors only.
+func PrintfLogger(l interface{ Printf(string, ...interface{}) }) Logger {
+	return printfLogger{l, false}
+}
+
+// VerbosePrintfLogger wraps a Printf-based logger (such as the standard library
+// "log") into an implementation of the Logger interface which logs everything.
+func VerbosePrintfLogger(l interface{ Printf(string, ...interface{}) }) Logger {
+	return printfLogger{l, true}
+}
+
+type printfLogger struct {
+	logger  interface{ Printf(string, ...interface{}) }
+	logInfo bool
+}
+
+func (pl printfLogger) Info(msg string, keysAndValues ...interface{}) {
+	if pl.logInfo {
+		keysAndValues = formatTimes(keysAndValues)
+		pl.logger.Printf(
+			formatString(len(keysAndValues)),
+			append([]interface{}{msg}, keysAndValues...)...)
+	}
+}
+
+func (pl printfLogger) Error(err error, msg string, keysAndValues ...interface{}) {
+	keysAndValues = formatTimes(keysAndValues)
+	pl.logger.Printf(
+		formatString(len(keysAndValues)+2),
+		append([]interface{}{msg, "error", err}, keysAndValues...)...)
+}
+
+// formatString returns a logfmt-like format string for the number of
+// key/values.
+func formatString(numKeysAndValues int) string {
+	var sb strings.Builder
+	sb.WriteString("%s")
+	if numKeysAndValues > 0 {
+		sb.WriteString(", ")
+	}
+	for i := 0; i < numKeysAndValues/2; i++ {
+		if i > 0 {
+			sb.WriteString(", ")
+		}
+		sb.WriteString("%v=%v")
+	}
+	return sb.String()
+}
+
+// formatTimes formats any time.Time values as RFC3339.
+func formatTimes(keysAndValues []interface{}) []interface{} {
+	var formattedArgs []interface{}
+	for _, arg := range keysAndValues {
+		if t, ok := arg.(time.Time); ok {
+			arg = t.Format(time.RFC3339)
+		}
+		formattedArgs = append(formattedArgs, arg)
+	}
+	return formattedArgs
+}

vendor/github.com/robfig/cron/v3/option.go

@@ -0,0 +1,45 @@
+package cron
+
+import (
+	"time"
+)
+
+// Option represents a modification to the default behavior of a Cron.
+type Option func(*Cron)
+
+// WithLocation overrides the timezone of the cron instance.
+func WithLocation(loc *time.Location) Option {
+	return func(c *Cron) {
+		c.location = loc
+	}
+}
+
+// WithSeconds overrides the parser used for interpreting job schedules to
+// include a seconds field as the first one.
+func WithSeconds() Option {
+	return WithParser(NewParser(
+		Second | Minute | Hour | Dom | Month | Dow | Descriptor,
+	))
+}
+
+// WithParser overrides the parser used for interpreting job schedules.
+func WithParser(p ScheduleParser) Option {
+	return func(c *Cron) {
+		c.parser = p
+	}
+}
+
+// WithChain specifies Job wrappers to apply to all jobs added to this cron.
+// Refer to the Chain* functions in this package for provided wrappers.
+func WithChain(wrappers ...JobWrapper) Option {
+	return func(c *Cron) {
+		c.chain = NewChain(wrappers...)
+	}
+}
+
+// WithLogger uses the provided logger.
+func WithLogger(logger Logger) Option {
+	return func(c *Cron) {
+		c.logger = logger
+	}
+}

vendor/github.com/robfig/cron/v3/parser.go

@@ -0,0 +1,434 @@
+package cron
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Configuration options for creating a parser. Most options specify which
+// fields should be included, while others enable features. If a field is not
+// included the parser will assume a default value. These options do not change
+// the order fields are parse in.
+type ParseOption int
+
+const (
+	Second         ParseOption = 1 << iota // Seconds field, default 0
+	SecondOptional                         // Optional seconds field, default 0
+	Minute                                 // Minutes field, default 0
+	Hour                                   // Hours field, default 0
+	Dom                                    // Day of month field, default *
+	Month                                  // Month field, default *
+	Dow                                    // Day of week field, default *
+	DowOptional                            // Optional day of week field, default *
+	Descriptor                             // Allow descriptors such as @monthly, @weekly, etc.
+)
+
+var places = []ParseOption{
+	Second,
+	Minute,
+	Hour,
+	Dom,
+	Month,
+	Dow,
+}
+
+var defaults = []string{
+	"0",
+	"0",
+	"0",
+	"*",
+	"*",
+	"*",
+}
+
+// A custom Parser that can be configured.
+type Parser struct {
+	options ParseOption
+}
+
+// NewParser creates a Parser with custom options.
+//
+// It panics if more than one Optional is given, since it would be impossible to
+// correctly infer which optional is provided or missing in general.
+//
+// Examples
+//
+//  // Standard parser without descriptors
+//  specParser := NewParser(Minute | Hour | Dom | Month | Dow)
+//  sched, err := specParser.Parse("0 0 15 */3 *")
+//
+//  // Same as above, just excludes time fields
+//  subsParser := NewParser(Dom | Month | Dow)
+//  sched, err := specParser.Parse("15 */3 *")
+//
+//  // Same as above, just makes Dow optional
+//  subsParser := NewParser(Dom | Month | DowOptional)
+//  sched, err := specParser.Parse("15 */3")
+//
+func NewParser(options ParseOption) Parser {
+	optionals := 0
+	if options&DowOptional > 0 {
+		optionals++
+	}
+	if options&SecondOptional > 0 {
+		optionals++
+	}
+	if optionals > 1 {
+		panic("multiple optionals may not be configured")
+	}
+	return Parser{options}
+}
+
+// Parse returns a new crontab schedule representing the given spec.
+// It returns a descriptive error if the spec is not valid.
+// It accepts crontab specs and features configured by NewParser.
+func (p Parser) Parse(spec string) (Schedule, error) {
+	if len(spec) == 0 {
+		return nil, fmt.Errorf("empty spec string")
+	}
+
+	// Extract timezone if present
+	var loc = time.Local
+	if strings.HasPrefix(spec, "TZ=") || strings.HasPrefix(spec, "CRON_TZ=") {
+		var err error
+		i := strings.Index(spec, " ")
+		eq := strings.Index(spec, "=")
+		if loc, err = time.LoadLocation(spec[eq+1 : i]); err != nil {
+			return nil, fmt.Errorf("provided bad location %s: %v", spec[eq+1:i], err)
+		}
+		spec = strings.TrimSpace(spec[i:])
+	}
+
+	// Handle named schedules (descriptors), if configured
+	if strings.HasPrefix(spec, "@") {
+		if p.options&Descriptor == 0 {
+			return nil, fmt.Errorf("parser does not accept descriptors: %v", spec)
+		}
+		return parseDescriptor(spec, loc)
+	}
+
+	// Split on whitespace.
+	fields := strings.Fields(spec)
+
+	// Validate & fill in any omitted or optional fields
+	var err error
+	fields, err = normalizeFields(fields, p.options)
+	if err != nil {
+		return nil, err
+	}
+
+	field := func(field string, r bounds) uint64 {
+		if err != nil {
+			return 0
+		}
+		var bits uint64
+		bits, err = getField(field, r)
+		return bits
+	}
+
+	var (
+		second     = field(fields[0], seconds)
+		minute     = field(fields[1], minutes)
+		hour       = field(fields[2], hours)
+		dayofmonth = field(fields[3], dom)
+		month      = field(fields[4], months)
+		dayofweek  = field(fields[5], dow)
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return &SpecSchedule{
+		Second:   second,
+		Minute:   minute,
+		Hour:     hour,
+		Dom:      dayofmonth,
+		Month:    month,
+		Dow:      dayofweek,
+		Location: loc,
+	}, nil
+}
+
+// normalizeFields takes a subset set of the time fields and returns the full set
+// with defaults (zeroes) populated for unset fields.
+//
+// As part of performing this function, it also validates that the provided
+// fields are compatible with the configured options.
+func normalizeFields(fields []string, options ParseOption) ([]string, error) {
+	// Validate optionals & add their field to options
+	optionals := 0
+	if options&SecondOptional > 0 {
+		options |= Second
+		optionals++
+	}
+	if options&DowOptional > 0 {
+		options |= Dow
+		optionals++
+	}
+	if optionals > 1 {
+		return nil, fmt.Errorf("multiple optionals may not be configured")
+	}
+
+	// Figure out how many fields we need
+	max := 0
+	for _, place := range places {
+		if options&place > 0 {
+			max++
+		}
+	}
+	min := max - optionals
+
+	// Validate number of fields
+	if count := len(fields); count < min || count > max {
+		if min == max {
+			return nil, fmt.Errorf("expected exactly %d fields, found %d: %s", min, count, fields)
+		}
+		return nil, fmt.Errorf("expected %d to %d fields, found %d: %s", min, max, count, fields)
+	}
+
+	// Populate the optional field if not provided
+	if min < max && len(fields) == min {
+		switch {
+		case options&DowOptional > 0:
+			fields = append(fields, defaults[5]) // TODO: improve access to default
+		case options&SecondOptional > 0:
+			fields = append([]string{defaults[0]}, fields...)
+		default:
+			return nil, fmt.Errorf("unknown optional field")
+		}
+	}
+
+	// Populate all fields not part of options with their defaults
+	n := 0
+	expandedFields := make([]string, len(places))
+	copy(expandedFields, defaults)
+	for i, place := range places {
+		if options&place > 0 {
+			expandedFields[i] = fields[n]
+			n++
+		}
+	}
+	return expandedFields, nil
+}
+
+var standardParser = NewParser(
+	Minute | Hour | Dom | Month | Dow | Descriptor,
+)
+
+// ParseStandard returns a new crontab schedule representing the given
+// standardSpec (https://en.wikipedia.org/wiki/Cron). It requires 5 entries
+// representing: minute, hour, day of month, month and day of week, in that
+// order. It returns a descriptive error if the spec is not valid.
+//
+// It accepts
+//   - Standard crontab specs, e.g. "* * * * ?"
+//   - Descriptors, e.g. "@midnight", "@every 1h30m"
+func ParseStandard(standardSpec string) (Schedule, error) {
+	return standardParser.Parse(standardSpec)
+}
+
+// getField returns an Int with the bits set representing all of the times that
+// the field represents or error parsing field value.  A "field" is a comma-separated
+// list of "ranges".
+func getField(field string, r bounds) (uint64, error) {
+	var bits uint64
+	ranges := strings.FieldsFunc(field, func(r rune) bool { return r == ',' })
+	for _, expr := range ranges {
+		bit, err := getRange(expr, r)
+		if err != nil {
+			return bits, err
+		}
+		bits |= bit
+	}
+	return bits, nil
+}
+
+// getRange returns the bits indicated by the given expression:
+//   number | number "-" number [ "/" number ]
+// or error parsing range.
+func getRange(expr string, r bounds) (uint64, error) {
+	var (
+		start, end, step uint
+		rangeAndStep     = strings.Split(expr, "/")
+		lowAndHigh       = strings.Split(rangeAndStep[0], "-")
+		singleDigit      = len(lowAndHigh) == 1
+		err              error
+	)
+
+	var extra uint64
+	if lowAndHigh[0] == "*" || lowAndHigh[0] == "?" {
+		start = r.min
+		end = r.max
+		extra = starBit
+	} else {
+		start, err = parseIntOrName(lowAndHigh[0], r.names)
+		if err != nil {
+			return 0, err
+		}
+		switch len(lowAndHigh) {
+		case 1:
+			end = start
+		case 2:
+			end, err = parseIntOrName(lowAndHigh[1], r.names)
+			if err != nil {
+				return 0, err
+			}
+		default:
+			return 0, fmt.Errorf("too many hyphens: %s", expr)
+		}
+	}
+
+	switch len(rangeAndStep) {
+	case 1:
+		step = 1
+	case 2:
+		step, err = mustParseInt(rangeAndStep[1])
+		if err != nil {
+			return 0, err
+		}
+
+		// Special handling: "N/step" means "N-max/step".
+		if singleDigit {
+			end = r.max
+		}
+		if step > 1 {
+			extra = 0
+		}
+	default:
+		return 0, fmt.Errorf("too many slashes: %s", expr)
+	}
+
+	if start < r.min {
+		return 0, fmt.Errorf("beginning of range (%d) below minimum (%d): %s", start, r.min, expr)
+	}
+	if end > r.max {
+		return 0, fmt.Errorf("end of range (%d) above maximum (%d): %s", end, r.max, expr)
+	}
+	if start > end {
+		return 0, fmt.Errorf("beginning of range (%d) beyond end of range (%d): %s", start, end, expr)
+	}
+	if step == 0 {
+		return 0, fmt.Errorf("step of range should be a positive number: %s", expr)
+	}
+
+	return getBits(start, end, step) | extra, nil
+}
+
+// parseIntOrName returns the (possibly-named) integer contained in expr.
+func parseIntOrName(expr string, names map[string]uint) (uint, error) {
+	if names != nil {
+		if namedInt, ok := names[strings.ToLower(expr)]; ok {
+			return namedInt, nil
+		}
+	}
+	return mustParseInt(expr)
+}
+
+// mustParseInt parses the given expression as an int or returns an error.
+func mustParseInt(expr string) (uint, error) {
+	num, err := strconv.Atoi(expr)
+	if err != nil {
+		return 0, fmt.Errorf("failed to parse int from %s: %s", expr, err)
+	}
+	if num < 0 {
+		return 0, fmt.Errorf("negative number (%d) not allowed: %s", num, expr)
+	}
+
+	return uint(num), nil
+}
+
+// getBits sets all bits in the range [min, max], modulo the given step size.
+func getBits(min, max, step uint) uint64 {
+	var bits uint64
+
+	// If step is 1, use shifts.
+	if step == 1 {
+		return ^(math.MaxUint64 << (max + 1)) & (math.MaxUint64 << min)
+	}
+
+	// Else, use a simple loop.
+	for i := min; i <= max; i += step {
+		bits |= 1 << i
+	}
+	return bits
+}
+
+// all returns all bits within the given bounds.  (plus the star bit)
+func all(r bounds) uint64 {
+	return getBits(r.min, r.max, 1) | starBit
+}
+
+// parseDescriptor returns a predefined schedule for the expression, or error if none matches.
+func parseDescriptor(descriptor string, loc *time.Location) (Schedule, error) {
+	switch descriptor {
+	case "@yearly", "@annually":
+		return &SpecSchedule{
+			Second:   1 << seconds.min,
+			Minute:   1 << minutes.min,
+			Hour:     1 << hours.min,
+			Dom:      1 << dom.min,
+			Month:    1 << months.min,
+			Dow:      all(dow),
+			Location: loc,
+		}, nil
+
+	case "@monthly":
+		return &SpecSchedule{
+			Second:   1 << seconds.min,
+			Minute:   1 << minutes.min,
+			Hour:     1 << hours.min,
+			Dom:      1 << dom.min,
+			Month:    all(months),
+			Dow:      all(dow),
+			Location: loc,
+		}, nil
+
+	case "@weekly":
+		return &SpecSchedule{
+			Second:   1 << seconds.min,
+			Minute:   1 << minutes.min,
+			Hour:     1 << hours.min,
+			Dom:      all(dom),
+			Month:    all(months),
+			Dow:      1 << dow.min,
+			Location: loc,
+		}, nil
+
+	case "@daily", "@midnight":
+		return &SpecSchedule{
+			Second:   1 << seconds.min,
+			Minute:   1 << minutes.min,
+			Hour:     1 << hours.min,
+			Dom:      all(dom),
+			Month:    all(months),
+			Dow:      all(dow),
+			Location: loc,
+		}, nil
+
+	case "@hourly":
+		return &SpecSchedule{
+			Second:   1 << seconds.min,
+			Minute:   1 << minutes.min,
+			Hour:     all(hours),
+			Dom:      all(dom),
+			Month:    all(months),
+			Dow:      all(dow),
+			Location: loc,
+		}, nil
+
+	}
+
+	const every = "@every "
+	if strings.HasPrefix(descriptor, every) {
+		duration, err := time.ParseDuration(descriptor[len(every):])
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse duration %s: %s", descriptor, err)
+		}
+		return Every(duration), nil
+	}
+
+	return nil, fmt.Errorf("unrecognized descriptor: %s", descriptor)
+}

vendor/github.com/robfig/cron/v3/spec.go

@@ -0,0 +1,188 @@
+package cron
+
+import "time"
+
+// SpecSchedule specifies a duty cycle (to the second granularity), based on a
+// traditional crontab specification. It is computed initially and stored as bit sets.
+type SpecSchedule struct {
+	Second, Minute, Hour, Dom, Month, Dow uint64
+
+	// Override location for this schedule.
+	Location *time.Location
+}
+
+// bounds provides a range of acceptable values (plus a map of name to value).
+type bounds struct {
+	min, max uint
+	names    map[string]uint
+}
+
+// The bounds for each field.
+var (
+	seconds = bounds{0, 59, nil}
+	minutes = bounds{0, 59, nil}
+	hours   = bounds{0, 23, nil}
+	dom     = bounds{1, 31, nil}
+	months  = bounds{1, 12, map[string]uint{
+		"jan": 1,
+		"feb": 2,
+		"mar": 3,
+		"apr": 4,
+		"may": 5,
+		"jun": 6,
+		"jul": 7,
+		"aug": 8,
+		"sep": 9,
+		"oct": 10,
+		"nov": 11,
+		"dec": 12,
+	}}
+	dow = bounds{0, 6, map[string]uint{
+		"sun": 0,
+		"mon": 1,
+		"tue": 2,
+		"wed": 3,
+		"thu": 4,
+		"fri": 5,
+		"sat": 6,
+	}}
+)
+
+const (
+	// Set the top bit if a star was included in the expression.
+	starBit = 1 << 63
+)
+
+// Next returns the next time this schedule is activated, greater than the given
+// time.  If no time can be found to satisfy the schedule, return the zero time.
+func (s *SpecSchedule) Next(t time.Time) time.Time {
+	// General approach
+	//
+	// For Month, Day, Hour, Minute, Second:
+	// Check if the time value matches.  If yes, continue to the next field.
+	// If the field doesn't match the schedule, then increment the field until it matches.
+	// While incrementing the field, a wrap-around brings it back to the beginning
+	// of the field list (since it is necessary to re-verify previous field
+	// values)
+
+	// Convert the given time into the schedule's timezone, if one is specified.
+	// Save the original timezone so we can convert back after we find a time.
+	// Note that schedules without a time zone specified (time.Local) are treated
+	// as local to the time provided.
+	origLocation := t.Location()
+	loc := s.Location
+	if loc == time.Local {
+		loc = t.Location()
+	}
+	if s.Location != time.Local {
+		t = t.In(s.Location)
+	}
+
+	// Start at the earliest possible time (the upcoming second).
+	t = t.Add(1*time.Second - time.Duration(t.Nanosecond())*time.Nanosecond)
+
+	// This flag indicates whether a field has been incremented.
+	added := false
+
+	// If no time is found within five years, return zero.
+	yearLimit := t.Year() + 5
+
+WRAP:
+	if t.Year() > yearLimit {
+		return time.Time{}
+	}
+
+	// Find the first applicable month.
+	// If it's this month, then do nothing.
+	for 1<<uint(t.Month())&s.Month == 0 {
+		// If we have to add a month, reset the other parts to 0.
+		if !added {
+			added = true
+			// Otherwise, set the date at the beginning (since the current time is irrelevant).
+			t = time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, loc)
+		}
+		t = t.AddDate(0, 1, 0)
+
+		// Wrapped around.
+		if t.Month() == time.January {
+			goto WRAP
+		}
+	}
+
+	// Now get a day in that month.
+	//
+	// NOTE: This causes issues for daylight savings regimes where midnight does
+	// not exist.  For example: Sao Paulo has DST that transforms midnight on
+	// 11/3 into 1am. Handle that by noticing when the Hour ends up != 0.
+	for !dayMatches(s, t) {
+		if !added {
+			added = true
+			t = time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, loc)
+		}
+		t = t.AddDate(0, 0, 1)
+		// Notice if the hour is no longer midnight due to DST.
+		// Add an hour if it's 23, subtract an hour if it's 1.
+		if t.Hour() != 0 {
+			if t.Hour() > 12 {
+				t = t.Add(time.Duration(24-t.Hour()) * time.Hour)
+			} else {
+				t = t.Add(time.Duration(-t.Hour()) * time.Hour)
+			}
+		}
+
+		if t.Day() == 1 {
+			goto WRAP
+		}
+	}
+
+	for 1<<uint(t.Hour())&s.Hour == 0 {
+		if !added {
+			added = true
+			t = time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), 0, 0, 0, loc)
+		}
+		t = t.Add(1 * time.Hour)
+
+		if t.Hour() == 0 {
+			goto WRAP
+		}
+	}
+
+	for 1<<uint(t.Minute())&s.Minute == 0 {
+		if !added {
+			added = true
+			t = t.Truncate(time.Minute)
+		}
+		t = t.Add(1 * time.Minute)
+
+		if t.Minute() == 0 {
+			goto WRAP
+		}
+	}
+
+	for 1<<uint(t.Second())&s.Second == 0 {
+		if !added {
+			added = true
+			t = t.Truncate(time.Second)
+		}
+		t = t.Add(1 * time.Second)
+
+		if t.Second() == 0 {
+			goto WRAP
+		}
+	}
+
+	return t.In(origLocation)
+}
+
+// dayMatches returns true if the schedule's day-of-week and day-of-month
+// restrictions are satisfied by the given time.
+func dayMatches(s *SpecSchedule, t time.Time) bool {
+	var (
+		domMatch bool = 1<<uint(t.Day())&s.Dom > 0
+		dowMatch bool = 1<<uint(t.Weekday())&s.Dow > 0
+	)
+	if s.Dom&starBit > 0 || s.Dow&starBit > 0 {
+		return domMatch && dowMatch
+	}
+	return domMatch || dowMatch
+}

vendor/github.com/uptrace/bun/CHANGELOG.md

@@ -1,3 +1,26 @@
+## [1.0.20](https://github.com/uptrace/bun/compare/v1.0.19...v1.0.20) (2021-12-19)
+
+
+### Bug Fixes
+
+* add Event.QueryTemplate and change Event.Query to be always formatted ([52b1ccd](https://github.com/uptrace/bun/commit/52b1ccdf3578418aa427adef9dcf942d90ae4fdd))
+* change GetTableName to return formatted table name in case ModelTableExpr ([95144dd](https://github.com/uptrace/bun/commit/95144dde937b4ac88b36b0bd8b01372421069b44))
+* change ScanAndCount to work with transactions ([5b3f2c0](https://github.com/uptrace/bun/commit/5b3f2c021c424da366caffd33589e8adde821403))
+* **dbfixture:** directly call funcs bypassing template eval ([a61974b](https://github.com/uptrace/bun/commit/a61974ba2d24361c5357fb9bda1f3eceec5a45cd))
+* don't append CASCADE by default in drop table/column queries ([26457ea](https://github.com/uptrace/bun/commit/26457ea5cb20862d232e6e5fa4dbdeac5d444bf1))
+* **migrate:** mark migrations as applied on error so the migration can be rolled back ([8ce33fb](https://github.com/uptrace/bun/commit/8ce33fbbac8e33077c20daf19a14c5ff2291bcae))
+* respect nullzero when appending struct fields. Fixes [#339](https://github.com/uptrace/bun/issues/339) ([ffd02f3](https://github.com/uptrace/bun/commit/ffd02f3170b3cccdd670a48d563cfb41094c05d6))
+* reuse tx for relation join ([#366](https://github.com/uptrace/bun/issues/366)) ([60bdb1a](https://github.com/uptrace/bun/commit/60bdb1ac84c0a699429eead3b7fdfbf14fe69ac6))
+
+
+### Features
+
+* add `Dialect()` to Transaction and IDB interface ([693f1e1](https://github.com/uptrace/bun/commit/693f1e135999fc31cf83b99a2530a695b20f4e1b))
+* add model embedding via embed:prefix_ ([9a2cedc](https://github.com/uptrace/bun/commit/9a2cedc8b08fa8585d4bfced338bd0a40d736b1d))
+* change the default logoutput to stderr ([4bf5773](https://github.com/uptrace/bun/commit/4bf577382f19c64457cbf0d64490401450954654)), closes [#349](https://github.com/uptrace/bun/issues/349)
+
+
+
 ## [1.0.19](https://github.com/uptrace/bun/compare/v1.0.18...v1.0.19) (2021-11-30)
 
 

vendor/github.com/uptrace/bun/README.md

@@ -43,8 +43,147 @@ Projects using Bun:
   demand.
 - [RealWorld app](https://github.com/go-bun/bun-realworld-app)
 
+## Benchmark
+[https://github.com/davars/dbeval](https://github.com/davars/dbeval)
 <details>
-  <summary>github.com/frederikhors/orm-benchmark results</summary>
+<summary>results</summary>
+
+```
+BenchmarkInsert
+BenchmarkInsert/*dbeval.Memory/Authors
+BenchmarkInsert/*dbeval.Memory/Authors-4         	   84450	     12104 ns/op	    2623 B/op	      70 allocs/op
+BenchmarkInsert/*dbeval.Xorm/Authors
+BenchmarkInsert/*dbeval.Xorm/Authors-4           	    7291	    153505 ns/op	    9024 B/op	     311 allocs/op
+BenchmarkInsert/*dbeval.UpperDB/Authors
+BenchmarkInsert/*dbeval.UpperDB/Authors-4        	    4608	    223672 ns/op	   24160 B/op	    1100 allocs/op
+BenchmarkInsert/*dbeval.Bun/Authors
+BenchmarkInsert/*dbeval.Bun/Authors-4            	    6034	    186439 ns/op	    6818 B/op	      80 allocs/op
+BenchmarkInsert/*dbeval.PQ/Authors
+BenchmarkInsert/*dbeval.PQ/Authors-4             	    1141	    907494 ns/op	    6487 B/op	     193 allocs/op
+BenchmarkInsert/*dbeval.SQLX/Authors
+BenchmarkInsert/*dbeval.SQLX/Authors-4           	    1165	    916987 ns/op	   10089 B/op	     271 allocs/op
+BenchmarkInsert/*dbeval.Ozzo/Authors
+BenchmarkInsert/*dbeval.Ozzo/Authors-4           	    1105	   1058082 ns/op	   27826 B/op	     588 allocs/op
+BenchmarkInsert/*dbeval.PGXStdlib/Authors
+BenchmarkInsert/*dbeval.PGXStdlib/Authors-4      	    1228	    900207 ns/op	    6032 B/op	     180 allocs/op
+BenchmarkInsert/*dbeval.Gorm/Authors
+BenchmarkInsert/*dbeval.Gorm/Authors-4           	     946	   1184285 ns/op	   35634 B/op	     918 allocs/op
+BenchmarkInsert/*dbeval.PGX/Authors
+BenchmarkInsert/*dbeval.PGX/Authors-4            	    1116	    923728 ns/op	    3839 B/op	     130 allocs/op
+BenchmarkInsert/*dbeval.DBR/Authors
+BenchmarkInsert/*dbeval.DBR/Authors-4            	    5800	    183982 ns/op	    8646 B/op	     230 allocs/op
+BenchmarkInsert/*dbeval.GoPG/Authors
+BenchmarkInsert/*dbeval.GoPG/Authors-4           	    6110	    173923 ns/op	    2906 B/op	      87 allocs/op
+
+BenchmarkInsert/*dbeval.DBR/Articles
+BenchmarkInsert/*dbeval.DBR/Articles-4           	    1706	    684466 ns/op	  133346 B/op	    1614 allocs/op
+BenchmarkInsert/*dbeval.PQ/Articles
+BenchmarkInsert/*dbeval.PQ/Articles-4            	     884	   1249791 ns/op	  100403 B/op	    1491 allocs/op
+BenchmarkInsert/*dbeval.PGX/Articles
+BenchmarkInsert/*dbeval.PGX/Articles-4           	     916	   1288143 ns/op	   83539 B/op	    1392 allocs/op
+BenchmarkInsert/*dbeval.GoPG/Articles
+BenchmarkInsert/*dbeval.GoPG/Articles-4          	    1726	    622639 ns/op	   78638 B/op	    1359 allocs/op
+BenchmarkInsert/*dbeval.SQLX/Articles
+BenchmarkInsert/*dbeval.SQLX/Articles-4          	     860	   1262599 ns/op	   92030 B/op	    1574 allocs/op
+BenchmarkInsert/*dbeval.Gorm/Articles
+BenchmarkInsert/*dbeval.Gorm/Articles-4          	     782	   1421550 ns/op	  136534 B/op	    2411 allocs/op
+BenchmarkInsert/*dbeval.PGXStdlib/Articles
+BenchmarkInsert/*dbeval.PGXStdlib/Articles-4     	     938	   1230576 ns/op	   86743 B/op	    1441 allocs/op
+BenchmarkInsert/*dbeval.Bun/Articles
+BenchmarkInsert/*dbeval.Bun/Articles-4           	    1843	    626681 ns/op	  101610 B/op	    1323 allocs/op
+BenchmarkInsert/*dbeval.Xorm/Articles
+BenchmarkInsert/*dbeval.Xorm/Articles-4          	    1677	    650244 ns/op	  126677 B/op	    1752 allocs/op
+BenchmarkInsert/*dbeval.Memory/Articles
+BenchmarkInsert/*dbeval.Memory/Articles-4        	    1988	   1223308 ns/op	   77576 B/op	    1310 allocs/op
+BenchmarkInsert/*dbeval.UpperDB/Articles
+BenchmarkInsert/*dbeval.UpperDB/Articles-4       	    1696	    687130 ns/op	  139680 B/op	    2862 allocs/op
+BenchmarkInsert/*dbeval.Ozzo/Articles
+BenchmarkInsert/*dbeval.Ozzo/Articles-4          	     697	   1496859 ns/op	  114780 B/op	    1950 allocs/op
+
+BenchmarkFindAuthorByID
+BenchmarkFindAuthorByID/*dbeval.UpperDB
+BenchmarkFindAuthorByID/*dbeval.UpperDB-4        	   10184	    117527 ns/op	    9953 B/op	     441 allocs/op
+BenchmarkFindAuthorByID/*dbeval.Bun
+BenchmarkFindAuthorByID/*dbeval.Bun-4            	   20716	     54261 ns/op	    5096 B/op	      15 allocs/op
+BenchmarkFindAuthorByID/*dbeval.Ozzo
+BenchmarkFindAuthorByID/*dbeval.Ozzo-4           	   11166	     91043 ns/op	    3088 B/op	      64 allocs/op
+BenchmarkFindAuthorByID/*dbeval.PQ
+BenchmarkFindAuthorByID/*dbeval.PQ-4             	   13875	     86171 ns/op	     844 B/op	      24 allocs/op
+BenchmarkFindAuthorByID/*dbeval.PGX
+BenchmarkFindAuthorByID/*dbeval.PGX-4            	   13846	     79983 ns/op	     719 B/op	      15 allocs/op
+BenchmarkFindAuthorByID/*dbeval.Memory
+BenchmarkFindAuthorByID/*dbeval.Memory-4         	14113720	        82.33 ns/op	       0 B/op	       0 allocs/op
+BenchmarkFindAuthorByID/*dbeval.Xorm
+BenchmarkFindAuthorByID/*dbeval.Xorm-4           	   12027	     98519 ns/op	    3633 B/op	     106 allocs/op
+BenchmarkFindAuthorByID/*dbeval.Gorm
+BenchmarkFindAuthorByID/*dbeval.Gorm-4           	   11521	    102241 ns/op	    6592 B/op	     143 allocs/op
+BenchmarkFindAuthorByID/*dbeval.PGXStdlib
+BenchmarkFindAuthorByID/*dbeval.PGXStdlib-4      	   13933	     82626 ns/op	    1174 B/op	      28 allocs/op
+BenchmarkFindAuthorByID/*dbeval.DBR
+BenchmarkFindAuthorByID/*dbeval.DBR-4            	   21920	     51175 ns/op	    1756 B/op	      39 allocs/op
+BenchmarkFindAuthorByID/*dbeval.SQLX
+BenchmarkFindAuthorByID/*dbeval.SQLX-4           	   13603	     80788 ns/op	    1327 B/op	      32 allocs/op
+BenchmarkFindAuthorByID/*dbeval.GoPG
+BenchmarkFindAuthorByID/*dbeval.GoPG-4           	   23174	     50042 ns/op	     869 B/op	      17 allocs/op
+
+BenchmarkFindAuthorByName
+BenchmarkFindAuthorByName/*dbeval.SQLX
+BenchmarkFindAuthorByName/*dbeval.SQLX-4         	    1070	   1065272 ns/op	  126348 B/op	    4018 allocs/op
+BenchmarkFindAuthorByName/*dbeval.Bun
+BenchmarkFindAuthorByName/*dbeval.Bun-4          	     877	   1231377 ns/op	  115803 B/op	    5005 allocs/op
+BenchmarkFindAuthorByName/*dbeval.Xorm
+BenchmarkFindAuthorByName/*dbeval.Xorm-4         	     471	   2345445 ns/op	  455711 B/op	   19080 allocs/op
+BenchmarkFindAuthorByName/*dbeval.DBR
+BenchmarkFindAuthorByName/*dbeval.DBR-4          	     954	   1089977 ns/op	  120070 B/op	    6023 allocs/op
+BenchmarkFindAuthorByName/*dbeval.PQ
+BenchmarkFindAuthorByName/*dbeval.PQ-4           	    1333	    784400 ns/op	   87159 B/op	    4006 allocs/op
+BenchmarkFindAuthorByName/*dbeval.GoPG
+BenchmarkFindAuthorByName/*dbeval.GoPG-4         	    1580	    770966 ns/op	   87525 B/op	    3028 allocs/op
+BenchmarkFindAuthorByName/*dbeval.UpperDB
+BenchmarkFindAuthorByName/*dbeval.UpperDB-4      	     789	   1314164 ns/op	  190689 B/op	    6428 allocs/op
+BenchmarkFindAuthorByName/*dbeval.Ozzo
+BenchmarkFindAuthorByName/*dbeval.Ozzo-4         	     948	   1255282 ns/op	  238764 B/op	    6053 allocs/op
+BenchmarkFindAuthorByName/*dbeval.PGXStdlib
+BenchmarkFindAuthorByName/*dbeval.PGXStdlib-4    	    1279	    920391 ns/op	  126163 B/op	    4014 allocs/op
+BenchmarkFindAuthorByName/*dbeval.PGX
+BenchmarkFindAuthorByName/*dbeval.PGX-4          	    1364	    780970 ns/op	  101967 B/op	    2028 allocs/op
+BenchmarkFindAuthorByName/*dbeval.Gorm
+BenchmarkFindAuthorByName/*dbeval.Gorm-4         	     340	   3445818 ns/op	 1573637 B/op	   27102 allocs/op
+BenchmarkFindAuthorByName/*dbeval.Memory
+BenchmarkFindAuthorByName/*dbeval.Memory-4       	38081223	        31.24 ns/op	       0 B/op	       0 allocs/op
+
+BenchmarkRecentArticles
+BenchmarkRecentArticles/*dbeval.PGXStdlib
+BenchmarkRecentArticles/*dbeval.PGXStdlib-4      	     358	   3344119 ns/op	 3425578 B/op	   14177 allocs/op
+BenchmarkRecentArticles/*dbeval.GoPG
+BenchmarkRecentArticles/*dbeval.GoPG-4           	     364	   3156372 ns/op	 1794091 B/op	   10032 allocs/op
+BenchmarkRecentArticles/*dbeval.Xorm
+BenchmarkRecentArticles/*dbeval.Xorm-4           	     157	   7567835 ns/op	 5018011 B/op	   81425 allocs/op
+BenchmarkRecentArticles/*dbeval.Gorm
+BenchmarkRecentArticles/*dbeval.Gorm-4           	     139	   7980084 ns/op	 6776277 B/op	   85418 allocs/op
+BenchmarkRecentArticles/*dbeval.SQLX
+BenchmarkRecentArticles/*dbeval.SQLX-4           	     338	   3289802 ns/op	 3425890 B/op	   14181 allocs/op
+BenchmarkRecentArticles/*dbeval.Ozzo
+BenchmarkRecentArticles/*dbeval.Ozzo-4           	     320	   3508322 ns/op	 4025966 B/op	   18207 allocs/op
+BenchmarkRecentArticles/*dbeval.DBR
+BenchmarkRecentArticles/*dbeval.DBR-4            	     237	   5248644 ns/op	 3331003 B/op	   21370 allocs/op
+BenchmarkRecentArticles/*dbeval.Bun
+BenchmarkRecentArticles/*dbeval.Bun-4            	     280	   4528582 ns/op	 1864362 B/op	   15965 allocs/op
+BenchmarkRecentArticles/*dbeval.UpperDB
+BenchmarkRecentArticles/*dbeval.UpperDB-4        	     297	   3704663 ns/op	 3607287 B/op	   18542 allocs/op
+BenchmarkRecentArticles/*dbeval.PQ
+BenchmarkRecentArticles/*dbeval.PQ-4             	     308	   3489229 ns/op	 3277050 B/op	   17359 allocs/op
+BenchmarkRecentArticles/*dbeval.Memory
+BenchmarkRecentArticles/*dbeval.Memory-4         	29590380	        42.27 ns/op	       0 B/op	       0 allocs/op
+BenchmarkRecentArticles/*dbeval.PGX
+BenchmarkRecentArticles/*dbeval.PGX-4            	     356	   3345500 ns/op	 3297316 B/op	    6226 allocs/op
+```
+
+</details>
+
+[https://github.com/frederikhors/orm-benchmark](https://github.com/frederikhors/orm-benchmark)
+<details>
+<summary>results</summary>
 
 ```
   4000 times - Insert

vendor/github.com/uptrace/bun/db.go

@@ -32,6 +32,7 @@ func WithDiscardUnknownColumns() DBOption {
 
 type DB struct {
 	*sql.DB
+
 	dialect  schema.Dialect
 	features feature.Feature
 
@@ -125,7 +126,7 @@ func (db *DB) NewDropColumn() *DropColumnQuery {
 
 func (db *DB) ResetModel(ctx context.Context, models ...interface{}) error {
 	for _, model := range models {
-		if _, err := db.NewDropTable().Model(model).IfExists().Exec(ctx); err != nil {
+		if _, err := db.NewDropTable().Model(model).IfExists().Cascade().Exec(ctx); err != nil {
 			return err
 		}
 		if _, err := db.NewCreateTable().Model(model).Exec(ctx); err != nil {
@@ -226,8 +227,9 @@ func (db *DB) Exec(query string, args ...interface{}) (sql.Result, error) {
 func (db *DB) ExecContext(
 	ctx context.Context, query string, args ...interface{},
 ) (sql.Result, error) {
-	ctx, event := db.beforeQuery(ctx, nil, query, args, nil)
+	formattedQuery := db.format(query, args)
-	res, err := db.DB.ExecContext(ctx, db.format(query, args))
+	ctx, event := db.beforeQuery(ctx, nil, query, args, formattedQuery, nil)
+	res, err := db.DB.ExecContext(ctx, formattedQuery)
 	db.afterQuery(ctx, event, res, err)
 	return res, err
 }
@@ -239,8 +241,9 @@ func (db *DB) Query(query string, args ...interface{}) (*sql.Rows, error) {
 func (db *DB) QueryContext(
 	ctx context.Context, query string, args ...interface{},
 ) (*sql.Rows, error) {
-	ctx, event := db.beforeQuery(ctx, nil, query, args, nil)
+	formattedQuery := db.format(query, args)
-	rows, err := db.DB.QueryContext(ctx, db.format(query, args))
+	ctx, event := db.beforeQuery(ctx, nil, query, args, formattedQuery, nil)
+	rows, err := db.DB.QueryContext(ctx, formattedQuery)
 	db.afterQuery(ctx, event, nil, err)
 	return rows, err
 }
@@ -250,8 +253,9 @@ func (db *DB) QueryRow(query string, args ...interface{}) *sql.Row {
 }
 
 func (db *DB) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row {
-	ctx, event := db.beforeQuery(ctx, nil, query, args, nil)
+	formattedQuery := db.format(query, args)
-	row := db.DB.QueryRowContext(ctx, db.format(query, args))
+	ctx, event := db.beforeQuery(ctx, nil, query, args, formattedQuery, nil)
+	row := db.DB.QueryRowContext(ctx, formattedQuery)
 	db.afterQuery(ctx, event, nil, row.Err())
 	return row
 }
@@ -281,8 +285,9 @@ func (db *DB) Conn(ctx context.Context) (Conn, error) {
 func (c Conn) ExecContext(
 	ctx context.Context, query string, args ...interface{},
 ) (sql.Result, error) {
-	ctx, event := c.db.beforeQuery(ctx, nil, query, args, nil)
+	formattedQuery := c.db.format(query, args)
-	res, err := c.Conn.ExecContext(ctx, c.db.format(query, args))
+	ctx, event := c.db.beforeQuery(ctx, nil, query, args, formattedQuery, nil)
+	res, err := c.Conn.ExecContext(ctx, formattedQuery)
 	c.db.afterQuery(ctx, event, res, err)
 	return res, err
 }
@@ -290,19 +295,25 @@ func (c Conn) ExecContext(
 func (c Conn) QueryContext(
 	ctx context.Context, query string, args ...interface{},
 ) (*sql.Rows, error) {
-	ctx, event := c.db.beforeQuery(ctx, nil, query, args, nil)
+	formattedQuery := c.db.format(query, args)
-	rows, err := c.Conn.QueryContext(ctx, c.db.format(query, args))
+	ctx, event := c.db.beforeQuery(ctx, nil, query, args, formattedQuery, nil)
+	rows, err := c.Conn.QueryContext(ctx, formattedQuery)
 	c.db.afterQuery(ctx, event, nil, err)
 	return rows, err
 }
 
 func (c Conn) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row {
-	ctx, event := c.db.beforeQuery(ctx, nil, query, args, nil)
+	formattedQuery := c.db.format(query, args)
-	row := c.Conn.QueryRowContext(ctx, c.db.format(query, args))
+	ctx, event := c.db.beforeQuery(ctx, nil, query, args, formattedQuery, nil)
+	row := c.Conn.QueryRowContext(ctx, formattedQuery)
 	c.db.afterQuery(ctx, event, nil, row.Err())
 	return row
 }
 
+func (c Conn) Dialect() schema.Dialect {
+	return c.db.Dialect()
+}
+
 func (c Conn) NewValues(model interface{}) *ValuesQuery {
 	return NewValuesQuery(c.db, model).Conn(c)
 }
@@ -408,7 +419,7 @@ func (db *DB) Begin() (Tx, error) {
 }
 
 func (db *DB) BeginTx(ctx context.Context, opts *sql.TxOptions) (Tx, error) {
-	ctx, event := db.beforeQuery(ctx, nil, "BEGIN", nil, nil)
+	ctx, event := db.beforeQuery(ctx, nil, "BEGIN", nil, "BEGIN", nil)
 	tx, err := db.DB.BeginTx(ctx, opts)
 	db.afterQuery(ctx, event, nil, err)
 	if err != nil {
@@ -422,14 +433,14 @@ func (db *DB) BeginTx(ctx context.Context, opts *sql.TxOptions) (Tx, error) {
 }
 
 func (tx Tx) Commit() error {
-	ctx, event := tx.db.beforeQuery(tx.ctx, nil, "COMMIT", nil, nil)
+	ctx, event := tx.db.beforeQuery(tx.ctx, nil, "COMMIT", nil, "COMMIT", nil)
 	err := tx.Tx.Commit()
 	tx.db.afterQuery(ctx, event, nil, err)
 	return err
 }
 
 func (tx Tx) Rollback() error {
-	ctx, event := tx.db.beforeQuery(tx.ctx, nil, "ROLLBACK", nil, nil)
+	ctx, event := tx.db.beforeQuery(tx.ctx, nil, "ROLLBACK", nil, "ROLLBACK", nil)
 	err := tx.Tx.Rollback()
 	tx.db.afterQuery(ctx, event, nil, err)
 	return err
@@ -442,8 +453,9 @@ func (tx Tx) Exec(query string, args ...interface{}) (sql.Result, error) {
 func (tx Tx) ExecContext(
 	ctx context.Context, query string, args ...interface{},
 ) (sql.Result, error) {
-	ctx, event := tx.db.beforeQuery(ctx, nil, query, args, nil)
+	formattedQuery := tx.db.format(query, args)
-	res, err := tx.Tx.ExecContext(ctx, tx.db.format(query, args))
+	ctx, event := tx.db.beforeQuery(ctx, nil, query, args, formattedQuery, nil)
+	res, err := tx.Tx.ExecContext(ctx, formattedQuery)
 	tx.db.afterQuery(ctx, event, res, err)
 	return res, err
 }
@@ -455,8 +467,9 @@ func (tx Tx) Query(query string, args ...interface{}) (*sql.Rows, error) {
 func (tx Tx) QueryContext(
 	ctx context.Context, query string, args ...interface{},
 ) (*sql.Rows, error) {
-	ctx, event := tx.db.beforeQuery(ctx, nil, query, args, nil)
+	formattedQuery := tx.db.format(query, args)
-	rows, err := tx.Tx.QueryContext(ctx, tx.db.format(query, args))
+	ctx, event := tx.db.beforeQuery(ctx, nil, query, args, formattedQuery, nil)
+	rows, err := tx.Tx.QueryContext(ctx, formattedQuery)
 	tx.db.afterQuery(ctx, event, nil, err)
 	return rows, err
 }
@@ -466,14 +479,19 @@ func (tx Tx) QueryRow(query string, args ...interface{}) *sql.Row {
 }
 
 func (tx Tx) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row {
-	ctx, event := tx.db.beforeQuery(ctx, nil, query, args, nil)
+	formattedQuery := tx.db.format(query, args)
-	row := tx.Tx.QueryRowContext(ctx, tx.db.format(query, args))
+	ctx, event := tx.db.beforeQuery(ctx, nil, query, args, formattedQuery, nil)
+	row := tx.Tx.QueryRowContext(ctx, formattedQuery)
 	tx.db.afterQuery(ctx, event, nil, row.Err())
 	return row
 }
 
 //------------------------------------------------------------------------------
 
+func (tx Tx) Dialect() schema.Dialect {
+	return tx.db.Dialect()
+}
+
 func (tx Tx) NewValues(model interface{}) *ValuesQuery {
 	return NewValuesQuery(tx.db, model).Conn(tx)
 }

vendor/github.com/uptrace/bun/hook.go

@@ -13,9 +13,10 @@ import (
 type QueryEvent struct {
 	DB *DB
 
-	QueryAppender schema.QueryAppender // Deprecated: use IQuery instead
+	QueryAppender schema.QueryAppender // DEPRECATED: use IQuery instead
 	IQuery        Query
 	Query         string
+	QueryTemplate string
 	QueryArgs     []interface{}
 	Model         Model
 
@@ -51,8 +52,9 @@ type QueryHook interface {
 func (db *DB) beforeQuery(
 	ctx context.Context,
 	iquery Query,
-	query string,
+	queryTemplate string,
 	queryArgs []interface{},
+	query string,
 	model Model,
 ) (context.Context, *QueryEvent) {
 	atomic.AddUint32(&db.stats.Queries, 1)
@@ -68,6 +70,7 @@ func (db *DB) beforeQuery(
 		QueryAppender: iquery,
 		IQuery:        iquery,
 		Query:         query,
+		QueryTemplate: queryTemplate,
 		QueryArgs:     queryArgs,
 
 		StartTime: time.Now(),

vendor/github.com/uptrace/bun/migrate/migrator.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"path/filepath"
 	"regexp"
 	"time"
@@ -59,11 +58,16 @@ func (m *Migrator) DB() *bun.DB {
 
 // MigrationsWithStatus returns migrations with status in ascending order.
 func (m *Migrator) MigrationsWithStatus(ctx context.Context) (MigrationSlice, error) {
+	sorted, _, err := m.migrationsWithStatus(ctx)
+	return sorted, err
+}
+
+func (m *Migrator) migrationsWithStatus(ctx context.Context) (MigrationSlice, int64, error) {
 	sorted := m.migrations.Sorted()
 
 	applied, err := m.selectAppliedMigrations(ctx)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	appliedMap := migrationMap(applied)
@@ -76,7 +80,7 @@ func (m *Migrator) MigrationsWithStatus(ctx context.Context) (MigrationSlice, er
 		}
 	}
 
-	return sorted, nil
+	return sorted, applied.LastGroupID(), nil
 }
 
 func (m *Migrator) Init(ctx context.Context) error {
@@ -128,7 +132,7 @@ func (m *Migrator) Migrate(ctx context.Context, opts ...MigrationOption) (*Migra
 	}
 	defer m.Unlock(ctx) //nolint:errcheck
 
-	migrations, err := m.MigrationsWithStatus(ctx)
+	migrations, lastGroupID, err := m.migrationsWithStatus(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -139,21 +143,22 @@ func (m *Migrator) Migrate(ctx context.Context, opts ...MigrationOption) (*Migra
 	if len(group.Migrations) == 0 {
 		return group, nil
 	}
-	group.ID = migrations.LastGroupID() + 1
+	group.ID = lastGroupID + 1
 
 	for i := range group.Migrations {
 		migration := &group.Migrations[i]
 		migration.GroupID = group.ID
 
+		// Always mark migration as applied so the rollback has a chance to fix the database.
+		if err := m.MarkApplied(ctx, migration); err != nil {
+			return nil, err
+		}
+
 		if !cfg.nop && migration.Up != nil {
 			if err := migration.Up(ctx, m.db); err != nil {
 				return group, err
 			}
 		}
-
-		if err := m.MarkApplied(ctx, migration); err != nil {
-			return nil, err
-		}
 	}
 
 	return group, nil
@@ -195,36 +200,6 @@ func (m *Migrator) Rollback(ctx context.Context, opts ...MigrationOption) (*Migr
 	return lastGroup, nil
 }
 
-type MigrationStatus struct {
-	Migrations    MigrationSlice
-	NewMigrations MigrationSlice
-	LastGroup     *MigrationGroup
-}
-
-func (m *Migrator) Status(ctx context.Context) (*MigrationStatus, error) {
-	log.Printf(
-		"DEPRECATED: bun: replace Status(ctx) with " +
-			"MigrationsWithStatus(ctx)")
-
-	migrations, err := m.MigrationsWithStatus(ctx)
-	if err != nil {
-		return nil, err
-	}
-	return &MigrationStatus{
-		Migrations:    migrations,
-		NewMigrations: migrations.Unapplied(),
-		LastGroup:     migrations.LastGroup(),
-	}, nil
-}
-
-func (m *Migrator) MarkCompleted(ctx context.Context) (*MigrationGroup, error) {
-	log.Printf(
-		"DEPRECATED: bun: replace MarkCompleted(ctx) with " +
-			"Migrate(ctx, migrate.WithNopMigration())")
-
-	return m.Migrate(ctx, WithNopMigration())
-}
-
 type goMigrationConfig struct {
 	packageName string
 }
@@ -320,7 +295,7 @@ func (m *Migrator) genMigrationName(name string) (string, error) {
 	return fmt.Sprintf("%s_%s", version, name), nil
 }
 
-// MarkApplied marks the migration as applied (applied).
+// MarkApplied marks the migration as applied (completed).
 func (m *Migrator) MarkApplied(ctx context.Context, migration *Migration) error {
 	_, err := m.db.NewInsert().Model(migration).
 		ModelTableExpr(m.table).

vendor/github.com/uptrace/bun/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bun",
-  "version": "1.0.19",
+  "version": "1.0.20",
   "main": "index.js",
   "repository": "git@github.com:uptrace/bun.git",
   "author": "Vladimir Mihailenco <vladimir.webdev@gmail.com>",

vendor/github.com/uptrace/bun/query_base.go

@@ -43,6 +43,7 @@ var (
 // IDB is a common interface for *bun.DB, bun.Conn, and bun.Tx.
 type IDB interface {
 	IConn
+	Dialect() schema.Dialect
 
 	NewValues(model interface{}) *ValuesQuery
 	NewSelect() *SelectQuery
@@ -59,9 +60,9 @@ type IDB interface {
 }
 
 var (
-	_ IConn = (*DB)(nil)
+	_ IDB = (*DB)(nil)
-	_ IConn = (*Conn)(nil)
+	_ IDB = (*Conn)(nil)
-	_ IConn = (*Tx)(nil)
+	_ IDB = (*Tx)(nil)
 )
 
 type baseQuery struct {
@@ -75,7 +76,7 @@ type baseQuery struct {
 	table      *schema.Table
 
 	with           []withQuery
-	modelTable schema.QueryWithArgs
+	modelTableName schema.QueryWithArgs
 	tables         []schema.QueryWithArgs
 	columns        []schema.QueryWithArgs
 
@@ -86,13 +87,6 @@ func (q *baseQuery) DB() *DB {
 	return q.db
 }
 
-type query interface {
-	GetModel() Model
-	GetTableName() string
-}
-
-var _ query = (*baseQuery)(nil)
-
 func (q *baseQuery) GetModel() Model {
 	return q.model
 }
@@ -103,15 +97,16 @@ func (q *baseQuery) GetTableName() string {
 	}
 
 	for _, wq := range q.with {
-		if v, ok := wq.query.(query); ok {
+		if v, ok := wq.query.(Query); ok {
 			if model := v.GetModel(); model != nil {
 				return v.GetTableName()
 			}
 		}
 	}
 
-	if q.modelTable.Query != "" {
+	if q.modelTableName.Query != "" {
-		return q.modelTable.Query
+		b, _ := q.modelTableName.AppendQuery(q.db.fmter, nil)
+		return string(b)
 	}
 	if len(q.tables) > 0 {
 		return q.tables[0].Query
@@ -304,8 +299,8 @@ func (q *baseQuery) _excludeColumn(column string) bool {
 //------------------------------------------------------------------------------
 
 func (q *baseQuery) modelHasTableName() bool {
-	if !q.modelTable.IsZero() {
+	if !q.modelTableName.IsZero() {
-		return q.modelTable.Query != ""
+		return q.modelTableName.Query != ""
 	}
 	return q.table != nil
 }
@@ -332,8 +327,8 @@ func (q *baseQuery) _appendTables(
 	startLen := len(b)
 
 	if q.modelHasTableName() {
-		if !q.modelTable.IsZero() {
+		if !q.modelTableName.IsZero() {
-			b, err = q.modelTable.AppendQuery(fmter, b)
+			b, err = q.modelTableName.AppendQuery(fmter, b)
 			if err != nil {
 				return nil, err
 			}
@@ -372,8 +367,8 @@ func (q *baseQuery) appendFirstTableWithAlias(
 func (q *baseQuery) _appendFirstTable(
 	fmter schema.Formatter, b []byte, withAlias bool,
 ) ([]byte, error) {
-	if !q.modelTable.IsZero() {
+	if !q.modelTableName.IsZero() {
-		return q.modelTable.AppendQuery(fmter, b)
+		return q.modelTableName.AppendQuery(fmter, b)
 	}
 
 	if q.table != nil {
@@ -473,7 +468,7 @@ func (q *baseQuery) scan(
 	model Model,
 	hasDest bool,
 ) (sql.Result, error) {
-	ctx, event := q.db.beforeQuery(ctx, iquery, query, nil, q.model)
+	ctx, event := q.db.beforeQuery(ctx, iquery, query, nil, query, q.model)
 
 	rows, err := q.conn.QueryContext(ctx, query)
 	if err != nil {
@@ -503,16 +498,10 @@ func (q *baseQuery) exec(
 	iquery Query,
 	query string,
 ) (sql.Result, error) {
-	ctx, event := q.db.beforeQuery(ctx, iquery, query, nil, q.model)
+	ctx, event := q.db.beforeQuery(ctx, iquery, query, nil, query, q.model)
-
 	res, err := q.conn.ExecContext(ctx, query)
-	if err != nil {
 	q.db.afterQuery(ctx, event, nil, err)
 	return res, err
-	}
-
-	q.db.afterQuery(ctx, event, res, err)
-	return res, nil
 }
 
 //------------------------------------------------------------------------------
@@ -607,34 +596,30 @@ func (q *whereBaseQuery) addWhereCols(cols []string) {
 		q.setErr(err)
 		return
 	}
-
-	var fields []*schema.Field
-
-	if cols == nil {
-		if err := q.table.CheckPKs(); err != nil {
-			q.setErr(err)
-			return
-		}
-		fields = q.table.PKs
-	} else {
-		fields = make([]*schema.Field, len(cols))
-		for i, col := range cols {
-			field, err := q.table.Field(col)
-			if err != nil {
-				q.setErr(err)
-				return
-			}
-			fields[i] = field
-		}
-	}
-
 	if q.whereFields != nil {
 		err := errors.New("bun: WherePK can only be called once")
 		q.setErr(err)
 		return
 	}
 
-	q.whereFields = fields
+	if cols == nil {
+		if err := q.table.CheckPKs(); err != nil {
+			q.setErr(err)
+			return
+		}
+		q.whereFields = q.table.PKs
+		return
+	}
+
+	q.whereFields = make([]*schema.Field, len(cols))
+	for i, col := range cols {
+		field, err := q.table.Field(col)
+		if err != nil {
+			q.setErr(err)
+			return
+		}
+		q.whereFields[i] = field
+	}
 }
 
 func (q *whereBaseQuery) mustAppendWhere(
@@ -951,6 +936,7 @@ func (q setQuery) appendSet(fmter schema.Formatter, b []byte) (_ []byte, err err
 //------------------------------------------------------------------------------
 
 type cascadeQuery struct {
+	cascade  bool
 	restrict bool
 }
 
@@ -958,10 +944,11 @@ func (q cascadeQuery) appendCascade(fmter schema.Formatter, b []byte) []byte {
 	if !fmter.HasFeature(feature.TableCascade) {
 		return b
 	}
+	if q.cascade {
+		b = append(b, " CASCADE"...)
+	}
 	if q.restrict {
 		b = append(b, " RESTRICT"...)
-	} else {
-		b = append(b, " CASCADE"...)
 	}
 	return b
 }

vendor/github.com/uptrace/bun/query_column_add.go

@@ -15,6 +15,8 @@ type AddColumnQuery struct {
 	ifNotExists bool
 }
 
+var _ Query = (*AddColumnQuery)(nil)
+
 func NewAddColumnQuery(db *DB) *AddColumnQuery {
 	q := &AddColumnQuery{
 		baseQuery: baseQuery{
@@ -50,7 +52,7 @@ func (q *AddColumnQuery) TableExpr(query string, args ...interface{}) *AddColumn
 }
 
 func (q *AddColumnQuery) ModelTableExpr(query string, args ...interface{}) *AddColumnQuery {
-	q.modelTable = schema.SafeQuery(query, args)
+	q.modelTableName = schema.SafeQuery(query, args)
 	return q
 }
 

vendor/github.com/uptrace/bun/query_column_drop.go

@@ -13,6 +13,8 @@ type DropColumnQuery struct {
 	baseQuery
 }
 
+var _ Query = (*DropColumnQuery)(nil)
+
 func NewDropColumnQuery(db *DB) *DropColumnQuery {
 	q := &DropColumnQuery{
 		baseQuery: baseQuery{
@@ -48,7 +50,7 @@ func (q *DropColumnQuery) TableExpr(query string, args ...interface{}) *DropColu
 }
 
 func (q *DropColumnQuery) ModelTableExpr(query string, args ...interface{}) *DropColumnQuery {
-	q.modelTable = schema.SafeQuery(query, args)
+	q.modelTableName = schema.SafeQuery(query, args)
 	return q
 }
 

vendor/github.com/uptrace/bun/query_delete.go

@@ -15,6 +15,8 @@ type DeleteQuery struct {
 	returningQuery
 }
 
+var _ Query = (*DeleteQuery)(nil)
+
 func NewDeleteQuery(db *DB) *DeleteQuery {
 	q := &DeleteQuery{
 		whereBaseQuery: whereBaseQuery{
@@ -60,7 +62,7 @@ func (q *DeleteQuery) TableExpr(query string, args ...interface{}) *DeleteQuery
 }
 
 func (q *DeleteQuery) ModelTableExpr(query string, args ...interface{}) *DeleteQuery {
-	q.modelTable = schema.SafeQuery(query, args)
+	q.modelTableName = schema.SafeQuery(query, args)
 	return q
 }
 
@@ -134,6 +136,10 @@ func (q *DeleteQuery) Operation() string {
 }
 
 func (q *DeleteQuery) AppendQuery(fmter schema.Formatter, b []byte) (_ []byte, err error) {
+	if q.err != nil {
+		return nil, q.err
+	}
+
 	fmter = formatterWithModel(fmter, q)
 
 	if q.isSoftDelete() {

vendor/github.com/uptrace/bun/query_index_create.go

@@ -22,6 +22,8 @@ type CreateIndexQuery struct {
 	include []schema.QueryWithArgs
 }
 
+var _ Query = (*CreateIndexQuery)(nil)
+
 func NewCreateIndexQuery(db *DB) *CreateIndexQuery {
 	q := &CreateIndexQuery{
 		whereBaseQuery: whereBaseQuery{
@@ -86,7 +88,7 @@ func (q *CreateIndexQuery) TableExpr(query string, args ...interface{}) *CreateI
 }
 
 func (q *CreateIndexQuery) ModelTableExpr(query string, args ...interface{}) *CreateIndexQuery {
-	q.modelTable = schema.SafeQuery(query, args)
+	q.modelTableName = schema.SafeQuery(query, args)
 	return q
 }
 

vendor/github.com/uptrace/bun/query_index_drop.go

@@ -18,6 +18,8 @@ type DropIndexQuery struct {
 	index schema.QueryWithArgs
 }
 
+var _ Query = (*DropIndexQuery)(nil)
+
 func NewDropIndexQuery(db *DB) *DropIndexQuery {
 	q := &DropIndexQuery{
 		baseQuery: baseQuery{
@@ -50,6 +52,11 @@ func (q *DropIndexQuery) IfExists() *DropIndexQuery {
 	return q
 }
 
+func (q *DropIndexQuery) Cascade() *DropIndexQuery {
+	q.cascade = true
+	return q
+}
+
 func (q *DropIndexQuery) Restrict() *DropIndexQuery {
 	q.restrict = true
 	return q

vendor/github.com/uptrace/bun/query_insert.go

@@ -24,6 +24,8 @@ type InsertQuery struct {
 	replace bool
 }
 
+var _ Query = (*InsertQuery)(nil)
+
 func NewInsertQuery(db *DB) *InsertQuery {
 	q := &InsertQuery{
 		whereBaseQuery: whereBaseQuery{
@@ -71,7 +73,7 @@ func (q *InsertQuery) TableExpr(query string, args ...interface{}) *InsertQuery
 }
 
 func (q *InsertQuery) ModelTableExpr(query string, args ...interface{}) *InsertQuery {
-	q.modelTable = schema.SafeQuery(query, args)
+	q.modelTableName = schema.SafeQuery(query, args)
 	return q
 }
 
@@ -159,6 +161,10 @@ func (q *InsertQuery) Operation() string {
 }
 
 func (q *InsertQuery) AppendQuery(fmter schema.Formatter, b []byte) (_ []byte, err error) {
+	if q.err != nil {
+		return nil, q.err
+	}
+
 	fmter = formatterWithModel(fmter, q)
 
 	b, err = q.appendWith(fmter, b)

vendor/github.com/uptrace/bun/query_select.go

@@ -34,6 +34,8 @@ type SelectQuery struct {
 	union []union
 }
 
+var _ Query = (*SelectQuery)(nil)
+
 func NewSelectQuery(db *DB) *SelectQuery {
 	return &SelectQuery{
 		whereBaseQuery: whereBaseQuery{
@@ -90,7 +92,7 @@ func (q *SelectQuery) TableExpr(query string, args ...interface{}) *SelectQuery
 }
 
 func (q *SelectQuery) ModelTableExpr(query string, args ...interface{}) *SelectQuery {
-	q.modelTable = schema.SafeQuery(query, args)
+	q.modelTableName = schema.SafeQuery(query, args)
 	return q
 }
 
@@ -342,9 +344,9 @@ func (q *SelectQuery) selectJoins(ctx context.Context, joins []relationJoin) err
 		case schema.HasOneRelation, schema.BelongsToRelation:
 			err = q.selectJoins(ctx, j.JoinModel.getJoins())
 		case schema.HasManyRelation:
-			err = j.selectMany(ctx, q.db.NewSelect())
+			err = j.selectMany(ctx, q.db.NewSelect().Conn(q.conn))
 		case schema.ManyToManyRelation:
-			err = j.selectM2M(ctx, q.db.NewSelect())
+			err = j.selectM2M(ctx, q.db.NewSelect().Conn(q.conn))
 		default:
 			panic("not reached")
 		}
@@ -369,6 +371,10 @@ func (q *SelectQuery) AppendQuery(fmter schema.Formatter, b []byte) (_ []byte, e
 func (q *SelectQuery) appendQuery(
 	fmter schema.Formatter, b []byte, count bool,
 ) (_ []byte, err error) {
+	if q.err != nil {
+		return nil, q.err
+	}
+
 	fmter = formatterWithModel(fmter, q)
 
 	cteCount := count && (len(q.group) > 0 || q.distinctOn != nil)
@@ -767,7 +773,7 @@ func (q *SelectQuery) Count(ctx context.Context) (int, error) {
 	}
 
 	query := internal.String(queryBytes)
-	ctx, event := q.db.beforeQuery(ctx, qq, query, nil, q.model)
+	ctx, event := q.db.beforeQuery(ctx, qq, query, nil, query, q.model)
 
 	var num int
 	err = q.conn.QueryRowContext(ctx, query).Scan(&num)
@@ -778,6 +784,13 @@ func (q *SelectQuery) Count(ctx context.Context) (int, error) {
 }
 
 func (q *SelectQuery) ScanAndCount(ctx context.Context, dest ...interface{}) (int, error) {
+	if _, ok := q.conn.(*DB); ok {
+		return q.scanAndCountConc(ctx, dest...)
+	}
+	return q.scanAndCountSeq(ctx, dest...)
+}
+
+func (q *SelectQuery) scanAndCountConc(ctx context.Context, dest ...interface{}) (int, error) {
 	var count int
 	var wg sync.WaitGroup
 	var mu sync.Mutex
@@ -817,6 +830,21 @@ func (q *SelectQuery) ScanAndCount(ctx context.Context, dest ...interface{}) (in
 	return count, firstErr
 }
 
+func (q *SelectQuery) scanAndCountSeq(ctx context.Context, dest ...interface{}) (int, error) {
+	var firstErr error
+
+	if q.limit >= 0 {
+		firstErr = q.Scan(ctx, dest...)
+	}
+
+	count, err := q.Count(ctx)
+	if err != nil && firstErr == nil {
+		firstErr = err
+	}
+
+	return count, firstErr
+}
+
 func (q *SelectQuery) Exists(ctx context.Context) (bool, error) {
 	if q.err != nil {
 		return false, q.err
@@ -830,7 +858,7 @@ func (q *SelectQuery) Exists(ctx context.Context) (bool, error) {
 	}
 
 	query := internal.String(queryBytes)
-	ctx, event := q.db.beforeQuery(ctx, qq, query, nil, q.model)
+	ctx, event := q.db.beforeQuery(ctx, qq, query, nil, query, q.model)
 
 	var exists bool
 	err = q.conn.QueryRowContext(ctx, query).Scan(&exists)

vendor/github.com/uptrace/bun/query_table_create.go

@@ -24,6 +24,8 @@ type CreateTableQuery struct {
 	tablespace  schema.QueryWithArgs
 }
 
+var _ Query = (*CreateTableQuery)(nil)
+
 func NewCreateTableQuery(db *DB) *CreateTableQuery {
 	q := &CreateTableQuery{
 		baseQuery: baseQuery{
@@ -59,7 +61,7 @@ func (q *CreateTableQuery) TableExpr(query string, args ...interface{}) *CreateT
 }
 
 func (q *CreateTableQuery) ModelTableExpr(query string, args ...interface{}) *CreateTableQuery {
-	q.modelTable = schema.SafeQuery(query, args)
+	q.modelTableName = schema.SafeQuery(query, args)
 	return q
 }
 

vendor/github.com/uptrace/bun/query_table_drop.go

@@ -15,6 +15,8 @@ type DropTableQuery struct {
 	ifExists bool
 }
 
+var _ Query = (*DropTableQuery)(nil)
+
 func NewDropTableQuery(db *DB) *DropTableQuery {
 	q := &DropTableQuery{
 		baseQuery: baseQuery{
@@ -50,7 +52,7 @@ func (q *DropTableQuery) TableExpr(query string, args ...interface{}) *DropTable
 }
 
 func (q *DropTableQuery) ModelTableExpr(query string, args ...interface{}) *DropTableQuery {
-	q.modelTable = schema.SafeQuery(query, args)
+	q.modelTableName = schema.SafeQuery(query, args)
 	return q
 }
 
@@ -61,6 +63,11 @@ func (q *DropTableQuery) IfExists() *DropTableQuery {
 	return q
 }
 
+func (q *DropTableQuery) Cascade() *DropTableQuery {
+	q.cascade = true
+	return q
+}
+
 func (q *DropTableQuery) Restrict() *DropTableQuery {
 	q.restrict = true
 	return q

vendor/github.com/uptrace/bun/query_table_truncate.go

@@ -16,6 +16,8 @@ type TruncateTableQuery struct {
 	continueIdentity bool
 }
 
+var _ Query = (*TruncateTableQuery)(nil)
+
 func NewTruncateTableQuery(db *DB) *TruncateTableQuery {
 	q := &TruncateTableQuery{
 		baseQuery: baseQuery{
@@ -57,6 +59,11 @@ func (q *TruncateTableQuery) ContinueIdentity() *TruncateTableQuery {
 	return q
 }
 
+func (q *TruncateTableQuery) Cascade() *TruncateTableQuery {
+	q.cascade = true
+	return q
+}
+
 func (q *TruncateTableQuery) Restrict() *TruncateTableQuery {
 	q.restrict = true
 	return q

vendor/github.com/uptrace/bun/query_update.go

@@ -20,6 +20,8 @@ type UpdateQuery struct {
 	omitZero bool
 }
 
+var _ Query = (*UpdateQuery)(nil)
+
 func NewUpdateQuery(db *DB) *UpdateQuery {
 	q := &UpdateQuery{
 		whereBaseQuery: whereBaseQuery{
@@ -67,7 +69,7 @@ func (q *UpdateQuery) TableExpr(query string, args ...interface{}) *UpdateQuery
 }
 
 func (q *UpdateQuery) ModelTableExpr(query string, args ...interface{}) *UpdateQuery {
-	q.modelTable = schema.SafeQuery(query, args)
+	q.modelTableName = schema.SafeQuery(query, args)
 	return q
 }
 
@@ -170,6 +172,10 @@ func (q *UpdateQuery) Operation() string {
 }
 
 func (q *UpdateQuery) AppendQuery(fmter schema.Formatter, b []byte) (_ []byte, err error) {
+	if q.err != nil {
+		return nil, q.err
+	}
+
 	fmter = formatterWithModel(fmter, q)
 
 	b, err = q.appendWith(fmter, b)

vendor/github.com/uptrace/bun/query_values.go

@@ -16,7 +16,10 @@ type ValuesQuery struct {
 	withOrder bool
 }
 
-var _ schema.NamedArgAppender = (*ValuesQuery)(nil)
+var (
+	_ Query                   = (*ValuesQuery)(nil)
+	_ schema.NamedArgAppender = (*ValuesQuery)(nil)
+)
 
 func NewValuesQuery(db *DB, model interface{}) *ValuesQuery {
 	q := &ValuesQuery{

vendor/github.com/uptrace/bun/schema/formatter.go

@@ -177,8 +177,6 @@ type NamedArgAppender interface {
 	AppendNamedArg(fmter Formatter, b []byte, name string) ([]byte, bool)
 }
 
-//------------------------------------------------------------------------------
-
 type namedArgList struct {
 	arg  NamedArgAppender
 	next *namedArgList
@@ -219,13 +217,13 @@ func (a *namedArg) AppendNamedArg(fmter Formatter, b []byte, name string) ([]byt
 
 //------------------------------------------------------------------------------
 
-var _ NamedArgAppender = (*structArgs)(nil)
-
 type structArgs struct {
 	table *Table
 	strct reflect.Value
 }
 
+var _ NamedArgAppender = (*structArgs)(nil)
+
 func newStructArgs(fmter Formatter, strct interface{}) (*structArgs, bool) {
 	v := reflect.ValueOf(strct)
 	if !v.IsValid() {

vendor/github.com/uptrace/bun/schema/table.go

@@ -203,7 +203,7 @@ func (t *Table) fieldByGoName(name string) *Field {
 func (t *Table) initFields() {
 	t.Fields = make([]*Field, 0, t.Type.NumField())
 	t.FieldMap = make(map[string]*Field, t.Type.NumField())
-	t.addFields(t.Type, nil)
+	t.addFields(t.Type, "", nil)
 
 	if len(t.PKs) == 0 {
 		for _, name := range []string{"id", "uuid", "pk_" + t.ModelName} {
@@ -230,7 +230,7 @@ func (t *Table) initFields() {
 	}
 }
 
-func (t *Table) addFields(typ reflect.Type, baseIndex []int) {
+func (t *Table) addFields(typ reflect.Type, prefix string, index []int) {
 	for i := 0; i < typ.NumField(); i++ {
 		f := typ.Field(i)
 		unexported := f.PkgPath != ""
@@ -242,10 +242,6 @@ func (t *Table) addFields(typ reflect.Type, baseIndex []int) {
 			continue
 		}
 
-		// Make a copy so the slice is not shared between fields.
-		index := make([]int, len(baseIndex))
-		copy(index, baseIndex)
-
 		if f.Anonymous {
 			if f.Name == "BaseModel" && f.Type == baseModelType {
 				if len(index) == 0 {
@@ -258,7 +254,7 @@ func (t *Table) addFields(typ reflect.Type, baseIndex []int) {
 			if fieldType.Kind() != reflect.Struct {
 				continue
 			}
-			t.addFields(fieldType, append(index, f.Index...))
+			t.addFields(fieldType, "", withIndex(index, f.Index))
 
 			tag := tagparser.Parse(f.Tag.Get("bun"))
 			if _, inherit := tag.Options["inherit"]; inherit {
@@ -274,7 +270,7 @@ func (t *Table) addFields(typ reflect.Type, baseIndex []int) {
 			continue
 		}
 
-		if field := t.newField(f, index); field != nil {
+		if field := t.newField(f, prefix, index); field != nil {
 			t.addField(field)
 		}
 	}
@@ -315,10 +311,20 @@ func (t *Table) processBaseModelField(f reflect.StructField) {
 }
 
 //nolint
-func (t *Table) newField(f reflect.StructField, index []int) *Field {
+func (t *Table) newField(f reflect.StructField, prefix string, index []int) *Field {
-	sqlName := internal.Underscore(f.Name)
 	tag := tagparser.Parse(f.Tag.Get("bun"))
 
+	if prefix, ok := tag.Option("embed"); ok {
+		fieldType := indirectType(f.Type)
+		if fieldType.Kind() != reflect.Struct {
+			panic(fmt.Errorf("bun: embed %s.%s: got %s, wanted reflect.Struct",
+				t.TypeName, f.Name, fieldType.Kind()))
+		}
+		t.addFields(fieldType, prefix, withIndex(index, f.Index))
+		return nil
+	}
+
+	sqlName := internal.Underscore(f.Name)
 	if tag.Name != "" && tag.Name != sqlName {
 		if isKnownFieldOption(tag.Name) {
 			internal.Warn.Printf(
@@ -328,10 +334,10 @@ func (t *Table) newField(f reflect.StructField, index []int) *Field {
 		}
 		sqlName = tag.Name
 	}
-
 	if s, ok := tag.Option("column"); ok {
 		sqlName = s
 	}
+	sqlName = prefix + sqlName
 
 	for name := range tag.Options {
 		if !isKnownFieldOption(name) {
@@ -339,7 +345,7 @@ func (t *Table) newField(f reflect.StructField, index []int) *Field {
 		}
 	}
 
-	index = append(index, f.Index...)
+	index = withIndex(index, f.Index)
 	if field := t.fieldWithLock(sqlName); field != nil {
 		if indexEqual(field.Index, index) {
 			return field
@@ -795,7 +801,7 @@ func (t *Table) inlineFields(field *Field, seen map[reflect.Type]struct{}) {
 		f.GoName = field.GoName + "_" + f.GoName
 		f.Name = field.Name + "__" + f.Name
 		f.SQLName = t.quoteIdent(f.Name)
-		f.Index = appendNew(field.Index, f.Index...)
+		f.Index = withIndex(field.Index, f.Index)
 
 		t.fieldsMapMu.Lock()
 		if _, ok := t.FieldMap[f.Name]; !ok {
@@ -834,7 +840,7 @@ func (t *Table) AppendNamedArg(
 	fmter Formatter, b []byte, name string, strct reflect.Value,
 ) ([]byte, bool) {
 	if field, ok := t.FieldMap[name]; ok {
-		return fmter.appendArg(b, field.Value(strct).Interface()), true
+		return field.AppendValue(fmter, b, strct), true
 	}
 	return b, false
 }
@@ -853,13 +859,6 @@ func (t *Table) quoteIdent(s string) Safe {
 	return Safe(NewFormatter(t.dialect).AppendIdent(nil, s))
 }
 
-func appendNew(dst []int, src ...int) []int {
-	cp := make([]int, len(dst)+len(src))
-	copy(cp, dst)
-	copy(cp[len(dst):], src)
-	return cp
-}
-
 func isKnownTableOption(name string) bool {
 	switch name {
 	case "table", "alias", "select":
@@ -991,3 +990,10 @@ func softDeleteFieldUpdaterFallback(field *Field) func(fv reflect.Value, tm time
 		return field.ScanWithCheck(fv, tm)
 	}
 }
+
+func withIndex(a, b []int) []int {
+	dest := make([]int, 0, len(a)+len(b))
+	dest = append(dest, a...)
+	dest = append(dest, b...)
+	return dest
+}

vendor/github.com/uptrace/bun/version.go

@@ -2,5 +2,5 @@ package bun
 
 // Version is the current release version.
 func Version() string {
-	return "1.0.19"
+	return "1.0.20"
 }

vendor/golang.org/x/sys/unix/mkerrors.sh

@@ -261,6 +261,7 @@ struct ltchars {
 #include <linux/vm_sockets.h>
 #include <linux/wait.h>
 #include <linux/watchdog.h>
+#include <linux/wireguard.h>
 
 #include <mtd/ubi-user.h>
 #include <mtd/mtd-user.h>
@@ -606,6 +607,7 @@ ccflags="$@"
 		$2 ~ /^MTD/ ||
 		$2 ~ /^OTP/ ||
 		$2 ~ /^MEM/ ||
+		$2 ~ /^WG/ ||
 		$2 ~ /^BLK[A-Z]*(GET$|SET$|BUF$|PART$|SIZE)/ {printf("\t%s = C.%s\n", $2, $2)}
 		$2 ~ /^__WCOREFLAG$/ {next}
 		$2 ~ /^__W[A-Z0-9]+$/ {printf("\t%s = C.%s\n", substr($2,3), $2)}

vendor/golang.org/x/sys/unix/zerrors_linux.go

@@ -741,6 +741,7 @@ const (
 	ETH_P_QINQ2                                 = 0x9200
 	ETH_P_QINQ3                                 = 0x9300
 	ETH_P_RARP                                  = 0x8035
+	ETH_P_REALTEK                               = 0x8899
 	ETH_P_SCA                                   = 0x6007
 	ETH_P_SLOW                                  = 0x8809
 	ETH_P_SNAP                                  = 0x5
@@ -810,10 +811,12 @@ const (
 	FAN_EPIDFD                                  = -0x2
 	FAN_EVENT_INFO_TYPE_DFID                    = 0x3
 	FAN_EVENT_INFO_TYPE_DFID_NAME               = 0x2
+	FAN_EVENT_INFO_TYPE_ERROR                   = 0x5
 	FAN_EVENT_INFO_TYPE_FID                     = 0x1
 	FAN_EVENT_INFO_TYPE_PIDFD                   = 0x4
 	FAN_EVENT_METADATA_LEN                      = 0x18
 	FAN_EVENT_ON_CHILD                          = 0x8000000
+	FAN_FS_ERROR                                = 0x8000
 	FAN_MARK_ADD                                = 0x1
 	FAN_MARK_DONT_FOLLOW                        = 0x4
 	FAN_MARK_FILESYSTEM                         = 0x100
@@ -1827,6 +1830,8 @@ const (
 	PERF_MEM_BLK_DATA                           = 0x2
 	PERF_MEM_BLK_NA                             = 0x1
 	PERF_MEM_BLK_SHIFT                          = 0x28
+	PERF_MEM_HOPS_0                             = 0x1
+	PERF_MEM_HOPS_SHIFT                         = 0x2b
 	PERF_MEM_LOCK_LOCKED                        = 0x2
 	PERF_MEM_LOCK_NA                            = 0x1
 	PERF_MEM_LOCK_SHIFT                         = 0x18
@@ -1986,6 +1991,9 @@ const (
 	PR_SCHED_CORE_CREATE                        = 0x1
 	PR_SCHED_CORE_GET                           = 0x0
 	PR_SCHED_CORE_MAX                           = 0x4
+	PR_SCHED_CORE_SCOPE_PROCESS_GROUP           = 0x2
+	PR_SCHED_CORE_SCOPE_THREAD                  = 0x0
+	PR_SCHED_CORE_SCOPE_THREAD_GROUP            = 0x1
 	PR_SCHED_CORE_SHARE_FROM                    = 0x3
 	PR_SCHED_CORE_SHARE_TO                      = 0x2
 	PR_SET_CHILD_SUBREAPER                      = 0x24
@@ -2167,12 +2175,23 @@ const (
 	RTCF_NAT                                    = 0x800000
 	RTCF_VALVE                                  = 0x200000
 	RTC_AF                                      = 0x20
+	RTC_BSM_DIRECT                              = 0x1
+	RTC_BSM_DISABLED                            = 0x0
+	RTC_BSM_LEVEL                               = 0x2
+	RTC_BSM_STANDBY                             = 0x3
 	RTC_FEATURE_ALARM                           = 0x0
+	RTC_FEATURE_ALARM_RES_2S                    = 0x3
 	RTC_FEATURE_ALARM_RES_MINUTE                = 0x1
-	RTC_FEATURE_CNT                             = 0x3
+	RTC_FEATURE_BACKUP_SWITCH_MODE              = 0x6
+	RTC_FEATURE_CNT                             = 0x7
+	RTC_FEATURE_CORRECTION                      = 0x5
 	RTC_FEATURE_NEED_WEEK_DAY                   = 0x2
+	RTC_FEATURE_UPDATE_INTERRUPT                = 0x4
 	RTC_IRQF                                    = 0x80
 	RTC_MAX_FREQ                                = 0x2000
+	RTC_PARAM_BACKUP_SWITCH_MODE                = 0x2
+	RTC_PARAM_CORRECTION                        = 0x1
+	RTC_PARAM_FEATURES                          = 0x0
 	RTC_PF                                      = 0x40
 	RTC_UF                                      = 0x10
 	RTF_ADDRCLASSMASK                           = 0xf8000000
@@ -2532,6 +2551,8 @@ const (
 	SO_VM_SOCKETS_BUFFER_MIN_SIZE               = 0x1
 	SO_VM_SOCKETS_BUFFER_SIZE                   = 0x0
 	SO_VM_SOCKETS_CONNECT_TIMEOUT               = 0x6
+	SO_VM_SOCKETS_CONNECT_TIMEOUT_NEW           = 0x8
+	SO_VM_SOCKETS_CONNECT_TIMEOUT_OLD           = 0x6
 	SO_VM_SOCKETS_NONBLOCK_TXRX                 = 0x7
 	SO_VM_SOCKETS_PEER_HOST_VM_ID               = 0x3
 	SO_VM_SOCKETS_TRUSTED                       = 0x5
@@ -2826,6 +2847,13 @@ const (
 	WDIOS_TEMPPANIC                             = 0x4
 	WDIOS_UNKNOWN                               = -0x1
 	WEXITED                                     = 0x4
+	WGALLOWEDIP_A_MAX                           = 0x3
+	WGDEVICE_A_MAX                              = 0x8
+	WGPEER_A_MAX                                = 0xa
+	WG_CMD_MAX                                  = 0x1
+	WG_GENL_NAME                                = "wireguard"
+	WG_GENL_VERSION                             = 0x1
+	WG_KEY_LEN                                  = 0x20
 	WIN_ACKMEDIACHANGE                          = 0xdb
 	WIN_CHECKPOWERMODE1                         = 0xe5
 	WIN_CHECKPOWERMODE2                         = 0x98

vendor/golang.org/x/sys/unix/zerrors_linux_386.go

@@ -250,6 +250,8 @@ const (
 	RTC_EPOCH_SET                    = 0x4004700e
 	RTC_IRQP_READ                    = 0x8004700b
 	RTC_IRQP_SET                     = 0x4004700c
+	RTC_PARAM_GET                    = 0x40187013
+	RTC_PARAM_SET                    = 0x40187014
 	RTC_PIE_OFF                      = 0x7006
 	RTC_PIE_ON                       = 0x7005
 	RTC_PLL_GET                      = 0x801c7011
@@ -327,6 +329,7 @@ const (
 	SO_RCVTIMEO                      = 0x14
 	SO_RCVTIMEO_NEW                  = 0x42
 	SO_RCVTIMEO_OLD                  = 0x14
+	SO_RESERVE_MEM                   = 0x49
 	SO_REUSEADDR                     = 0x2
 	SO_REUSEPORT                     = 0xf
 	SO_RXQ_OVFL                      = 0x28

vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go

@@ -251,6 +251,8 @@ const (
 	RTC_EPOCH_SET                    = 0x4008700e
 	RTC_IRQP_READ                    = 0x8008700b
 	RTC_IRQP_SET                     = 0x4008700c
+	RTC_PARAM_GET                    = 0x40187013
+	RTC_PARAM_SET                    = 0x40187014
 	RTC_PIE_OFF                      = 0x7006
 	RTC_PIE_ON                       = 0x7005
 	RTC_PLL_GET                      = 0x80207011
@@ -328,6 +330,7 @@ const (
 	SO_RCVTIMEO                      = 0x14
 	SO_RCVTIMEO_NEW                  = 0x42
 	SO_RCVTIMEO_OLD                  = 0x14
+	SO_RESERVE_MEM                   = 0x49
 	SO_REUSEADDR                     = 0x2
 	SO_REUSEPORT                     = 0xf
 	SO_RXQ_OVFL                      = 0x28

vendor/golang.org/x/sys/unix/zerrors_linux_arm.go

@@ -257,6 +257,8 @@ const (
 	RTC_EPOCH_SET                    = 0x4004700e
 	RTC_IRQP_READ                    = 0x8004700b
 	RTC_IRQP_SET                     = 0x4004700c
+	RTC_PARAM_GET                    = 0x40187013
+	RTC_PARAM_SET                    = 0x40187014
 	RTC_PIE_OFF                      = 0x7006
 	RTC_PIE_ON                       = 0x7005
 	RTC_PLL_GET                      = 0x801c7011
@@ -334,6 +336,7 @@ const (
 	SO_RCVTIMEO                      = 0x14
 	SO_RCVTIMEO_NEW                  = 0x42
 	SO_RCVTIMEO_OLD                  = 0x14
+	SO_RESERVE_MEM                   = 0x49
 	SO_REUSEADDR                     = 0x2
 	SO_REUSEPORT                     = 0xf
 	SO_RXQ_OVFL                      = 0x28

vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go

@@ -247,6 +247,8 @@ const (
 	RTC_EPOCH_SET                    = 0x4008700e
 	RTC_IRQP_READ                    = 0x8008700b
 	RTC_IRQP_SET                     = 0x4008700c
+	RTC_PARAM_GET                    = 0x40187013
+	RTC_PARAM_SET                    = 0x40187014
 	RTC_PIE_OFF                      = 0x7006
 	RTC_PIE_ON                       = 0x7005
 	RTC_PLL_GET                      = 0x80207011
@@ -324,6 +326,7 @@ const (
 	SO_RCVTIMEO                      = 0x14
 	SO_RCVTIMEO_NEW                  = 0x42
 	SO_RCVTIMEO_OLD                  = 0x14
+	SO_RESERVE_MEM                   = 0x49
 	SO_REUSEADDR                     = 0x2
 	SO_REUSEPORT                     = 0xf
 	SO_RXQ_OVFL                      = 0x28

vendor/golang.org/x/sys/unix/zerrors_linux_mips.go

@@ -250,6 +250,8 @@ const (
 	RTC_EPOCH_SET                    = 0x8004700e
 	RTC_IRQP_READ                    = 0x4004700b
 	RTC_IRQP_SET                     = 0x8004700c
+	RTC_PARAM_GET                    = 0x80187013
+	RTC_PARAM_SET                    = 0x80187014
 	RTC_PIE_OFF                      = 0x20007006
 	RTC_PIE_ON                       = 0x20007005
 	RTC_PLL_GET                      = 0x401c7011
@@ -327,6 +329,7 @@ const (
 	SO_RCVTIMEO                      = 0x1006
 	SO_RCVTIMEO_NEW                  = 0x42
 	SO_RCVTIMEO_OLD                  = 0x1006
+	SO_RESERVE_MEM                   = 0x49
 	SO_REUSEADDR                     = 0x4
 	SO_REUSEPORT                     = 0x200
 	SO_RXQ_OVFL                      = 0x28

vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go

@@ -250,6 +250,8 @@ const (
 	RTC_EPOCH_SET                    = 0x8008700e
 	RTC_IRQP_READ                    = 0x4008700b
 	RTC_IRQP_SET                     = 0x8008700c
+	RTC_PARAM_GET                    = 0x80187013
+	RTC_PARAM_SET                    = 0x80187014
 	RTC_PIE_OFF                      = 0x20007006
 	RTC_PIE_ON                       = 0x20007005
 	RTC_PLL_GET                      = 0x40207011
@@ -327,6 +329,7 @@ const (
 	SO_RCVTIMEO                      = 0x1006
 	SO_RCVTIMEO_NEW                  = 0x42
 	SO_RCVTIMEO_OLD                  = 0x1006
+	SO_RESERVE_MEM                   = 0x49
 	SO_REUSEADDR                     = 0x4
 	SO_REUSEPORT                     = 0x200
 	SO_RXQ_OVFL                      = 0x28

vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go

@@ -250,6 +250,8 @@ const (
 	RTC_EPOCH_SET                    = 0x8008700e
 	RTC_IRQP_READ                    = 0x4008700b
 	RTC_IRQP_SET                     = 0x8008700c
+	RTC_PARAM_GET                    = 0x80187013
+	RTC_PARAM_SET                    = 0x80187014
 	RTC_PIE_OFF                      = 0x20007006
 	RTC_PIE_ON                       = 0x20007005
 	RTC_PLL_GET                      = 0x40207011
@@ -327,6 +329,7 @@ const (
 	SO_RCVTIMEO                      = 0x1006
 	SO_RCVTIMEO_NEW                  = 0x42
 	SO_RCVTIMEO_OLD                  = 0x1006
+	SO_RESERVE_MEM                   = 0x49
 	SO_REUSEADDR                     = 0x4
 	SO_REUSEPORT                     = 0x200
 	SO_RXQ_OVFL                      = 0x28

vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go

@@ -250,6 +250,8 @@ const (
 	RTC_EPOCH_SET                    = 0x8004700e
 	RTC_IRQP_READ                    = 0x4004700b
 	RTC_IRQP_SET                     = 0x8004700c
+	RTC_PARAM_GET                    = 0x80187013
+	RTC_PARAM_SET                    = 0x80187014
 	RTC_PIE_OFF                      = 0x20007006
 	RTC_PIE_ON                       = 0x20007005
 	RTC_PLL_GET                      = 0x401c7011
@@ -327,6 +329,7 @@ const (
 	SO_RCVTIMEO                      = 0x1006
 	SO_RCVTIMEO_NEW                  = 0x42
 	SO_RCVTIMEO_OLD                  = 0x1006
+	SO_RESERVE_MEM                   = 0x49
 	SO_REUSEADDR                     = 0x4
 	SO_REUSEPORT                     = 0x200
 	SO_RXQ_OVFL                      = 0x28

vendor/golang.org/x/sys/unix/zerrors_linux_ppc.go

@@ -305,6 +305,8 @@ const (
 	RTC_EPOCH_SET                    = 0x8004700e
 	RTC_IRQP_READ                    = 0x4004700b
 	RTC_IRQP_SET                     = 0x8004700c
+	RTC_PARAM_GET                    = 0x80187013
+	RTC_PARAM_SET                    = 0x80187014
 	RTC_PIE_OFF                      = 0x20007006
 	RTC_PIE_ON                       = 0x20007005
 	RTC_PLL_GET                      = 0x401c7011
@@ -382,6 +384,7 @@ const (
 	SO_RCVTIMEO                      = 0x12
 	SO_RCVTIMEO_NEW                  = 0x42
 	SO_RCVTIMEO_OLD                  = 0x12
+	SO_RESERVE_MEM                   = 0x49
 	SO_REUSEADDR                     = 0x2
 	SO_REUSEPORT                     = 0xf
 	SO_RXQ_OVFL                      = 0x28

vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go

@@ -309,6 +309,8 @@ const (
 	RTC_EPOCH_SET                    = 0x8008700e
 	RTC_IRQP_READ                    = 0x4008700b
 	RTC_IRQP_SET                     = 0x8008700c
+	RTC_PARAM_GET                    = 0x80187013
+	RTC_PARAM_SET                    = 0x80187014
 	RTC_PIE_OFF                      = 0x20007006
 	RTC_PIE_ON                       = 0x20007005
 	RTC_PLL_GET                      = 0x40207011
@@ -386,6 +388,7 @@ const (
 	SO_RCVTIMEO                      = 0x12
 	SO_RCVTIMEO_NEW                  = 0x42
 	SO_RCVTIMEO_OLD                  = 0x12
+	SO_RESERVE_MEM                   = 0x49
 	SO_REUSEADDR                     = 0x2
 	SO_REUSEPORT                     = 0xf
 	SO_RXQ_OVFL                      = 0x28

vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go

@@ -309,6 +309,8 @@ const (
 	RTC_EPOCH_SET                    = 0x8008700e
 	RTC_IRQP_READ                    = 0x4008700b
 	RTC_IRQP_SET                     = 0x8008700c
+	RTC_PARAM_GET                    = 0x80187013
+	RTC_PARAM_SET                    = 0x80187014
 	RTC_PIE_OFF                      = 0x20007006
 	RTC_PIE_ON                       = 0x20007005
 	RTC_PLL_GET                      = 0x40207011
@@ -386,6 +388,7 @@ const (
 	SO_RCVTIMEO                      = 0x12
 	SO_RCVTIMEO_NEW                  = 0x42
 	SO_RCVTIMEO_OLD                  = 0x12
+	SO_RESERVE_MEM                   = 0x49
 	SO_REUSEADDR                     = 0x2
 	SO_REUSEPORT                     = 0xf
 	SO_RXQ_OVFL                      = 0x28